From 69d085079cedc1df40061c157f05d24ffa549357 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 13:29:39 -0400 Subject: [PATCH] ci(python): run round-trip sweeps on latest only, move experimental legs to their own workflow (#1732) - Owner's ruling: the round-trip sweeps run on `latest` alone. Every other `test` leg adds `--ignore-glob='*roundtrip*'` (ROUNDTRIP_SWEEP_GLOB); every other `integration` leg drops the three `*roundtrip*_runtime.py` modules from its selection and adds that glob (ROUNDTRIP_RUNTIME_GLOB), since pytest ignores no path it is handed. `latest` keeps both whole. - Owner's option 2: free-threaded 3.14t and GraalPy move to a new workflow, experimental.yml ("Experimental Tests"), so neither `required-checks` nor any `workflow_run` consumer of "Unit Tests" waits on them. Check names unchanged; caps 140 (job) and 5/10/120 (steps); job-level `continue-on-error`; unit-tests.yml's `changes` script, byte for byte. - `test` keeps main's caps: 45 for the job, no step caps. - util/python_versions.py writes an `experimental` matrix output; util/update_python_versions.py counts its legs. - util/ci_timing.py reads experimental.yml as one of its other workflows, and reports H1, H2 and H4 on a non-blocking job (any of experimental.yml's, a `(KTLO)` leg, or an `(experimental)` leg from before the split) as warnings, so they do not fail the run. - workflows.rst, releasing.rst and the tier file describe it; line citations renumbered. Tests: test_sweeps_on_latest_only_1732.py pins the structure; test_workflows_at_a_glance_1732.py pins workflows.rst's workflow count, table and graph to .github/workflows/*.yml; test_ci_timing.py the warning levels. Closes #1732 --- .github/python-versions.toml | 19 +- .github/workflows/experimental.yml | 242 +++++++++++ .github/workflows/unit-tests.yml | 111 +++-- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 140 +++++-- tests/_dependency_gates.py | 6 +- tests/_tiers.py | 36 ++ tests/project/test_ci_timing.py | 100 ++++- .../project/test_ktlo_legs_stay_green_1672.py | 106 ++--- .../project/test_python_matrix_tiers_1633.py | 33 +- tests/project/test_python_versions_1660.py | 48 +-- .../test_python_versions_update_1660.py | 5 +- .../test_sweeps_on_latest_only_1732.py | 384 ++++++++++++++++++ .../test_workflows_at_a_glance_1732.py | 173 ++++++++ tests/test_tier_guard.py | 35 +- util/ci_timing.py | 66 ++- util/python_versions.py | 26 +- util/update_python_versions.py | 10 +- 18 files changed, 1310 insertions(+), 234 deletions(-) create mode 100644 .github/workflows/experimental.yml create mode 100644 tests/project/test_sweeps_on_latest_only_1732.py create mode 100644 tests/project/test_workflows_at_a_glance_1732.py diff --git a/.github/python-versions.toml b/.github/python-versions.toml index 378375b766..e4923f8701 100644 --- a/.github/python-versions.toml +++ b/.github/python-versions.toml @@ -5,10 +5,10 @@ # and proposes each change, and the ruleset 23497679 edit it needs, as a PR. # # Tiers: supported (blocking), ktlo (end-of-life, non-blocking), pre-release -# (non-blocking; none listed today) and experimental (non-blocking `test` legs; -# see below). `runner` is the image a version needs when ubuntu-latest cannot -# provide it. `latest` is the newest supported CPython, which the -# single-interpreter jobs run on. See docs/source/contributing/workflows.rst. +# (non-blocking; none listed today) and experimental (non-blocking, a workflow +# of their own; see below). `runner` is the image a version needs when +# ubuntu-latest cannot provide it. `latest` is the newest supported CPython, +# which the single-interpreter jobs run on. See docs/source/contributing/workflows.rst. latest = "3.15" @@ -30,11 +30,12 @@ latest = "3.15" # Experimental (#1660): free-threaded CPython (a `t` version, above) and # GraalPy, by its release line, spelt as setup-python reads them. Each runs as -# one non-blocking leg of unit-tests.yml's `test` job, named `(experimental)`, -# and in no other matrix: no wheel, conda build or required context. The weekly -# job has no source for either, so it keeps these lines as written, after the -# other versions of their table. Promoting one once its leg is green is a hand -# edit, which util/python_versions.py refuses until it has those three. +# one non-blocking leg of experimental.yml, named `(experimental)`, a workflow +# of its own that nothing waits on (#1732), and in no other matrix: +# no wheel, conda build or required context. The weekly job has no source for +# either, so it keeps these lines as written, after the other versions of their +# table. Promoting one once its leg is green is a hand edit, which +# util/python_versions.py refuses until it has those three. [versions.graalpy] "graalpy-25.4" = { tier = "experimental" } diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml new file mode 100644 index 0000000000..753fa308b1 --- /dev/null +++ b/.github/workflows/experimental.yml @@ -0,0 +1,242 @@ +name: Experimental Tests + +# The experimental tier (#1660): free-threaded CPython and GraalPy, one leg each, +# here and in no other workflow. #1732, the owner's option 2: nothing waits on +# these legs. They used to be legs of unit-tests.yml, first of `test` and then of +# a job of their own there, and either way the Unit Tests run stayed in progress +# until GraalPy finished, up to 140 minutes. Every `workflow_run` consumer of +# "Unit Tests" -- coverage-comment.yml, deploy-pages.yml, cron-vendor.yml and +# cron-conda.yml -- fires only once that run completes, so each waited too. A +# workflow of their own reports separately: no `workflow_run` names it, it has no +# `workflow_call` trigger, and a job cannot `needs:` one in another workflow. +# +# Its check names are the ones the legs had in unit-tests.yml, `Python +# (experimental)`. No required context names one: ruleset 23497679 requires +# `Required checks passed` and the `Compat` legs alone. +# +# The same events as unit-tests.yml's own, and the same `changes` job, so a +# docs-only pull request skips the legs here as it skips `test` there. +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +# Only pull-request runs may be superseded, as in unit-tests.yml: every other +# event gets a group of its own, by run id, so a push is never cancelled while +# pending behind the next one. +concurrency: + group: experimental-tests-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # unit-tests.yml's job of the same name, reading the one output this file needs. + python-versions: + name: Read the Python versions + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + experimental: ${{ steps.read.outputs.experimental }} + + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: | + .github/python-versions.toml + util/python_versions.py + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Read .github/python-versions.toml + id: read + run: python3 util/python_versions.py --github-output + + # unit-tests.yml's `changes`, whose comment says how a file is classified; the + # script is the same, byte for byte (tests/project/test_sweeps_on_latest_only_1732.py + # pins it). This workflow is never called, so its own pull_request run is every + # pull_request run. + changes: + name: Classify changed files + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + code: ${{ steps.classify.outputs.code }} + + steps: + - uses: actions/checkout@v7 + if: ${{ github.event_name == 'pull_request' }} + with: + fetch-depth: 2 + + - name: Classify the diff + id: classify + env: + OWN_PR: ${{ github.event_name == 'pull_request' }} + run: | + python3 - <<'PY' + import os + import subprocess + + CODE_DIRS = ('.github/', 'pcapkit/', 'tests/', 'util/', 'examples/') + + def is_docs(path): + if path.startswith(CODE_DIRS): + return False + return path.startswith('docs/') or path.endswith('.md') + + code = 'true' + if os.environ['OWN_PR'] != 'true': + print("Not this workflow's own pull_request run: everything runs.") + else: + # -z: names are NUL-terminated, so a name holding whitespace + # (or one git would otherwise quote) stays one path. + diff = subprocess.run(['git', 'diff', '--name-only', '-z', '--no-renames', 'HEAD^1', 'HEAD'], + capture_output=True, text=True) + files = [path for path in diff.stdout.split('\0') if path] + if diff.returncode != 0 or not files: + print(f'::warning title=Could not list the changed files::git diff exited {diff.returncode} ' + f'with {len(files)} files: {diff.stderr.strip()}. Treating the diff as code.') + else: + code_files = [path for path in files if not is_docs(path)] + print(f'{len(files)} changed files, {len(code_files)} of them code:') + for path in files: + print(f" {'code' if path in code_files else 'docs'} {path}") + if not code_files: + code = 'false' + print(f'code={code}') + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as file: + print(f'code={code}', file=file) + PY + + # Its legs never block, by the KTLO mechanics: every step that can fail has + # `continue-on-error: true`, so the job concludes success, and `Experimental + # summary` reports each failed step as a warning. A failure no step contains + # -- a lost runner -- fails the job, and its job-level `continue-on-error` + # then passes the run: "Prevents a workflow run from failing when a job + # fails" (docs.github.com, Workflow syntax, `jobs..continue-on-error`, + # whose own example is an experimental matrix leg). Its check run still reads + # `failure`, so the step-level guards stay for every failure a step can hold. + # + # A job past its `timeout-minutes` is cancelled (same page, + # `jobs..timeout-minutes`), and the page does not say whether the + # job-level guard covers that. So setup-python, the install and the tests + # carry caps of their own, 5, 10 and 120 minutes, under the job's 140: a step + # past its cap fails rather than cancels, and `continue-on-error` then applies + # as to any failure (actions/runner src/Runner.Worker/StepsRunner.cs:322-353, + # at 5f0b3ddcc). That leaves 5 minutes for the rest, which took at most 0.32, + # setup-python included, on every uncovered leg of runs + # 38569321543-38569522442. + # + # The tests' 120: GraalPy 25.4's tests step reached 26% of the unit tier's + # tests at its old 25-minute cap (job 115915976024; setup-python took 0.7 + # minutes, the install 3.3). Those tests hold 20-26% of the tier's time, and + # the tier less the round-trip sweeps, which this job never runs (see `Run + # unit tests`), holds 3.5-4.5 times as much, by per-module times from a local + # 3.14 run and the 3.14, 3.15 and 3.14t legs of run 38581834697: 88-113 + # minutes on GraalPy (estimate). 3.14t, which shares the caps, took 5.8 + # minutes over the whole unit tier in that run. + experimental: + name: Python ${{ matrix.python-version }} (experimental) + if: ${{ needs.changes.outputs.code != 'false' }} + needs: [changes, python-versions] + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + timeout-minutes: 140 + continue-on-error: true + # #1672: every annotation a failing step writes is a warning. + env: + ANNOTATION_LEVEL: warning + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.python-versions.outputs.experimental) }} + + steps: + - uses: actions/checkout@v7 + id: checkout + continue-on-error: true + + - uses: actions/setup-python@v7 + id: python + continue-on-error: true + timeout-minutes: 5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + # As unit-tests.yml `test`'s install, for the same reasons. + - name: Install package and test dependencies + id: install + if: ${{ steps.python.outcome == 'success' }} + continue-on-error: true + timeout-minutes: 10 + shell: bash + env: + LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) + run: | + python -m pip install -U pip setuptools wheel + if ! python -m pip install -e '.[test,DPKT,crypto,NGAP]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then + reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1) + if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi + echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." + exit 1 + fi + + - name: Report available parallelism + id: parallelism + if: ${{ steps.install.outcome == 'success' }} + continue-on-error: true + run: | + nproc + python -c "import os; print('cpu_count', os.cpu_count())" + + # unit-tests.yml `test`'s selection on every leg but `latest`: no + # experimental version is `latest`, which util/python_versions.py holds to + # a supported CPython, so the round-trip sweeps (#1732) are always left out + # here. + - name: Run unit tests + id: tests + if: ${{ steps.install.outcome == 'success' }} + continue-on-error: true + timeout-minutes: 120 + run: | + set -- -q -n auto --dist load \ + --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ + --ignore=tests/integration \ + --ignore-glob='*_runtime.py' \ + --ignore-glob='*_regression.py' \ + --ignore-glob='*roundtrip*' + python -m pytest "$@" + + # As `test`'s, under a name of its own: ci-timing.yml reads `test`'s + # `junit-unit-*` files alone, and this one is the record of GraalPy's + # runtime. + - name: Upload test timings + if: ${{ !cancelled() && steps.install.outcome == 'success' }} + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: junit-experimental-${{ matrix.python-version }} + path: ${{ runner.temp }}/junit.xml + retention-days: 14 + + # #1660: an experimental leg's report, as unit-tests.yml `test`'s `KTLO + # summary` is a KTLO leg's, and for the same reasons. + - name: Experimental summary + if: ${{ !cancelled() }} + continue-on-error: true + shell: bash + env: + LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::Experimental legs never block (#1660), so this job still succeeds; see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 689dc0e80f..2c2f3be47d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -89,16 +89,15 @@ jobs: # this workflow used to) therefore ran the whole matrix in every caller # and skipped the gate everywhere. An input is the only reliable way for a # reusable workflow to know how it was invoked. - name: Python ${{ matrix.python-version }}${{ matrix.experimental && ' (experimental)' || '' }}${{ matrix.ktlo && ' (KTLO)' || '' }} + name: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} needs: [changes, python-versions] runs-on: ${{ matrix.os || 'ubuntu-latest' }} timeout-minutes: 45 # #1672: the level of every annotation a failing step writes -- a warning on - # a KTLO or experimental leg, which must not show red, an error on a - # supported one. + # a KTLO leg, which must not show red, an error on a supported one. env: - ANNOTATION_LEVEL: ${{ (matrix.ktlo || matrix.experimental) && 'warning' || 'error' }} + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false # #1660: .github/python-versions.toml's CPython tiers, through the @@ -134,33 +133,18 @@ jobs: # A brownout fails the leg before any step runs, which no step-level # `continue-on-error` can contain, so this leg must go before then. # - # The experimental tier (#1660): free-threaded CPython and GraalPy, one - # leg each with `experimental: true`, here and in no other job. They - # never block, by the KTLO mechanics: the same step-level guard, and - # `Experimental summary` reports each failed step as a warning. Nor can - # they be a required context, as no `Compat` leg is experimental. - # - # Neither build's runtime here is measured, and a leg past this job's - # `timeout-minutes` is cancelled, which no step-level guard contains and - # `required-checks` refuses. So on these legs alone, setup-python, the - # install and the tests carry caps of their own, 5, 10 and 25 minutes: a - # step past its cap fails rather than cancels, and `continue-on-error` - # then applies as to any failure (actions/runner - # src/Runner.Worker/StepsRunner.cs:322-353, at 5f0b3ddcc). That leaves 5 - # minutes for the rest, which took at most 0.32, setup-python included, - # on every uncovered leg of runs 38569321543-38569522442. Every other - # leg's caps are 360, the documented maximum, so the job's 45 binds first. + # The experimental tier (#1660) runs in experimental.yml, a workflow of + # its own, so that nothing here waits on it (#1732). matrix: ${{ fromJSON(needs.python-versions.outputs.test) }} steps: - uses: actions/checkout@v7 id: checkout - continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} + continue-on-error: ${{ matrix.ktlo == true }} - uses: actions/setup-python@v7 id: python - continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} - timeout-minutes: ${{ matrix.experimental && 5 || 360 }} + continue-on-error: ${{ matrix.ktlo == true }} with: python-version: ${{ matrix.python-version }} cache: pip @@ -202,15 +186,14 @@ jobs: # than failing against a missing package. The install needs its own # interpreter: if setup-python failed, the image's `python` would be # tested under this leg's name. On a supported leg either fails the job; - # on a KTLO or experimental leg the annotation is a warning (#1672). + # on a KTLO leg the annotation is a warning (#1672). - name: Install package and test dependencies id: install if: ${{ steps.python.outcome == 'success' }} - continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} - timeout-minutes: ${{ matrix.experimental && 10 || 360 }} + continue-on-error: ${{ matrix.ktlo == true }} shell: bash env: - LEG: Python ${{ matrix.python-version }}${{ matrix.experimental && ' (experimental, non-blocking)' || '' }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} run: | python -m pip install -U pip setuptools wheel if ! python -m pip install -e '.[test,DPKT,crypto,NGAP]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then @@ -232,7 +215,7 @@ jobs: - name: Report available parallelism id: parallelism if: ${{ steps.install.outcome == 'success' }} - continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} + continue-on-error: ${{ matrix.ktlo == true }} run: | nproc python -c "import os; print('cpu_count', os.cpu_count())" @@ -240,22 +223,33 @@ jobs: - name: Run unit tests id: tests if: ${{ steps.install.outcome == 'success' }} - continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} - timeout-minutes: ${{ matrix.experimental && 25 || 360 }} + continue-on-error: ${{ matrix.ktlo == true }} # `--dist load` is pinned explicitly rather than left to default: a # future change to xdist's own default, or someone "optimising" to # `loadfile`/`loadscope`, cannot silently change how this suite is - # distributed. The coverage leg runs the identical selection. The - # durations table and JUnit file are #1538's per-test timings, uploaded - # by the last step of this job. + # distributed. The durations table and JUnit file are #1538's per-test + # timings, uploaded by the last step of this job. + # + # #1732, the owner's ruling: the round-trip sweeps -- every module whose + # path holds `roundtrip`, tests/_tiers.py's ROUNDTRIP_SWEEP_GLOB -- run + # on `latest` alone, which is also the coverage leg, so coverage keeps + # its selection. Every other leg ignores them, as do experimental.yml's. + # pytest matches the glob, made absolute, against each path it + # collects, with `*` spanning `/` (_pytest/main.py:437-461, pytest + # 9.1.1). The modules that import the sweeps' harnesses for their + # packet builders still import them. env: COVERAGE: ${{ matrix.coverage }} + SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }} run: | set -- -q -n auto --dist load \ --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ --ignore-glob='*_regression.py' + if [ "$SWEEPS" != "true" ]; then + set -- "$@" --ignore-glob='*roundtrip*' + fi if [ "$COVERAGE" = "true" ]; then # Logs every measured process and its parent, for the check below. COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="$PWD/coverage-processes.log" \ @@ -394,27 +388,6 @@ jobs: path: ${{ runner.temp }}/junit.xml retention-days: 14 - # #1660: an experimental leg's report, as `KTLO summary` below is a KTLO - # leg's, and for the same reasons. Its own step, so that one stays a - # single script shared by every KTLO job; it is skipped on a KTLO leg. - - name: Experimental summary - if: ${{ !cancelled() && matrix.experimental == true }} - continue-on-error: ${{ matrix.experimental == true }} - shell: bash - env: - LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) - STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::Experimental legs never block (#1660), so this job still succeeds; see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" - # #1672: the KTLO leg's report, and the last step of every KTLO matrix # job, in one script. Each step above that can fail on a KTLO leg has an # `id` and `continue-on-error: ${{ matrix.ktlo == true }}`, so its failure @@ -567,11 +540,21 @@ jobs: # duplication this change exists to remove, with nothing in the log to # say so. `--dist load` is pinned explicitly; see the `test` job above # for why. + # + # #1732, the owner's ruling: the round-trip sweeps here, the modules + # tests/_tiers.py's ROUNDTRIP_RUNTIME_GLOB names, run on `latest` alone, as + # `test`'s do. Every other leg adds the glob as an `--ignore-glob`, which + # covers a module a directory argument reaches, and drops each path or + # node ID the glob matches from the selection: pytest never applies an + # ignore to a path it was handed (`Dir.collect` asks `isinitpath` first, + # _pytest/main.py:564-584, pytest 9.1.1). - name: Run full test suite id: suite if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash + env: + SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }} run: | if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." @@ -582,8 +565,21 @@ jobs: exit 1 fi echo "Fixture-dependent selection: $selection" + # shellcheck disable=SC2086 + set -- $selection + if [ "$SWEEPS" != "true" ]; then + kept=() + for arg in "$@"; do + case "${arg%%::*}" in + *roundtrip*_runtime.py) ;; + *) kept+=("$arg") ;; + esac + done + set -- "${kept[@]}" --ignore-glob='*roundtrip*_runtime.py' + echo "Not latest, so without the round-trip sweeps (#1732): $*" + fi python -m pytest -q -n auto --dist load \ - --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" $selection + --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" "$@" # See the `test` job's step of the same name. - name: Upload test timings @@ -1864,8 +1860,9 @@ jobs: # sees such a job's failure as `failure` or `success`, so a KTLO-only # regression could block every merge. The one failure no step can contain is # a leg that never gets a runner: the 3.7 legs' deprecated ubuntu-22.04 image - # has brownouts from 2027-03-23 -- see the `test` job's matrix. `test`'s - # experimental legs (#1660) are non-blocking the same way. + # has brownouts from 2027-03-23 -- see the `test` job's matrix. The + # experimental legs (#1660) are in experimental.yml, which nothing here + # needs or waits on (#1732). # # The docs-only path. On a pull request whose diff touches nothing but # `docs/**` and `*.md` files outside the code directories (the `changes` diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 9f48fa71f7..8a79912510 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:467-481``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1721-1791``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1717-1787``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:500,685,850,1085``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1569-1588``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1565-1584``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 050ac13528..8c5bef23f6 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -3,7 +3,7 @@ GitHub Actions Workflows .. important:: - The twelve workflows under :file:`.github/workflows/` trigger each other, and + The thirteen workflows under :file:`.github/workflows/` trigger each other, and the chain that results is not visible from any single file -- reading one ``on:`` block never shows what a *different* workflow's completion goes on to start. This page is the repository-wide graph. The release *pipeline* @@ -37,6 +37,13 @@ At a Glance - -- - -- - ``workflow_call`` (reusable) + * - Experimental Tests + - :file:`experimental.yml` + - ``main`` + - ``main`` + - -- + - -- + - -- * - Python Compatibility - :file:`python-compatibility.yml` - ``main`` @@ -140,6 +147,7 @@ than starting a new one. CQ["CodeQL
codeql-analysis.yml"] UT["Unit Tests
unit-tests.yml"] + EX["Experimental Tests
experimental.yml"] PC["Python Compatibility
python-compatibility.yml"] LT["Lint
lint.yml"] VU["Vendor Update
cron-vendor.yml"] @@ -158,6 +166,9 @@ than starting a new one. PUSH --> UT PR --> UT + PUSH --> EX + PR --> EX + PUSH --> PC PR --> PC SCHED -->|04:00| PC @@ -285,8 +296,8 @@ repository target the same reusable workflow, with the same input: not one: ``gate`` (the full suite, on ``latest``), ``changelog`` (checks ``CHANGELOG.md`` against its source entry) and ``python-versions``, which reads ``gate``'s version (see `Python Versions`_ below). It skips the other -**eight** -- the five matrix jobs, ``test`` (eleven Python legs: five -supported, four KTLO, two experimental), ``integration`` (nine), +**eight** -- the five matrix jobs, ``test`` (nine Python legs: five +supported, four KTLO), ``integration`` (nine), ``engine-tests`` (one job for each of five engines, each looping over six Python versions), ``pypcap-parity`` (two) and ``unittest-ordering`` (eleven matrix cells), each of which has already run once for this commit @@ -298,7 +309,7 @@ and ``project-tests``, and ``required-checks`` (see `Required Status Checks`_ below), gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1557-1563``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1553-1559``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -444,27 +455,31 @@ CI Timing Review (`#1538 `__ **CI Timing** (:file:`ci-timing.yml`) runs :file:`util/ci_timing.py` every Monday, or on dispatch with a ``days`` window and an optional ``concurrency_hours`` runner-pool measurement. Through the API it lists every completed run in the window of Unit Tests, -Python Compatibility, Lint, CodeQL and GitHub Pages, a day per query because a -filtered listing stops at 1,000 runs (Pages completed 1,154 in the week to -2026-10-09). It then reads the jobs of an evenly spaced sample -- 150 push and 150 -pull-request runs of Unit Tests, 60 runs of each other workflow -- and the summary -says how many runs each sample holds and which span they cover. That is about 625 -calls (estimate) of ``GITHUB_TOKEN``'s 1,000 an hour, on a read-only token +Python Compatibility, Lint, CodeQL, GitHub Pages and Experimental Tests (#1732, so +its two-hour legs stay measured), a day per query because a filtered listing stops +at 1,000 runs (Pages completed 1,154 in the week to 2026-10-09). It then reads the +jobs of an evenly spaced sample -- 150 push and 150 pull-request runs of Unit Tests, +60 runs of each other workflow -- and the summary says how many runs each sample +holds and which span they cover. That is about 690 calls (estimate: 63 listing +pages, one per workflow per day and one more per further 100 runs, by the week to +2026-10-09's run counts with Experimental Tests taken as Unit Tests' 10, since it +runs on the same events; at most 300 + 300 job reads; 15 for the JUnit artifacts; +up to 12 workflow-file reads) of ``GITHUB_TOKEN``'s 1,000 an hour, on a read-only token (``actions: read``, ``contents: read``). A job is measured by its own outcome: one cancelled at 95% of its cap or later timed out and counts, even in a run that reads ``cancelled`` for that reason; one cancelled earlier was superseded and does not. A -step cap written as an expression, as three of ``test``'s are (#1660), differs by -leg and reads as none. +step cap written as an expression differs by leg and reads as none. The report goes to the job summary and to the ``ci-timing-report`` artifact (90 days), and the newest earlier report is the next run's baseline. It is not a ``workflow_run`` on Unit Tests, which would add a job per run to a runner pool measured saturated at 20 concurrent jobs. -Per-test timings come from the ``test`` and ``integration`` legs, which run pytest -with ``--durations=30 --durations-min=1.0`` and ``--junitxml`` and upload the file as -``junit-unit-`` or ``junit-integration-`` (14 days). Each upload -step is ``continue-on-error``, so an artifact-service outage cannot fail a leg that -``Required checks passed`` needs. pytest writes one ```` per test however +Per-test timings come from the ``test`` and ``integration`` legs, and from +:file:`experimental.yml`'s, which run pytest with ``--durations=30 +--durations-min=1.0`` and ``--junitxml`` and upload the file as +``junit-unit-``, ``junit-integration-`` or +``junit-experimental-`` (14 days). Each upload step is ``continue-on-error``, +so an artifact-service outage cannot fail a leg. pytest writes one ```` per test however many subtests it runs -- about 0.6 MB per unit leg (estimate) -- but the suite's ``tests=`` count includes subtests, so the review reads each testcase's ``time``. The ordering, ``gate``, ``engine-tests`` and ``pypcap-parity`` legs write no file, and nor @@ -536,6 +551,17 @@ while the pool stays saturated, and the first runs with JUnit will show whether ``W7``'s 10-second test line needs raising. What to do when one fires is on :doc:`conventions/process`. +``H1``, ``H2`` and ``H4`` on a non-blocking job are warnings, not HARD, so they do +not fail the run (#1732): on every job of :file:`experimental.yml`, which no +required context names, on every KTLO leg, a job whose name ends in ``(KTLO)`` +(#1633), and on a job whose name ends in ``(experimental)`` in any workflow. That +last covers the experimental legs that ran in :file:`unit-tests.yml` under +``test``'s 45-minute cap before the split, whose runs stay in the 7-day window for a +week after it. The finding keeps its ``H`` id and says it is non-blocking. A +KTLO leg of ``test``, ``integration`` or ``pypcap-parity`` past its job's +``timeout-minutes`` is still cancelled, which ``Required checks passed`` refuses, +so that warning is worth acting on before the cap. + Work Kept Off the Pull-Request Queue (`#1538 `__) ----------------------------------------------------------------------------------------------------- @@ -576,7 +602,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1884`` + - ``unit-tests.yml:1881`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -598,21 +624,21 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1884``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1881``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the five required legs are its supported tier (``python-versions.toml:20-24``), and its four KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1794`` mentions the string without defining it. +``unit-tests.yml:1790`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the five ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1793-1803``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1789-1799``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1883-1937``). Rulesets match check names literally, with no +(``unit-tests.yml:1880-1934``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the four legs as ``skipped`` only on a docs-only pull request, where @@ -621,7 +647,7 @@ instead. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1886``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1883``), so it is never produced -- and never expected -- on those paths. Python versions come in two tiers (#1633), and an experimental one (#1660, @@ -634,7 +660,7 @@ pull request red (#1672). Each KTLO leg shares its job with supported legs, in every such job works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:116-126``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:115-125``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -669,24 +695,54 @@ was measured on that job. The **experimental** tier is free-threaded CPython and GraalPy, spelt as setup-python reads them (``3.14t``, ``graalpy-25.4``). Each runs one leg of -``test``, named ``Python (experimental)``, and nothing else: no -``integration``, ``Compat``, wheel or conda leg, so never a required context. The -legs never block, by the KTLO mechanics above: ``test``'s guard is -``${{ matrix.ktlo == true || matrix.experimental == true }}``, their annotations -are warnings, and ``Experimental summary``, the step before ``KTLO summary``, -reports each failed step. A leg past the job's 45-minute ``timeout-minutes`` would -be cancelled, which no step-level guard contains and ``Required checks passed`` -refuses, and neither build's runtime here is measured. So on these legs alone, -setup-python, the install and the tests have caps of their own, of 5, 10 and 25 -minutes: a step at its cap fails rather than cancels, and the guard holds -(actions/runner ``src/Runner.Worker/StepsRunner.cs:322-353``). The runner's -``has timed out`` error stays, changing no conclusion, as ``Process completed with -exit code N.`` does. Everything but the install and the tests took under 20 -seconds on every uncovered leg of runs 38569321543-38569522442. Every other leg's -caps are 360, the documented maximum, so its job's 45 minutes bind first, as with -none. Promoting one whose leg stays green is a hand edit, -which :file:`util/python_versions.py` refuses until the build has a wheel, a conda -build and a ``Compat`` leg. +``experimental`` (two), the one matrix job of :file:`.github/workflows/experimental.yml` +("Experimental Tests"), named ``Python (experimental)``, and nothing +else: no ``integration``, ``Compat``, wheel or conda leg, so never a required +context. They were legs of ``test`` until `#1732 +`__, which moved them to a +workflow of their own, by the owner's ruling that nothing waits on them. **That +workflow reports separately and holds up nothing**: a job cannot ``needs:`` one in +another workflow, nothing calls it, and no ``workflow_run`` names it, so the Unit +Tests run completes without it, and with it ``Required checks passed``, +``unittest-ordering`` and the four workflows that wait on Unit Tests completing +(``coverage-comment.yml``, ``deploy-pages.yml``, ``cron-vendor.yml`` and +``cron-conda.yml``). It runs on Unit Tests' ``push`` and ``pull_request`` events, +skips a docs-only pull request by the same ``changes`` script, and cancels a +pull request's superseded run. The legs never +block, by the KTLO mechanics above: every step that can fail has +``continue-on-error: true``, their annotations are warnings, and ``Experimental +summary``, the job's last step, reports each failed step. A failure no step +contains, such as a lost runner, fails the job, and the job's own +``continue-on-error: true`` keeps it from failing the run: it "prevents a workflow +run from failing when a job fails" (`Workflow syntax +`__), +though the leg's check run still reads ``failure``. A job past its +``timeout-minutes`` is cancelled, and that page does not say whether the job-level +guard covers a cancellation. So setup-python, the install and the tests have caps +of their own, of 5, 10 and 120 minutes, under the job's 140: a step at its cap +fails rather than cancels, and the guard holds (actions/runner +``src/Runner.Worker/StepsRunner.cs:322-353``). The runner's ``has timed out`` error +stays, changing no conclusion, as ``Process completed with exit code N.`` does. +Everything but the install and the tests took under 20 seconds on every uncovered +leg of runs 38569321543-38569522442. ``test``'s caps are main's: 45 minutes for the +job, and none on a step. Promoting one whose leg stays green is a hand edit, which +:file:`util/python_versions.py` refuses until the build has a wheel, a conda build +and a ``Compat`` leg. + +The tests' 120 minutes are sized for GraalPy, the slower of the two legs that share +them: its leg reached 26% of the unit tier in 25 minutes (job 115915976024), where +3.14t's ran the whole tier in 5.8 (run 38581834697). By the owner's ruling the +round-trip sweeps run on ``latest`` alone: every leg but ``latest`` adds +``--ignore-glob='*roundtrip*'``, which leaves out each unit-tier module whose path +holds ``roundtrip`` (``ROUNDTRIP_SWEEP_GLOB`` in :file:`tests/_tiers.py`), and +``latest``, the coverage leg, runs them as before. The subset covers +``integration`` too: off ``latest`` its legs leave out the three +``*roundtrip*_runtime.py`` modules (``ROUNDTRIP_RUNTIME_GLOB``), dropping them from +``fixture_tier_paths()``'s selection and adding the glob as an ``--ignore-glob``, +since pytest applies no ignore to a path it was handed. On 2026-10-14 they were 41 modules +and 8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% on the +3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy 88-113 +minutes (estimate). Python Versions --------------- diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index 152d623814..e0988901b5 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -1478,7 +1478,11 @@ def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': answers it -- and :meth:`~tests.test_tier_guard.WorkflowAgreementTests\ .test_ignore_flags_match_the_fixture_tier_constants` - is what keeps that equivalence true. + is what keeps that equivalence true. It is the ``latest`` leg's: the + others also skip :data:`~tests._tiers.ROUNDTRIP_SWEEP_GLOB` (#1732), which + changes no answer, since every leg shares the job's one install line. The + same holds for ``'fixture'``, whose legs off ``latest`` drop + :data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB`'s modules. * ``'explicit'`` is exact membership in :attr:`Job.paths` -- the literal ``TEST_PATHS`` list a ``matrix.engine`` cell of ``engine-tests`` (#849) runs, at file granularity (that job names whole files, never node IDs). diff --git a/tests/_tiers.py b/tests/_tiers.py index f851ffe7b6..6dfd8ebe1e 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -46,6 +46,8 @@ which of their test methods) skip a generated capture rather than reading it for real, and the complete pytest selection the fixture-dependent tier needs in order to still exercise them. +* :func:`is_roundtrip_sweep` -- is this unit-tier module one of the round-trip + sweeps, which only the ``latest`` leg of the ``test`` job runs (#1732)? :file:`tests/conftest.py` drives the first four at collection time, and :func:`tests._support.sample_path` consults :func:`check_unit_tier_read` on @@ -118,6 +120,7 @@ from __future__ import annotations import ast +import fnmatch import functools import pathlib import subprocess @@ -130,6 +133,7 @@ 'GeneratedFixtureInUnitTierError', 'TierGuardWarning', 'SampleCall', 'ROOT', 'TESTS_ROOT', 'SAMPLE_ROOT', 'REGENERATE_SAMPLES_CMD', 'UNIT_TIER_SELECTION', 'FIXTURE_TIER_SUFFIXES', 'FIXTURE_TIER_DIRS', 'FIXTURE_TIER_SELECTOR', + 'ROUNDTRIP_SWEEP_GLOB', 'ROUNDTRIP_RUNTIME_GLOB', 'is_roundtrip_sweep', 'is_unit_tier', 'committed_captures', 'committed_capture_names', 'guard_unavailable_reason', 'handled_lines', 'sample_path_calls', 'audit_module', 'check_unit_tier_read', 'explain', @@ -172,6 +176,17 @@ 'python -c "from tests._tiers import fixture_tier_paths; ' 'print(\' \'.join(fixture_tier_paths()))"' ) +#: The round-trip sweeps (#1732), as the ``--ignore-glob`` that the ``test`` job +#: of :file:`.github/workflows/unit-tests.yml` adds to :data:`UNIT_TIER_SELECTION` +#: on every leg but ``latest``: a module whose path under :data:`ROOT` holds +#: ``roundtrip``. A selection within the unit tier, not a tier of its own: the +#: modules stay unit-tier, and ``latest`` runs them, under coverage. +ROUNDTRIP_SWEEP_GLOB = '*roundtrip*' +#: The round-trip sweeps of the fixture tier (#1732): the ``*_runtime.py`` modules +#: whose name holds ``roundtrip``, which the ``integration`` job runs on ``latest`` +#: alone. Every other leg drops them from :func:`fixture_tier_paths`'s selection and +#: passes this as an ``--ignore-glob``, since pytest ignores no path it was handed. +ROUNDTRIP_RUNTIME_GLOB = '*roundtrip*_runtime.py' #: Suffixes that make a tracked file worth suggesting as a replacement capture. #: Committedness itself is whatever git says -- this filter only keeps the #: suggestion in :func:`explain` from offering :file:`out.txt` as a capture. @@ -265,6 +280,27 @@ def is_unit_tier(path: 'pathlib.Path | str') -> 'bool': return FIXTURE_TIER_DIRS.isdisjoint(relative.parts[:-1]) +def is_roundtrip_sweep(path: 'pathlib.Path | str') -> 'bool': + """Whether ``path`` names a unit-tier round-trip sweep (#1732). + + The executable statement of :data:`ROUNDTRIP_SWEEP_GLOB`, matched the way + pytest matches an ``--ignore-glob`` run from :data:`ROOT`: the glob made + absolute, against the absolute path, with ``*`` spanning ``/``. + + Args: + path: Path to a module, absolute or relative to :data:`ROOT`. + + Returns: + :data:`True` for a unit-tier module the glob matches. + + """ + candidate = pathlib.Path(path) + if not candidate.is_absolute(): + candidate = ROOT / candidate + return is_unit_tier(candidate) and fnmatch.fnmatch(str(candidate.resolve()), + str(ROOT / ROUNDTRIP_SWEEP_GLOB)) + + def _git(*args: 'str') -> 'Optional[str]': """Run :program:`git` in :data:`ROOT` and return its standard output. diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py index 14f1a66164..867eaca3c1 100644 --- a/tests/project/test_ci_timing.py +++ b/tests/project/test_ci_timing.py @@ -29,6 +29,7 @@ from __future__ import annotations +import argparse import contextlib import datetime import importlib.util @@ -241,8 +242,15 @@ def test_the_real_unit_tests_workflow(self) -> 'None': ordering, 'Run tests/protocols and the root-level modules under plain unittest'), 20) caps = {spec['id']: spec['timeout'] for spec in specs} self.assertEqual((caps['test'], caps['integration'], caps['engine-tests']), (45, 45, 30)) - # #1660: its experimental legs' caps are expressions, so no leg's is known. + # #1732: the experimental legs are experimental.yml's, with caps of their own. self.assertIsNone(ci_timing.step_cap(test, ci_timing.TESTS_STEP)) + self.assertNotIn('experimental', [spec['id'] for spec in specs]) + own = ci_timing.parse_workflow((WORKFLOWS / 'experimental.yml').read_text(encoding='utf-8')) + for name in ('Python graalpy-25.4 (experimental)', 'Python 3.14t (experimental)'): + with self.subTest(job=name): + experimental = ci_timing.match_job(own, name) + self.assertEqual((experimental['id'], experimental['timeout'], + ci_timing.step_cap(experimental, ci_timing.TESTS_STEP)), ('experimental', 140, 120)) def test_a_cap_by_leg_reads_as_none(self) -> 'None': """An expression is no one cap; a literal beside it still reads.""" @@ -696,6 +704,96 @@ def test_the_documented_thresholds_are_the_evaluated_ones(self) -> 'None': self.assertEqual(documented, codes) +class TestNonBlockingLegs(unittest.TestCase): + """#1732: a cap finding on a leg nothing waits on is a warning, not HARD.""" + + #: (workflow, job name, its cap in the real workflow file). + EXPERIMENTAL = ('experimental.yml', 'Python graalpy-25.4 (experimental)', 140) + KTLO = ('unit-tests.yml', 'Python 3.10 (KTLO)', 45) + #: Experimental legs from before the split, run under ``test``'s 45-minute cap and + #: still in the 7-day window for a week after it merges. + LEGACY = (('unit-tests.yml', 'Python graalpy-25.4 (experimental)', 45), + ('unit-tests.yml', 'Python 3.14t (experimental)', 45)) + BLOCKING = ('unit-tests.yml', 'Python 3.14', 45) + + def levels(self, change: 'Callable[[dict], None]') -> 'list[tuple[str, str]]': + report, baseline = quiet() + change(report) + return [(item['id'], item['level']) for item in ci_timing.evaluate(report, baseline)] + + def test_each_cap_rule_only_warns_on_a_non_blocking_leg(self) -> 'None': + for workflow, name, cap in (self.EXPERIMENTAL, self.KTLO, *self.LEGACY): + def leg(row: 'dict') -> 'None': + row.update(workflow=workflow, name=name, cap=cap) + + def step(row: 'dict') -> 'None': + row.update(workflow=workflow, job=name, name=ci_timing.TESTS_STEP, cap=cap) + + cases = { + 'H1': lambda r: (leg(r['jobs'][0]), r['jobs'][0].update(p90=cap / 2)), + 'H2': lambda r: (step(r['steps'][0]), r['steps'][0].update(p90=cap / 2)), + 'H4 (job)': lambda r: (leg(r['jobs'][0]), r['jobs'][0].update(days_to_cap=14.0)), + 'H4 (step)': lambda r: (step(r['steps'][0]), r['steps'][0].update(days_to_cap=14.0)), + } + for code, change in cases.items(): + with self.subTest(job=name, rule=code): + self.assertEqual(self.levels(change), [(code.split()[0], 'warn')]) + + def test_a_blocking_leg_stays_hard(self) -> 'None': + workflow, name, cap = self.BLOCKING + self.assertFalse(ci_timing.non_blocking(workflow, name)) + self.assertEqual(self.levels(lambda r: r['jobs'][0].update(p90=cap / 2)), [('H1', 'hard')]) + self.assertEqual(self.levels(lambda r: r['steps'][0].update(p90=10.0)), [('H2', 'hard')]) + # The same name in experimental.yml would not be, and either suffix counts only + # at the end. + self.assertTrue(ci_timing.non_blocking('experimental.yml', name)) + self.assertFalse(ci_timing.non_blocking(workflow, 'Python 3.10 (KTLO) extra')) + self.assertFalse(ci_timing.non_blocking(workflow, 'Python 3.14t (experimental) extra')) + + def run_report(self, workflow: 'str', name: 'str', took: 'float') -> 'tuple[int, str]': + """``_report`` on five runs of one job taking ``took`` minutes, against the real file.""" + start = datetime.datetime(2026, 10, 9, 10, tzinfo=datetime.timezone.utc) + runs = [] + for number in range(ci_timing.MIN_SAMPLES): + begun = start + datetime.timedelta(hours=3 * number) + stamp_of = lambda moment: moment.strftime('%Y-%m-%dT%H:%M:%SZ') # noqa: E731 + runs.append({'id': number, 'event': 'push', 'conclusion': 'success', 'head_sha': 'a', + 'created_at': stamp_of(begun), 'jobs': [{ + 'name': name, 'conclusion': 'success', 'runner_name': 'GitHub Actions 1', + 'run_attempt': 1, 'created_at': stamp_of(begun), 'started_at': stamp_of(begun), + 'completed_at': stamp_of(begun + datetime.timedelta(minutes=took)), + 'steps': []}]}) + data = {'repo': REPO, 'since': '2026-10-03T00:00:00Z', 'until': '2026-10-10T00:00:00Z', + 'days': 7, 'junit': [], 'workflows': {workflow: {'runs': runs, 'specs': { + 'newest': ('a', (WORKFLOWS / workflow).read_text(encoding='utf-8'))}}}} + args = argparse.Namespace(baseline=None, out=None, summary=None) + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + status = ci_timing._report(args, ci_timing.Client('', transport=mock.Mock()), data) + return status, stdout.getvalue() + + def test_the_run_stays_green_on_a_non_blocking_leg_past_its_share(self) -> 'None': + for workflow, name, cap in (self.EXPERIMENTAL, self.KTLO, *self.LEGACY): + with self.subTest(job=name): + status, stdout = self.run_report(workflow, name, cap / 2 + 1) + self.assertEqual(status, 0) + self.assertIn('::warning title=CI timing H1::', stdout) + self.assertNotIn('::error', stdout) + self.assertIn(f'**WARN H1:** `{name}` ({workflow})', stdout) + self.assertIn(f'its {cap}-minute cap', stdout) + + def test_the_run_fails_on_a_blocking_leg_past_its_share(self) -> 'None': + workflow, name, cap = self.BLOCKING + status, stdout = self.run_report(workflow, name, cap / 2 + 1) + self.assertEqual(status, ci_timing.EXIT_HARD) + self.assertIn('::error title=CI timing H1::', stdout) + + def test_the_rule_is_documented(self) -> 'None': + page = ' '.join(DOC.read_text(encoding='utf-8').split()) + self.assertIn('``H1``, ``H2`` and ``H4`` on a non-blocking job are warnings', page) + self.assertIn('a job whose name ends in ``(experimental)``', page) + + def step_blocks(text: 'str', name: 'str') -> 'list[str]': """Every step called ``name``, from its ``- name:`` line to the next step or job.""" return re.findall(rf'(?ms)^ - name: {re.escape(name)}\n(.*?)(?=^ - |^ \S|\Z)', text) diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 224bccec85..20754e36f9 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -21,9 +21,10 @@ write annotations are executed. #1660 added an experimental tier, free-threaded CPython and GraalPy, as legs of -``test`` only. They are non-blocking by the same mechanics, under a guard that -holds on both tiers, and report through ``Experimental summary``, the step before -``KTLO summary``: :class:`TestAnExperimentalLegIsNonBlockingTheSameWay`. Their +``test`` only; #1732 moved them to a job of their own, ``experimental``, which +nothing gating needs. They are non-blocking by the same mechanics, every step +that can fail guarded, and report through ``Experimental summary``, that job's +last step: :class:`TestAnExperimentalLegIsNonBlockingTheSameWay`. Their setup-python, install and test steps have caps of their own, so a slow or hung leg fails a step rather than being cancelled with its job: :class:`TestASlowExperimentalLegTimesOutAStepNotTheJob`. @@ -58,7 +59,8 @@ KTLO_JOBS = tuple((name, job_id) for name, job_id, _ in KTLO_LEGS) #: The jobs whose steps write annotations of their own on a KTLO leg, so set the level. LEVELLED = (('unit-tests.yml', 'test'), ('unit-tests.yml', 'integration'), - ('unit-tests.yml', 'pypcap-parity'), ('python-compatibility.yml', 'compatibility')) + ('unit-tests.yml', 'pypcap-parity'), ('python-compatibility.yml', 'compatibility'), + EXPERIMENTAL_JOB) #: Steps that stop every leg alike, KTLO or not, by job. #1634's: a leg that cannot #: show its tree is the tag's must not go on to publish, as #: ``test_release_tags_the_run_sha_1634.TestPublishersCheckTheirTags`` pins. @@ -99,18 +101,23 @@ SUMMARY_IF = '${{ !cancelled() && matrix.ktlo == true }}' SUMMARY_ENV = {'LEG': "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}", 'STEPS': '${{ toJSON(steps) }}'} -#: ``test``'s report of an experimental leg (#1660), the step before ``KTLO summary``. +#: ``experimental``'s report of a leg (#1660), its last step; every leg is one, so it +#: tests no ``matrix`` flag (#1732). EXPERIMENTAL_SUMMARY = 'Experimental summary' -EXPERIMENTAL_SUMMARY_IF = '${{ !cancelled() && matrix.experimental == true }}' +EXPERIMENTAL_SUMMARY_IF = '${{ !cancelled() }}' EXPERIMENTAL_SUMMARY_ENV = {'LEG': 'Python ${{ matrix.python-version }} (experimental, non-blocking)', 'STEPS': '${{ toJSON(steps) }}'} #: Why each summary's warning says the job still succeeds; the scripts differ in nothing else. KTLO_REASON = 'KTLO legs never block (#1633), so this job still succeeds (#1672)' EXPERIMENTAL_REASON = 'Experimental legs never block (#1660), so this job still succeeds' -#: ``test``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``, and the -#: cap on an experimental leg: setup-python, the install and the tests, whose runtimes -#: on GraalPy and free-threaded CPython are unmeasured. -EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'tests': 25} +#: ``experimental``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``: +#: setup-python, the install and the tests. #1732: the tests' is GraalPy's unit tier +#: less the round-trip sweeps, 88-113 minutes (estimate) from the 26% its leg reached +#: in 25 (job 115915976024). +EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'tests': 120} +#: ``experimental``'s own ``timeout-minutes`` (#1732), and ``test``'s, main's 45 again. +EXPERIMENTAL_JOB_CAP = 140 +JOB_CAP = 45 #: The job's minutes those caps leave for the rest of an experimental leg. Everything #: but the install and the tests took at most 0.32 minutes, setup-python included, on #: every uncovered leg of runs 38569321543, 38569429406 and 38569522442. @@ -141,7 +148,7 @@ def summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': def experimental_summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': - """``test``'s ``Experimental summary`` step (#1660).""" + """``experimental``'s ``Experimental summary`` step (#1660).""" found = [step for step in spec['steps'] if step.get('name') == EXPERIMENTAL_SUMMARY] if len(found) != 1: raise AssertionError(f'expected one {EXPERIMENTAL_SUMMARY!r} step, found {len(found)}') @@ -149,7 +156,7 @@ def experimental_summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': def experimental_legs() -> 'list[dict[str, Any]]': - """``test``'s experimental legs, from the tier file.""" + """``experimental``'s legs, from the tier file (#1732).""" return [leg for leg in legs(*EXPERIMENTAL_JOB) if is_experimental(leg)] @@ -506,14 +513,15 @@ def test_every_failure_still_fails_a_supported_leg(self) -> None: for gated in re.findall(r'steps\.([\w-]+)\.outcome', str(step.get('if', ''))): self.assertFalse(continue_on_error(by_id[gated], context), gated) - def test_neither_summary_runs_on_a_supported_leg(self) -> None: - """``test`` has two, one per non-blocking tier (#1660).""" - spec = job(*EXPERIMENTAL_JOB) - position = spec['steps'].index(experimental_summary_step(spec)) - for leg in (leg for leg in legs(*EXPERIMENTAL_JOB) if not is_nonblocking(leg)): - for _, _, record in scenarios(*EXPERIMENTAL_JOB, spec, leg): + def test_no_summary_runs_on_a_supported_leg(self) -> None: + """``test`` has one, ``KTLO summary``; ``Experimental summary`` left it with its legs (#1732).""" + spec = job('unit-tests.yml', 'test') + self.assertNotIn(EXPERIMENTAL_SUMMARY, [label(step) for step in spec['steps']]) + self.assertIs(spec['steps'][-1], summary_step(spec)) + for leg in (leg for leg in legs('unit-tests.yml', 'test') if not is_nonblocking(leg)): + for _, _, record in scenarios('unit-tests.yml', 'test', spec, leg): with self.subTest(python=leg['python-version']): - self.assertEqual((record[position].outcome, record[-1].outcome), ('skipped', 'skipped')) + self.assertEqual(record[-1].outcome, 'skipped') def test_a_supported_engine_cell_still_fails_its_job_as_an_error(self) -> None: spec = job('unit-tests.yml', 'engine-tests') @@ -530,14 +538,15 @@ def test_a_supported_engine_cell_still_fails_its_job_as_an_error(self) -> None: class TestAnExperimentalLegIsNonBlockingTheSameWay(unittest.TestCase): - """#1660: ``test``'s experimental legs conclude ``success`` and warn, as (a) and (b) pin for KTLO.""" + """#1660: the experimental legs conclude ``success`` and warn, as (a) and (b) pin for KTLO.""" def setUp(self) -> None: self.spec = job(*EXPERIMENTAL_JOB) self.summary = experimental_summary_step(self.spec) self.position = self.spec['steps'].index(self.summary) self.legs = experimental_legs() - self.assertTrue(self.legs, 'the tier file gives `test` no experimental leg') + self.assertTrue(self.legs, 'the tier file gives `experimental` no leg') + self.assertTrue(all(map(is_experimental, legs(*EXPERIMENTAL_JOB)))) def test_no_step_can_fail_an_experimental_leg(self) -> None: for leg in self.legs: @@ -548,8 +557,9 @@ def test_no_step_can_fail_an_experimental_leg(self) -> None: self.assertEqual(conclusion, 'success') def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: - """It runs after any failure on an experimental leg, and only there; ``KTLO summary`` never does.""" - self.assertIs(self.spec['steps'][self.position + 1], summary_step(self.spec)) + """It runs after any failure on an experimental leg, as the job's last step.""" + self.assertIs(self.spec['steps'][-1], self.summary) + self.assertNotIn(SUMMARY, [label(step) for step in self.spec['steps']]) self.assertEqual(self.summary.get('if'), EXPERIMENTAL_SUMMARY_IF) self.assertEqual(self.summary.get('continue-on-error'), EXPERIMENTAL_GUARD) self.assertEqual(self.summary.get('shell'), 'bash') @@ -561,28 +571,19 @@ def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: continue with self.subTest(python=leg['python-version'], step=label(step)): self.assertNotEqual(record[self.position].outcome, 'skipped') - self.assertEqual(record[-1].outcome, 'skipped') if step.get('continue-on-error') is not True: self.assertIn('id', step) - for leg in filter(is_ktlo, legs(*EXPERIMENTAL_JOB)): - for _, _, record in scenarios(*EXPERIMENTAL_JOB, self.spec, leg): - with self.subTest(python=leg['python-version']): - self.assertEqual(record[self.position].outcome, 'skipped') # One script with the KTLO summary's, so a fix to either reaches both. self.assertIn(EXPERIMENTAL_REASON, self.summary['run']) self.assertEqual(self.summary['run'].replace(EXPERIMENTAL_REASON, KTLO_REASON), - summary_step(self.spec)['run']) + summary_step(job('unit-tests.yml', 'test'))['run']) def test_each_step_names_the_leg_by_its_tier(self) -> None: - """``LEG`` in an annotation's title: ``(experimental, non-blocking)``, ``(KTLO, non-blocking)``, or bare.""" + """``LEG`` in an annotation's title: ``(experimental, non-blocking)``.""" named = [step for step in self.spec['steps'] if 'LEG' in (step.get('env') or {})] - self.assertEqual([label(step) for step in named], - ['Install package and test dependencies', EXPERIMENTAL_SUMMARY, SUMMARY]) + self.assertEqual([label(step) for step in named], ['Install package and test dependencies', EXPERIMENTAL_SUMMARY]) for leg in legs(*EXPERIMENTAL_JOB): - never_runs = {EXPERIMENTAL_SUMMARY: not is_experimental(leg), SUMMARY: not is_ktlo(leg)} for step in named: - if never_runs.get(step['name'], False): - continue with self.subTest(python=leg['python-version'], step=label(step)): self.assertEqual(interpolate(step['env']['LEG'], {'matrix': leg}), leg_label(leg)) @@ -627,9 +628,8 @@ class TestASlowExperimentalLegTimesOutAStepNotTheJob(unittest.TestCase): """#1660: an experimental leg's slow or hung step fails, which its guard contains. A job past its ``timeout-minutes`` is cancelled, which no step-level - ``continue-on-error`` contains, and ``Required checks passed`` takes only - ``success`` from ``test`` on a code diff. A step past its own is failed - instead, and ``continue-on-error`` then applies to it as to any failure + ``continue-on-error`` contains, and docs.github.com does not say whether the + job-level one does. A step past its own is failed instead, and ``continue-on-error`` then applies to it as to any failure (actions/runner src/Runner.Worker/StepsRunner.cs:322-353, at 5f0b3ddcc) -- the path :class:`TestAnExperimentalLegIsNonBlockingTheSameWay` runs. @@ -637,42 +637,46 @@ class TestASlowExperimentalLegTimesOutAStepNotTheJob(unittest.TestCase): def setUp(self) -> None: self.spec = job(*EXPERIMENTAL_JOB) - self.job_cap = self.spec['timeout-minutes'] self.capped = [step for step in self.spec['steps'] if 'timeout-minutes' in step] def caps(self, leg: 'dict[str, Any]') -> 'dict[str, Any]': """Each capped step's ``timeout-minutes`` on ``leg``, by ``id``.""" return {step.get('id', label(step)): evaluate_expression(str(step['timeout-minutes']), {'matrix': leg}) - for step in self.capped} + if isinstance(step['timeout-minutes'], str) else step['timeout-minutes'] for step in self.capped} def test_an_experimental_legs_steps_end_inside_its_job(self) -> None: """Each cap a whole number under the job's, the install's under the tests', all of them with room left.""" self.assertTrue(experimental_legs()) for leg in experimental_legs(): - caps = self.caps(leg) + caps, job_cap = self.caps(leg), self.spec['timeout-minutes'] with self.subTest(python=leg['python-version']): self.assertEqual(caps, EXPERIMENTAL_CAPS) + self.assertEqual(job_cap, EXPERIMENTAL_JOB_CAP) for cap in caps.values(): self.assertIsInstance(cap, int) - self.assertTrue(0 < cap < self.job_cap, cap) + self.assertTrue(0 < cap < job_cap, cap) self.assertLess(caps['install'], caps['tests']) - self.assertLessEqual(sum(caps.values()) + UNCAPPED_MINUTES, self.job_cap) + self.assertLessEqual(sum(caps.values()) + UNCAPPED_MINUTES, job_cap) for step in self.capped: self.assertTrue(continue_on_error(step, {'matrix': leg}), label(step)) - def test_every_other_leg_keeps_the_jobs_cap(self) -> None: - """Each cap reads the documented maximum, so the job's 45 minutes bind first, as with none.""" - others = [leg for leg in legs(*EXPERIMENTAL_JOB) if not is_experimental(leg)] - self.assertTrue(others) - for leg in others: - with self.subTest(python=leg['python-version']): - self.assertEqual(self.caps(leg), dict.fromkeys(EXPERIMENTAL_CAPS, DOCUMENTED_MAXIMUM)) - self.assertGreaterEqual(DOCUMENTED_MAXIMUM, self.job_cap) + def test_test_keeps_mains_caps(self) -> None: + """#1732: ``test``'s job cap is 45 again, and none of its steps has one, so the 45 binds. + + main's step caps were 360, the documented maximum, on every leg but the + experimental ones, which leave with them. + + """ + spec = job('unit-tests.yml', 'test') + self.assertEqual(spec['timeout-minutes'], JOB_CAP) + self.assertEqual([label(step) for step in spec['steps'] if 'timeout-minutes' in step], []) + self.assertGreaterEqual(DOCUMENTED_MAXIMUM, JOB_CAP) def test_the_workflows_page_states_the_caps(self) -> None: page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') self.assertEqual(re.findall(r'caps of their own, of (\d+), (\d+) and (\d+) minutes', page), [tuple(str(cap) for cap in EXPERIMENTAL_CAPS.values())]) + self.assertEqual(re.findall(r"minutes, under the job's (\d+)", page), [str(EXPERIMENTAL_JOB_CAP)]) if __name__ == '__main__': diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index f14320c570..b84d719914 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -87,18 +87,16 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': KTLO_GUARD = '${{ matrix.ktlo == true }}' #: The job-level level of every annotation a failing step writes (#1672). ANNOTATION_LEVEL = "${{ matrix.ktlo && 'warning' || 'error' }}" -#: ``test``'s, which also hold on its experimental legs (#1660), and the guard of its -#: ``Experimental summary``, which runs on those legs only. -NONBLOCKING_GUARD = '${{ matrix.ktlo == true || matrix.experimental == true }}' -EXPERIMENTAL_GUARD = '${{ matrix.experimental == true }}' -NONBLOCKING_LEVEL = "${{ (matrix.ktlo || matrix.experimental) && 'warning' || 'error' }}" -#: A step's ``LEG``, the leg's name in its annotations, in a KTLO job and in ``test``. +#: The experimental job's guard, level and ``LEG`` (#1732): every leg of it is +#: experimental, so each is a literal. +EXPERIMENTAL_GUARD = True +EXPERIMENTAL_LEVEL = 'warning' +#: A step's ``LEG``, the leg's name in its annotations, in a KTLO job and in ``experimental``. LEG_TEMPLATE = "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}" -NONBLOCKING_LEG_TEMPLATE = ("Python ${{ matrix.python-version }}" - "${{ matrix.experimental && ' (experimental, non-blocking)' || '' }}" - "${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}") -#: The job holding experimental legs, the only one (#1660). -EXPERIMENTAL_JOB = ('unit-tests.yml', 'test') +EXPERIMENTAL_LEG_TEMPLATE = 'Python ${{ matrix.python-version }} (experimental, non-blocking)' +#: The job holding experimental legs, the only one (#1660), in a workflow of its own +#: since #1732, so that nothing in unit-tests.yml waits on it. +EXPERIMENTAL_JOB = ('experimental.yml', 'experimental') #: Ruleset 23497679's required contexts, which a tier change edits with the file. REQUIRED = frozenset({'Required checks passed'} | {f'Compat Python {v}' for v in SUPPORTED}) @@ -212,19 +210,19 @@ def is_nonblocking(leg: 'dict[str, Any]') -> 'bool': return is_ktlo(leg) or is_experimental(leg) -def guard(name: 'str', job_id: 'str') -> 'str': +def guard(name: 'str', job_id: 'str') -> 'Any': """The step-level guard of job ``job_id``'s failure-prone steps.""" - return NONBLOCKING_GUARD if (name, job_id) == EXPERIMENTAL_JOB else KTLO_GUARD + return EXPERIMENTAL_GUARD if (name, job_id) == EXPERIMENTAL_JOB else KTLO_GUARD def annotation_level(name: 'str', job_id: 'str') -> 'str': """Job ``job_id``'s ``ANNOTATION_LEVEL``.""" - return NONBLOCKING_LEVEL if (name, job_id) == EXPERIMENTAL_JOB else ANNOTATION_LEVEL + return EXPERIMENTAL_LEVEL if (name, job_id) == EXPERIMENTAL_JOB else ANNOTATION_LEVEL def leg_template(name: 'str', job_id: 'str') -> 'str': """Job ``job_id``'s ``LEG``, on each step that names the leg in an annotation.""" - return NONBLOCKING_LEG_TEMPLATE if (name, job_id) == EXPERIMENTAL_JOB else LEG_TEMPLATE + return EXPERIMENTAL_LEG_TEMPLATE if (name, job_id) == EXPERIMENTAL_JOB else LEG_TEMPLATE def leg_label(leg: 'dict[str, Any]') -> 'str': @@ -325,9 +323,8 @@ def test_no_ktlo_leg_can_fail_an_aggregated_job(self) -> None: continue # only ever on a supported leg: the coverage one, the cache saver if label in ('Checkout code', "Check the release tags name this run's commit"): continue # #1634: blocks every leg alike, pinned in its own test module - # Each summary's own guard: it runs on its tier's legs only. - expected = {'KTLO summary': KTLO_GUARD, - 'Experimental summary': EXPERIMENTAL_GUARD}.get(label, guard(name, job_id)) + # The summary's own guard: it runs on KTLO legs only. + expected = {'KTLO summary': KTLO_GUARD}.get(label, guard(name, job_id)) with self.subTest(workflow=name, job=job_id, step=label): self.assertIn(step.get('continue-on-error'), (True, expected)) diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index a914299796..c91ee19aac 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -284,9 +284,8 @@ def setups(spec: 'dict[str, Any]') -> 'list[dict[str, Any]]': } -#: ``test``'s name since #1660, which names an experimental leg too. -TEST_TEMPLATE = ("Python ${{ matrix.python-version }}${{ matrix.experimental && ' (experimental)' || '' }}" - "${{ matrix.ktlo && ' (KTLO)' || '' }}") +#: The experimental legs' job name (#1732), which ``test``'s was on those legs from #1660. +EXPERIMENTAL_TEMPLATE = 'Python ${{ matrix.python-version }} (experimental)' def render(template: 'str', leg: 'dict[str, Any]') -> 'str': @@ -586,18 +585,13 @@ def test_engine_tests_installs_and_runs_mains_cells(self) -> None: def test_no_job_name_changes_but_the_gates(self) -> None: """Rulesets match check names literally, so each template and leg name stays. - ``test``'s template gained the experimental tier's suffix (#1660), which - names every one of :data:`MAIN`'s legs as before. + ``test``'s template is :data:`MAIN`'s again since its experimental legs + left it for a job of their own (#1732). """ for (name, job_id), template in TEMPLATES_AT_MAIN.items(): with self.subTest(workflow=name, job=job_id): - if (name, job_id) == ('unit-tests.yml', 'test'): - self.assertEqual(workflow(name)['jobs'][job_id]['name'], TEST_TEMPLATE) - self.assertEqual([render(TEST_TEMPLATE, leg) for leg in legs(matrix(name, job_id, self.values))], - [render(template, leg) for leg in legs(matrix(name, job_id, self.values))]) - else: - self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) + self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) compat = TEMPLATES_AT_MAIN[('python-compatibility.yml', 'compatibility')] compat_legs = legs(matrix('python-compatibility.yml', 'compatibility', self.values)) blocking = {render(compat, leg) for leg in compat_legs @@ -976,7 +970,10 @@ def experimental(data: 'dict[str, Any]') -> 'list[str]': class TestTheExperimentalTier(unittest.TestCase): - """#1660, the owner's option 1: GraalPy and free-threaded CPython run non-blocking ``test`` legs, nothing else. + """#1660, the owner's option 1: GraalPy and free-threaded CPython run non-blocking legs, nothing else. + + #1732, the owner's option 2: those legs are the ``experimental`` job of + experimental.yml, a workflow of its own, so nothing in unit-tests.yml waits on them. :mod:`tests.project.test_ktlo_legs_stay_green_1672` runs those legs' steps to show they cannot fail the job; this reads the file, the matrices and the names. @@ -1026,14 +1023,17 @@ def test_load_holds_the_tier_to_those_builds(self) -> None: with self.subTest(case=label), self.assertRaises(ValueError): script().load(write_tiers(self.tmp, dict(TIERS_WITH_EXPERIMENTAL, versions=data))) - def test_only_test_gains_a_leg_and_no_other_output_moves(self) -> None: - """No ``integration``, ``Compat``, parity, engine, wheel or conda leg, and no release count.""" + def test_only_the_experimental_output_gains_a_leg(self) -> None: + """No ``test``, ``integration``, ``Compat``, parity, engine, wheel or conda leg, and no release count.""" base, values = self.outputs(TIERS_AT_MAIN), self.outputs(TIERS_WITH_EXPERIMENTAL) - self.assertEqual({name: value for name, value in values.items() if name != 'test'}, - {name: value for name, value in base.items() if name != 'test'}) - self.assertEqual(legs(json.loads(values['test'])), legs(json.loads(base['test'])) + EXPERIMENTAL_LEGS) - - def test_on_main_no_job_but_test_runs_an_experimental_version(self) -> None: + self.assertEqual({name: value for name, value in values.items() if name != 'experimental'}, + {name: value for name, value in base.items() if name != 'experimental'}) + self.assertEqual(json.loads(base['experimental']), {'python-version': [], 'include': []}) + self.assertEqual(json.loads(values['experimental']), + {'python-version': ['3.14t', 'graalpy-25.4'], 'include': EXPERIMENTAL_LEGS}) + self.assertEqual(legs(json.loads(values['experimental'])), EXPERIMENTAL_LEGS) + + def test_on_main_no_job_but_experimental_runs_an_experimental_version(self) -> None: """Every matrix and setup-python input any workflow reads from the script, as the file stands.""" found = experimental(script().load()) self.assertTrue(found) @@ -1046,7 +1046,7 @@ def test_on_main_no_job_but_test_runs_an_experimental_version(self) -> None: continue read.append((name, job_id)) expected = [{'python-version': version, 'experimental': True} for version in found] \ - if (name, job_id) == ('unit-tests.yml', 'test') else [] + if (name, job_id) == ('experimental.yml', 'experimental') else [] with self.subTest(workflow=name, job=job_id): self.assertEqual([leg for leg in legs(matrix(name, job_id)) if leg['python-version'] in found], expected) @@ -1057,10 +1057,12 @@ def test_on_main_no_job_but_test_runs_an_experimental_version(self) -> None: self.assertFalse(set(re.findall(r'[\w.-]+', values[output])) & set(found)) def test_each_leg_is_named_experimental_and_never_required(self) -> None: + """#1732: the names ``test`` gave these legs, so no check name changes.""" found = experimental(script().load()) - template = workflow('unit-tests.yml')['jobs']['test']['name'] - self.assertEqual(template, TEST_TEMPLATE) - names = [render(template, leg) for leg in legs(matrix('unit-tests.yml', 'test'))] + self.assertEqual(workflow('experimental.yml')['jobs']['experimental']['name'], EXPERIMENTAL_TEMPLATE) + names = [render(workflow(name)['jobs'][job_id]['name'], leg) + for name, job_id in (('unit-tests.yml', 'test'), ('experimental.yml', 'experimental')) + for leg in legs(matrix(name, job_id))] self.assertEqual(len(names), len(set(names)), names) self.assertEqual([name for name in names if name.endswith(' (experimental)')], [f'Python {version} (experimental)' for version in found]) diff --git a/tests/project/test_python_versions_update_1660.py b/tests/project/test_python_versions_update_1660.py index ad3656d983..c24b4486c6 100644 --- a/tests/project/test_python_versions_update_1660.py +++ b/tests/project/test_python_versions_update_1660.py @@ -618,8 +618,9 @@ def test_a_run_that_changes_nothing_writes_nothing(self) -> None: root = self.scratch() plan, texts = updater().compute_plan(root, as_of_the_file(snapshot())) self.assertEqual((plan['changes'], texts, plan['notes']), ([], {}, [])) - self.assertIn('``test`` (eleven Python legs: five supported, four KTLO, two experimental)', - ' '.join((root / 'docs/source/contributing/workflows.rst').read_text(encoding='utf-8').split())) + page = ' '.join((root / 'docs/source/contributing/workflows.rst').read_text(encoding='utf-8').split()) + self.assertIn('``test`` (nine Python legs: five supported, four KTLO)', page) + self.assertIn('``experimental`` (two)', page) def test_a_tier_move_keeps_them_byte_for_byte_after_the_computed_versions(self) -> None: root = self.scratch() diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py new file mode 100644 index 0000000000..f76c42c6c9 --- /dev/null +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -0,0 +1,384 @@ +# -*- coding: utf-8 -*- +"""#1732: the round-trip sweeps run on ``latest`` alone. + +The owner's ruling, paraphrased: run a subset, and the round-trip sweeps need run +only on the latest version. So ``test``'s ``Run unit tests`` step adds +:data:`~tests._tiers.ROUNDTRIP_SWEEP_GLOB` as one more ``--ignore-glob`` on every +leg whose version is not ``latest`` -- the KTLO and older supported legs -- as +``experimental``'s does on each of its legs, and ``latest``, the coverage leg, keeps +the whole unit tier. The owner later ruled that the subset covers ``integration`` +too: off ``latest``, its ``Run full test suite`` drops the modules +:data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB` names, :class:`TestTheIntegrationSubset`. + +:class:`TestTheStepSelectsByLeg` runs the step's script on each leg the tier file +gives either job, with a ``python`` on ``PATH`` that records its arguments, so what +is checked is the selection bash builds, not what the text appears to say. +:class:`TestTheGlobIsTheSweeps` asks pytest itself what the glob ignores. + +The owner's second ruling, option 2: nothing waits on the experimental legs, which +with the caps GraalPy needs run for up to two hours. A job of unit-tests.yml would +still hold its run in progress, and with it every ``workflow_run`` consumer of +"Unit Tests", so they are a workflow of their own, experimental.yml: +:class:`TestTheExperimentalLegsAreAWorkflowOfTheirOwn`. + +""" + +from __future__ import annotations + +import fnmatch +import os +import pathlib +import re +import shutil +import subprocess # nosec B404 +import sys +import tempfile +import unittest +from typing import TYPE_CHECKING + +from tests import _tiers +from tests.project.test_python_matrix_tiers_1633 import REQUIRED, is_experimental, job, legs, workflow +from tests.project.test_python_versions_1660 import ROOT, live_outputs, prose +from tests.project.test_release_gates import NULL, evaluate_expression + +if TYPE_CHECKING: + from typing import Any + +#: The jobs and step that run the unit tier: ``test``, and the experimental legs' own (#1732). +JOB = ('unit-tests.yml', 'test') +EXPERIMENTAL = ('experimental.yml', 'experimental') +STEP = 'Run unit tests' +#: The job and step that run the fixture tier, which the subset covers too (#1732). +INTEGRATION = ('unit-tests.yml', 'integration') +INTEGRATION_STEP = 'Run full test suite' +#: The fixture tier's round-trip sweeps, by path, as the owner's ruling counted them. +RUNTIME_SWEEPS = ('tests/foundation/traceflow/test_traceflow_reread_roundtrip_runtime.py', + 'tests/protocols/test_capture_roundtrip_runtime.py', + 'tests/protocols/test_declared_length_roundtrip_runtime.py') +#: Every workflow that waits on "Unit Tests" completing, with ``workflow_run``. +CONSUMERS = ('coverage-comment.yml', 'deploy-pages.yml', 'cron-vendor.yml', 'cron-conda.yml') +#: Each job's ``timeout-minutes``, and its capped steps', by ``id``: ``test``'s are +#: main's, ``experimental``'s are sized for GraalPy (88-113 minutes, estimate). +CAPS = {JOB: (45, {}), EXPERIMENTAL: (140, {'python': 5, 'install': 10, 'tests': 120})} +#: The jobs whose result reaches a gate: ``required-checks`` and what it needs, and +#: ``unittest-ordering``, which runs on main's pushes and gates publishing. +GATING = ('required-checks', 'unittest-ordering') +#: The flag that leaves the sweeps out, as pytest receives it once bash has run. +SWEEP_FLAG = f'--ignore-glob={_tiers.ROUNDTRIP_SWEEP_GLOB}' +#: The unit tier's own flags, which every leg passes. +TIER_FLAGS = ('--ignore=tests/integration', '--ignore-glob=*_runtime.py', '--ignore-glob=*_regression.py') +#: A ``python`` that writes each argument on a line of ``$ARGS_FILE``, and hands a +#: ``-c`` to the real one, which is how ``integration`` asks tests/_tiers.py. +STUB = ('#!/bin/sh\nif [ "$1" = -c ]; then exec "$REAL_PYTHON" "$@"; fi\n' + 'printf \'%s\\n\' "$@" > "$ARGS_FILE"\n') +#: A directory holding round-trip sweeps and modules that only import their +#: harnesses' packet builders, for :class:`TestTheGlobIsTheSweeps` to collect. +PROBE = 'tests/protocols/internet' +#: ``$RUNNER_TEMP``, which only names the JUnit file here, so one value for every leg. +RUNNER_TEMP = '/home/runner/work/_temp' + + +def step(where: 'tuple[str, str]' = JOB) -> 'dict[str, Any]': + name = INTEGRATION_STEP if where == INTEGRATION else STEP + found = [entry for entry in job(*where)['steps'] if entry.get('name') == name] + if len(found) != 1: + raise AssertionError(f'expected one {name!r} step in {where}, found {len(found)}') + return found[0] + + +def stringify(value: 'object') -> 'str': + """``value`` as an ``env:`` entry holds it.""" + if value is NULL or value is None: + return '' + if isinstance(value, bool): + return 'true' if value else 'false' + return str(value) + + +def step_env(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'dict[str, str]': + """The step's ``env:`` on ``leg``, ``needs`` read from the tier file.""" + context = {'matrix': leg, 'needs': {'python-versions': {'outputs': live_outputs()}}} + return {key: re.sub(r'\$\{\{.*?\}\}', lambda match: stringify(evaluate_expression(match.group(0), context)), + str(value)) + for key, value in (step(where).get('env') or {}).items()} + + +def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'list[str]': + """The arguments the step of job ``where`` hands ``python`` on ``leg``.""" + with tempfile.TemporaryDirectory() as temp: + stub = pathlib.Path(temp, 'python') + stub.write_text(STUB, encoding='utf-8') + stub.chmod(0o755) + record = pathlib.Path(temp, 'args') + env = dict(os.environ, **step_env(leg, where), ARGS_FILE=str(record), RUNNER_TEMP=RUNNER_TEMP, + REAL_PYTHON=sys.executable, PATH=temp + os.pathsep + os.environ.get('PATH', '')) + # From the root, as the runner runs it: ``integration`` imports tests/_tiers.py. + result = subprocess.run(['bash', '-e', '-c', step(where)['run']], env=env, cwd=ROOT, # nosec B603 B607 + capture_output=True, text=True, check=False, timeout=60) + if result.returncode: + raise AssertionError(f'the step failed on {leg}: {result.stderr}') + return record.read_text(encoding='utf-8').splitlines() + + +def pytest_args(argv: 'list[str]') -> 'list[str]': + """The arguments after ``-m pytest``, under ``coverage run`` or not.""" + for index in range(len(argv) - 1): + if argv[index:index + 2] == ['-m', 'pytest']: + return argv[index + 2:] + raise AssertionError(f'no `-m pytest` in {argv}') + + +@unittest.skipIf(shutil.which('bash') is None, 'the step is a bash script') +class TestTheStepSelectsByLeg(unittest.TestCase): + """Each leg of ``test`` and ``experimental``, from the tier file, through the step's own script.""" + + @classmethod + def setUpClass(cls) -> None: + cls.latest = live_outputs()['latest'] + cls.runs = {leg['python-version']: (leg, run_step(leg, where)) + for where in (JOB, EXPERIMENTAL) for leg in legs(*where)} + assert len(cls.runs) == len(legs(*JOB)) + len(legs(*EXPERIMENTAL)), 'a version runs in both jobs' + + def test_only_latest_runs_the_sweeps(self) -> None: + self.assertIn(self.latest, self.runs) + self.assertTrue(any(is_experimental(leg) for leg, _ in self.runs.values())) + for version, (_, argv) in self.runs.items(): + with self.subTest(python=version): + self.assertEqual(SWEEP_FLAG in pytest_args(argv), version != self.latest) + + def test_latest_keeps_the_whole_unit_tier_under_coverage(self) -> None: + """Coverage is measured over the selection it was before #1732.""" + leg, argv = self.runs[self.latest] + self.assertIs(leg.get('coverage'), True) + self.assertEqual(argv[:4], ['-m', 'coverage', 'run', '--rcfile=.github/coverage.toml']) + flags = [arg for arg in pytest_args(argv) if arg.startswith('--ignore')] + self.assertEqual(flags, list(TIER_FLAGS)) + + def test_every_other_leg_runs_the_rest(self) -> None: + """The same selection as ``latest``, the sweeps' glob last, and no coverage.""" + expected = pytest_args(self.runs[self.latest][1]) + [SWEEP_FLAG] + for version, (leg, argv) in self.runs.items(): + if version == self.latest: + continue + with self.subTest(python=version): + self.assertNotIn('coverage', argv) + self.assertEqual(argv[:2], ['-m', 'pytest']) + self.assertEqual(pytest_args(argv), expected) + + +class TestTheGlobIsTheSweeps(unittest.TestCase): + """What the glob leaves out, by pytest's own collection.""" + + def collect(self, *extra: 'str') -> 'set[str]': + result = subprocess.run([sys.executable, '-m', 'pytest', '--collect-only', '-q', '-p', 'no:cacheprovider', + PROBE, *extra], cwd=ROOT, capture_output=True, text=True, # nosec B603 + check=False, timeout=300) + self.assertEqual(result.returncode, 0, result.stdout[-2000:] + result.stderr[-2000:]) + return {line.split('::', 1)[0] for line in result.stdout.splitlines() if '::' in line} + + def test_pytest_ignores_exactly_the_sweeps(self) -> None: + """Run from the root, as CI runs it: the modules the glob drops are :func:`~tests._tiers.is_roundtrip_sweep`'s.""" + everything = self.collect() + rest = self.collect(SWEEP_FLAG) + sweeps = {module for module in everything if _tiers.is_roundtrip_sweep(module)} + self.assertTrue(sweeps) + self.assertEqual(everything - rest, sweeps) + # A module that imports a sweep's harness for its packet builders is no sweep. + builders = [module for module in rest if 'tests.foundation import _roundtrip' + in (ROOT / module).read_text(encoding='utf-8')] + self.assertTrue(builders, f'no module under {PROBE} imports the harness without being a sweep') + + def test_the_sweeps_are_the_modules_named_for_a_round_trip(self) -> None: + """Every unit-tier test module whose name holds ``roundtrip``. + + That takes in every module that builds a table on the #1202 protocol-edge + or corekit harness. The foundation harness's registry module is named for + symmetry, so it runs on every leg; it took 1.8 s on 3.14 (measured). + + """ + modules = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / 'tests').rglob('test_*.py')) + sweeps = [module for module in modules if _tiers.is_roundtrip_sweep(module)] + self.assertEqual(sweeps, [module for module in modules + if 'roundtrip' in module.rsplit('/', 1)[1] and _tiers.is_unit_tier(module)]) + for harness in ('tests/protocols/_edge_roundtrip.py', 'tests/corekit/_roundtrip.py'): + users = [module for module in modules if _tiers.is_unit_tier(module) and re.search( + rf'^from {re.escape(harness[:-3].replace("/", "."))} import .*\b(?:Case|KnownFailureTable)\b', + (ROOT / module).read_text(encoding='utf-8'), re.M)] + with self.subTest(harness=harness): + self.assertTrue(users) + self.assertEqual([module for module in users if module not in sweeps], []) + + +class TestTheIntegrationSubset(unittest.TestCase): + """#1732: ``integration`` runs the fixture tier's round-trip sweeps on ``latest`` alone. + + Each leg from the tier file, through the step's own script, with + tests/_tiers.py really asked for the selection. + + """ + + @classmethod + def setUpClass(cls) -> None: + cls.latest = live_outputs()['latest'] + cls.selection = list(_tiers.fixture_tier_paths()) + cls.runs = {leg['python-version']: pytest_args(run_step(leg, INTEGRATION)) for leg in legs(*INTEGRATION)} + + @staticmethod + def flags() -> 'list[str]': + return ['-q', '-n', 'auto', '--dist', 'load', '--durations=30', '--durations-min=1.0', + f'--junitxml={RUNNER_TEMP}/junit.xml'] + + def test_the_glob_names_the_three_runtime_sweeps(self) -> None: + """On disk and in the selection: the three modules the ruling counted.""" + glob = _tiers.ROUNDTRIP_RUNTIME_GLOB + on_disk = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / 'tests').rglob('test_*.py') + if fnmatch.fnmatch(path.relative_to(ROOT).as_posix(), glob)) + self.assertEqual(on_disk, list(RUNTIME_SWEEPS)) + self.assertEqual(sorted({arg.split('::', 1)[0] for arg in self.selection + if fnmatch.fnmatch(arg.split('::', 1)[0], glob)}), list(RUNTIME_SWEEPS)) + + def test_only_latest_runs_them(self) -> None: + """``latest`` passes the whole selection; every other leg drops them and adds the glob.""" + self.assertIn(self.latest, self.runs) + self.assertGreater(len(self.runs), 1) + flag = f'--ignore-glob={_tiers.ROUNDTRIP_RUNTIME_GLOB}' + rest = [arg for arg in self.selection if arg.split('::', 1)[0] not in RUNTIME_SWEEPS] + self.assertLess(len(rest), len(self.selection)) + for version, args in self.runs.items(): + with self.subTest(python=version): + if version == self.latest: + self.assertEqual(args, self.flags() + self.selection) + else: + self.assertEqual(args, self.flags() + rest + [flag]) + + +def triggers(spec: 'dict[str, Any]') -> 'dict[str, Any]': + """A workflow's ``on:``, which PyYAML reads as the key :data:`True`.""" + found = spec.get('on', spec.get(True)) + return found if isinstance(found, dict) else {} + + +class TestTheExperimentalLegsAreAWorkflowOfTheirOwn(unittest.TestCase): + """#1732, option 2: nothing waits on the experimental legs, so they leave unit-tests.yml.""" + + def setUp(self) -> None: + self.unit = workflow('unit-tests.yml') + self.own = workflow('experimental.yml') + + def test_the_workflow_holds_every_experimental_leg_and_unit_tests_none(self) -> None: + found = legs(*EXPERIMENTAL) + self.assertEqual(self.own['name'], 'Experimental Tests') + self.assertEqual([leg['python-version'] for leg in found], ['3.14t', 'graalpy-25.4']) + self.assertTrue(all(map(is_experimental, found))) + self.assertEqual(self.own['jobs']['experimental']['name'], 'Python ${{ matrix.python-version }} (experimental)') + self.assertNotIn('experimental', self.unit['jobs']) + self.assertNotIn('experimental', self.unit['jobs']['python-versions']['outputs']) + for job_id, spec in self.unit['jobs'].items(): + if not isinstance(spec.get('strategy', {}).get('matrix'), str): + continue + with self.subTest(job=job_id): + self.assertFalse([leg for leg in legs('unit-tests.yml', job_id) if is_experimental(leg)]) + self.assertNotIn('experimental', str(spec.get('name'))) + + def test_the_same_events_and_its_own_concurrency(self) -> None: + """unit-tests.yml's push and pull_request, no ``workflow_call``, read-only, PR runs superseded.""" + events = triggers(self.own) + self.assertEqual(events, {key: value for key, value in triggers(self.unit).items() if key != 'workflow_call'}) + self.assertEqual(set(events), {'push', 'pull_request'}) + self.assertEqual(self.own['permissions'], {'contents': 'read'}) + concurrency = self.own['concurrency'] + self.assertEqual(concurrency['cancel-in-progress'], "${{ github.event_name == 'pull_request' }}") + self.assertIn("github.event_name == 'pull_request' && github.ref || github.run_id", concurrency['group']) + self.assertNotEqual(concurrency['group'], self.unit['concurrency']['group']) + + def test_it_skips_a_docs_only_diff_as_test_does(self) -> None: + """unit-tests.yml's ``changes`` script, byte for byte, gating the job as it gates ``test``.""" + def classify(spec: 'dict[str, Any]') -> 'str': + return next(entry['run'] for entry in spec['jobs']['changes']['steps'] if entry.get('id') == 'classify') + self.assertEqual(classify(self.own), classify(self.unit)) + spec = self.own['jobs']['experimental'] + self.assertEqual(spec['needs'], ['changes', 'python-versions']) + self.assertEqual(spec['if'], "${{ needs.changes.outputs.code != 'false' }}") + self.assertIn("needs.changes.outputs.code != 'false'", self.unit['jobs']['test']['if']) + + def test_nothing_waits_on_it(self) -> None: + """No ``workflow_run`` names it, nothing calls it, and no ``needs:`` anywhere names its job.""" + self.assertNotIn('workflow_call', triggers(self.own)) + consumers = [] + for path in sorted((ROOT / '.github' / 'workflows').glob('*.yml')): + spec = workflow(path.name) + waits = (triggers(spec).get('workflow_run') or {}).get('workflows', []) + if self.unit['name'] in waits: + consumers.append(path.name) + text = path.read_text(encoding='utf-8') + with self.subTest(workflow=path.name): + self.assertNotIn(self.own['name'], waits) + code = [line for line in text.splitlines() if not line.lstrip().startswith('#')] + self.assertFalse([line for line in code if 'experimental.yml' in line]) + for job_id, job_spec in (spec.get('jobs') or {}).items(): + needs = job_spec.get('needs', []) + self.assertNotIn('experimental', [needs] if isinstance(needs, str) else needs, job_id) + self.assertEqual(sorted(consumers), sorted(CONSUMERS)) + self.assertNotIn('needs.experimental', str(self.unit['jobs']['required-checks'])) + self.assertEqual(self.unit['jobs']['unittest-ordering']['needs'], ['test']) + + def test_the_caps(self) -> None: + """``test``'s are main's; ``experimental``'s fit GraalPy, each step's under the job's.""" + for where, (cap, steps) in CAPS.items(): + spec = job(*where) + with self.subTest(job=where): + self.assertEqual(spec['timeout-minutes'], cap) + self.assertEqual({entry.get('id'): entry['timeout-minutes'] + for entry in spec['steps'] if 'timeout-minutes' in entry}, steps) + self.assertLess(sum(steps.values()), cap) + + def test_a_failure_never_turns_the_run_red(self) -> None: + """Job-level ``continue-on-error`` there alone: a ``test`` leg's failure must still fail its gate.""" + spec = self.own['jobs']['experimental'] + self.assertIs(spec.get('continue-on-error'), True) + self.assertEqual(spec['env'], {'ANNOTATION_LEVEL': 'warning'}) + self.assertEqual(spec['steps'][-1]['name'], 'Experimental summary') + for name in ('unit-tests.yml', 'experimental.yml'): + with self.subTest(workflow=name): + self.assertEqual([job_id for job_id, entry in workflow(name)['jobs'].items() + if 'continue-on-error' in entry], ['experimental'] if name == 'experimental.yml' else []) + + def test_it_installs_what_test_installs(self) -> None: + """So tests/_dependency_gates.py, which reads unit-tests.yml alone, loses no gate by its leaving.""" + from tests import _dependency_gates # pylint: disable=import-outside-toplevel + self.assertEqual(_dependency_gates.WORKFLOW.name, 'unit-tests.yml') + installs = {} + for where in (JOB, EXPERIMENTAL): + found = [re.findall(r"pip install -e '\.\[([^]]*)\]'", str(entry.get('run', ''))) + for entry in job(*where)['steps']] + installs[where] = [extra for entries in found for extra in entries] + self.assertEqual(installs[EXPERIMENTAL], installs[JOB]) + self.assertEqual(len(installs[JOB]), 1) + self.assertIn('test', [found.name for found in _dependency_gates.pytest_jobs()]) + + def test_ci_timing_still_reads_its_legs(self) -> None: + """The weekly CI Timing review reads the workflow, so its two-hour legs stay measured.""" + from util import ci_timing # pylint: disable=import-outside-toplevel + self.assertIn('experimental.yml', ci_timing.OTHER_WORKFLOWS) + specs = ci_timing.parse_workflow((ROOT / '.github' / 'workflows' / 'experimental.yml').read_text(encoding='utf-8')) + for version in ('3.14t', 'graalpy-25.4'): + with self.subTest(python=version): + spec = ci_timing.match_job(specs, f'Python {version} (experimental)') + self.assertEqual((spec['id'], spec['timeout'], ci_timing.step_cap(spec, ci_timing.TESTS_STEP)), + ('experimental', 140, 120)) + + def test_no_required_context_names_a_leg(self) -> None: + names = {f'Python {leg["python-version"]} (experimental)' for leg in legs(*EXPERIMENTAL)} + self.assertEqual(len(names), 2) + self.assertFalse(names & REQUIRED) + + +class TestTheDocsSayIt(unittest.TestCase): + + def test_the_workflows_page_names_the_glob_and_the_leg(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + self.assertIn(f"every leg but ``latest`` adds ``--ignore-glob='{_tiers.ROUNDTRIP_SWEEP_GLOB}'``", page) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_workflows_at_a_glance_1732.py b/tests/project/test_workflows_at_a_glance_1732.py new file mode 100644 index 0000000000..17df551fa0 --- /dev/null +++ b/tests/project/test_workflows_at_a_glance_1732.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +"""The workflows page's count, "At a Glance" table and graph, against the files. + +:file:`docs/source/contributing/workflows.rst` opens by counting the workflows under +:file:`.github/workflows/`, tabulates each one's triggers, and draws them as a +graph. #1732 added :file:`experimental.yml` and left all three describing twelve, +so each is checked here against the ``on:`` block of every file: the count, one +table row per file with its name and triggers, and one graph node per file with +an edge for each ``push``, ``pull_request`` and ``workflow_run`` trigger. + +""" + +from __future__ import annotations + +import pathlib +import re +import unittest +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Optional + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / '.github' / 'workflows' +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' + +NUMBERS = {'ten': 10, 'eleven': 11, 'twelve': 12, 'thirteen': 13, 'fourteen': 14, + 'fifteen': 15, 'sixteen': 16} +DAYS = {'0': 'Sun', '1': 'Mon', '2': 'Tue', '3': 'Wed', '4': 'Thu', '5': 'Fri', '6': 'Sat', + '*': 'daily'} +#: Events the table gives a column of their own; the rest go under "other". +COLUMNS = ('push', 'pull_request', 'schedule', 'workflow_run') + + +def triggers(text: 'str') -> 'dict[str, str]': + """Each event of a workflow's ``on:`` block, with the text under it.""" + block = re.search(r'(?ms)^on:\s*\n(.*?)(?=^\S)', text) + assert block is not None, 'no on: block' + lines = [line for line in block.group(1).splitlines() if not line.lstrip().startswith('#')] + events = {} # type: dict[str, str] + current = None # type: Optional[str] + for line in lines: + key = re.match(r'^ ([\w-]+):', line) + if key: + current = key.group(1) + events[current] = '' + elif current is not None: + events[current] += line + '\n' + return events + + +def workflow_name(text: 'str') -> 'str': + """A workflow's top-level ``name:``, unquoted.""" + name = re.search(r'(?m)^name:\s*(.+?)\s*$', text) + assert name is not None, 'no name:' + return name.group(1).strip('"') + + +def expected_row(path: 'pathlib.Path', text: 'str') -> 'list[Any]': + """The seven cells the table should give this workflow file; "other" as a list.""" + events = triggers(text) + + def branches(event: 'str') -> 'str': + if event not in events: + return '--' + body = events[event] + if re.search(r'(?m)^ tags:', body): + return 'tags ' + ', '.join(f'``{tag}``' for tag in re.findall(r"- '([^']+)'", body)) + found = re.search(r'branches:\s*\[([^\]]*)\]', body) + assert found is not None, f'{event} has neither branches nor tags' + return ', '.join(f'``{item.strip()}``' for item in found.group(1).split(',')) + + if 'schedule' in events: + cron = re.search(r"cron:\s*'(\d+) (\d+) \* \* ([\d*])'", events['schedule']) + assert cron is not None, 'a schedule this test cannot read' + schedule = f'{DAYS[cron.group(3)]} {int(cron.group(2)):02d}:{int(cron.group(1)):02d}' + else: + schedule = '--' + if 'workflow_run' in events: + sources = re.search(r'workflows:\s*\[([^\]]*)\]', events['workflow_run']) + assert sources is not None + upstream = ', '.join(f'**{item.strip().strip(chr(34))}**' for item in sources.group(1).split(',')) + else: + upstream = '--' + other = sorted(event for event in events if event not in COLUMNS) + return [workflow_name(text), f':file:`{path.name}`', branches('push'), + branches('pull_request'), schedule, upstream, other or ['--']] + + +def table_rows(page: 'str') -> 'list[list[str]]': + """The "At a Glance" list-table's body rows, cell by cell.""" + section = page.split('At a Glance\n-----------\n', 1)[1].split('\n.. note::', 1)[0] + rows = [] # type: list[list[str]] + for line in section.splitlines(): + cell = re.match(r'^ (\*| ) - (.*)$', line) + if cell is None: + continue + if cell.group(1) == '*': + rows.append([]) + rows[-1].append(cell.group(2).strip()) + return rows[1:] + + +def graph(page: 'str') -> 'tuple[dict[str, str], set[str]]': + """The mermaid graph's workflow nodes, by file, and its edges.""" + section = page.split('.. mermaid::', 1)[1].split('\nThe double relationship', 1)[0] + nodes = {match.group(3): match.group(1) + for match in re.finditer(r'(?m)^ +(\w+)\["([^"<]+)
([\w.-]+\.yml)"\]$', section)} + edges = {' '.join(line.split()) for line in section.splitlines() if '-->' in line or '==>' in line} + return nodes, edges + + +def problems(page: 'str', files: 'dict[pathlib.Path, str]') -> 'list[str]': + """Every way the page disagrees with the workflow files.""" + found = [] # type: list[str] + count = re.search(r'The (\w+) workflows under :file:`\.github/workflows/`', ' '.join(page.split())) + if count is None or NUMBERS.get(count.group(1)) != len(files): + found.append(f'the note counts {count and count.group(1)!r} workflows; there are {len(files)}') + + rows = {row[1]: row for row in table_rows(page)} + nodes, edges = graph(page) + by_name = {workflow_name(text): nodes.get(path.name) for path, text in files.items()} + for path, text in sorted(files.items()): + want = expected_row(path, text) + key = f':file:`{path.name}`' + row = rows.pop(key, None) + if row is None: + found.append(f'no table row for {path.name}') + else: + other = sorted(re.findall(r'``(\w+)``', row[6])) or ['--'] + if row[:6] != want[:6] or other != want[6]: + found.append(f'{path.name}: the row reads {row}, the file gives {want}') + node = nodes.pop(path.name, None) + if node is None: + found.append(f'no graph node for {path.name}') + continue + events = triggers(text) + wanted = set() + if 'push' in events: + wanted.add(f'{"TAG" if "tags:" in events["push"] else "PUSH"} --> {node}') + if 'pull_request' in events: + wanted.add(f'PR --> {node}') + for upstream in re.findall(r'"([^"]+)"', events.get('workflow_run', '')): + wanted.add(f'{by_name.get(upstream.strip())} ==>|workflow_run: completed| {node}') + for edge in sorted(wanted - edges): + found.append(f'{path.name}: no graph edge {edge!r}') + found += [f'a table row for {key}, which is no file' for key in rows] + found += [f'a graph node for {name}, which is no file' for name in nodes] + return found + + +FILES = {path: path.read_text(encoding='utf-8') for path in sorted(WORKFLOWS.glob('*.yml'))} + + +class TestTheWorkflowsPage(unittest.TestCase): + + def test_the_page_describes_every_workflow_file(self) -> None: + self.assertEqual(problems(DOC.read_text(encoding='utf-8'), FILES), []) + + def test_a_missing_row_is_caught(self) -> None: + """The check bites: drop experimental.yml's row, node and count, and it fails.""" + page = DOC.read_text(encoding='utf-8') + row = re.search(r'(?m)^ \* - Experimental Tests\n(?: - .*\n)+', page) + self.assertIsNotNone(row) + assert row is not None + found = problems(page.replace(row.group(0), ''), FILES) + self.assertEqual(found, ['no table row for experimental.yml']) + found = problems(page.replace('The thirteen workflows', 'The twelve workflows'), FILES) + self.assertEqual(found, ["the note counts 'twelve' workflows; there are 13"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 10d0ce404e..8c36a2b222 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -188,19 +188,48 @@ def test_ignore_flags_match_the_fixture_tier_constants(self) -> None: The point of failure this catches: somebody adds a fourth fixture-dependent naming convention to the workflow and the guard goes on classifying those modules as unit-tier, flagging their perfectly - legal capture reads. + legal capture reads. The globs that are not a tier's are + :data:`~tests._tiers.ROUNDTRIP_SWEEP_GLOB` and + :data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB`, which only the legs other + than ``latest`` add (#1732), each a constant of its own. """ if not WORKFLOW.is_file(): self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') text = WORKFLOW.read_text(encoding='utf-8') - globs = set(re.findall(r"--ignore-glob='\*([^']+)'", text)) + globs = set(re.findall(r"--ignore-glob='([^']+)'", text)) directories = set(re.findall(r'--ignore=tests/(\S+)', text)) - self.assertEqual(globs, set(_tiers.FIXTURE_TIER_SUFFIXES)) + self.assertEqual(globs, {f'*{suffix}' for suffix in _tiers.FIXTURE_TIER_SUFFIXES} + | {_tiers.ROUNDTRIP_SWEEP_GLOB, _tiers.ROUNDTRIP_RUNTIME_GLOB}) self.assertEqual(directories, set(_tiers.FIXTURE_TIER_DIRS)) + def test_a_roundtrip_sweep_stays_in_the_unit_tier(self) -> None: + """#1732: the sweeps are a selection within the unit tier, not a tier. + + So the tier guard keeps auditing them, and ``latest`` runs them; a module + outside the unit tier is no sweep even when its name says round trip. + + """ + cases = { + 'tests/protocols/test_option_roundtrip_unit.py': True, + 'tests/protocols/misc/test_pcapng_roundtrip2_unit.py': True, + 'tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py': True, + 'tests/protocols/test_capture_roundtrip_runtime.py': False, + 'tests/integration/test_roundtrip_end_to_end.py': False, + 'tests/foundation/registry/test_registry_symmetry_unit.py': False, + 'tests/corekit/test_multidict.py': False, + 'examples/roundtrip/test_example.py': False, + } + for relative, expected in cases.items(): + with self.subTest(module=relative): + self.assertIs(_tiers.is_roundtrip_sweep(relative), expected) + if expected: + self.assertTrue(_tiers.is_unit_tier(relative)) + self.assertIs(_tiers.is_roundtrip_sweep(_tiers.ROOT / 'tests/protocols/test_option_roundtrip_unit.py'), + True) + def test_integration_job_selects_positively_by_asking_tiers_for_it(self) -> None: """The ``integration`` job's selection runs the other direction, so it is checked the other way. diff --git a/util/ci_timing.py b/util/ci_timing.py index 82b6231ff7..ad7a13a142 100644 --- a/util/ci_timing.py +++ b/util/ci_timing.py @@ -40,12 +40,14 @@ for the newest :data:`JUNIT_RUNS` successful push runs. Each testcase's own ``time`` is used; a suite's ``tests=`` count includes subtests and is not. -A default run makes about 625 API calls (estimate, for the week to 2026-10-09: ~60 -listing pages, 300 + 240 job reads, 15 for artifacts, up to 10 workflow reads), -inside ``GITHUB_TOKEN``'s 1,000 per hour; ``--max-calls`` stops it before it can -take more. ``--concurrency-hours`` adds the -runner-pool measurement, which costs one call per run of every workflow in that -window and is therefore off unless asked for. +A default run makes about 690 API calls (estimate, for the week to 2026-10-09: 63 +listing pages, one per workflow per day and one more per further 100 runs, from +that week's run counts, with Experimental Tests taken as Unit Tests' 10 since it +runs on the same events; at most 300 + 300 job reads; 15 for artifacts; up to 12 +workflow reads), inside ``GITHUB_TOKEN``'s 1,000 per hour; ``--max-calls`` stops +it before it can take more. ``--concurrency-hours`` adds the runner-pool +measurement, which costs one call per run of every workflow in that window and is +therefore off unless asked for. Standard library only, so the workflow runs it on a bare interpreter. The token is ``GH_TOKEN`` (or ``GITHUB_TOKEN``); locally, ``GH_TOKEN=$(gh auth token)`` works. @@ -88,9 +90,19 @@ UNIT_TESTS = 'unit-tests.yml' #: The other workflows on a pull request's path, read for job durations only. OTHER_WORKFLOWS = ('python-compatibility.yml', 'lint.yml', 'codeql-analysis.yml', - 'deploy-pages.yml') + 'deploy-pages.yml', 'experimental.yml') #: GitHub's own ``timeout-minutes`` when a job sets none. DEFAULT_TIMEOUT = 360 +#: Workflows of which no required context names a check, so every job is +#: non-blocking (#1732). +NON_BLOCKING_WORKFLOWS = ('experimental.yml',) +#: The suffix every KTLO leg's job name carries (#1633); such a leg's steps cannot +#: fail it (#1672). +KTLO_SUFFIX = ' (KTLO)' +#: The suffix an experimental leg's job name carried in ``unit-tests.yml`` before +#: #1732 moved it out; such a leg's steps could not fail it either, and its runs +#: stay in the window until they age out. +EXPERIMENTAL_SUFFIX = ' (experimental)' #: The aggregate job the ruleset requires; its completion is "mergeable". REQUIRED_JOB = 'Required checks passed' @@ -321,9 +333,9 @@ def parse_workflow(text: 'str') -> 'list[dict[str, Any]]': def _parse_steps(section: 'str') -> 'list[dict[str, Any]]': """The named steps of one job section, with any step-level cap. - A cap written as an expression differs by leg -- ``test``'s on its experimental - legs (#1660) -- and the job's spec does not say which leg a name ran, so it - reads as none, as an uncapped step's does. + A cap written as an expression differs by leg, and the job's spec does not say + which leg a name ran, so it reads as none, as an uncapped step's does. No + workflow writes one since #1732 moved the experimental legs out of ``test``. """ found = [] # type: list[dict[str, Any]] @@ -932,32 +944,52 @@ def _fmt(value: 'Optional[float]', digits: 'int' = 1) -> 'str': return '-' if value is None else f'{value:.{digits}f}' +def non_blocking(workflow: 'str', job: 'str') -> 'bool': + """Whether ``job``'s cap findings (H1, H2, H4) are warnings rather than HARD. + + Every job of :data:`NON_BLOCKING_WORKFLOWS` is non-blocking, and so is a KTLO leg + or an experimental leg from before the split, wherever it ran (#1732). + + """ + return (workflow in NON_BLOCKING_WORKFLOWS + or job.endswith(KTLO_SUFFIX) or job.endswith(EXPERIMENTAL_SUFFIX)) + + def evaluate(report: 'dict[str, Any]', baseline: 'Optional[dict[str, Any]]') -> 'list[dict[str, str]]': - """Every HARD (``H*``) and WARN (``W*``) threshold that fires on ``report``.""" + """Every HARD (``H*``) and WARN (``W*``) threshold that fires on ``report``. + + A cap finding on a :func:`non_blocking` job keeps its ``H`` id but is a warning. + + """ found = [] # type: list[dict[str, str]] - def fire(code: 'str', message: 'str') -> 'None': - found.append({'id': code, 'level': 'hard' if code[0] == 'H' else 'warn', 'message': message}) + def fire(code: 'str', message: 'str', soft: 'bool' = False) -> 'None': + if soft: + message += ' Non-blocking, so a warning (#1732).' + found.append({'id': code, 'level': 'hard' if code[0] == 'H' and not soft else 'warn', + 'message': message}) for row in report['jobs']: label = f'`{row["name"]}` ({row["workflow"]})' + soft = non_blocking(row['workflow'], row['name']) if row['n'] >= MIN_SAMPLES and row['p90'] >= HARD_CAP_SHARE * row['cap']: fire('H1', f'{label} p90 is {_fmt(row["p90"])} min, at or above ' - f'{HARD_CAP_SHARE:.0%} of its {row["cap"]}-minute cap.') + f'{HARD_CAP_SHARE:.0%} of its {row["cap"]}-minute cap.', soft) if row.get('days_to_cap') is not None and row['days_to_cap'] <= HARD_PROJECTION_DAYS: fire('H4', f'{label} reaches its {row["cap"]}-minute cap in ' f'{_fmt(row["days_to_cap"])} days on a linear fit ' - f'(+{_fmt(row["slope_per_day"], 2)} min/day).') + f'(+{_fmt(row["slope_per_day"], 2)} min/day).', soft) for row in report['steps']: if not row.get('cap'): continue label = f'`{row["job"]}` step `{row["name"]}`' + soft = non_blocking(row['workflow'], row['job']) if row['n'] >= MIN_SAMPLES and row['p90'] >= HARD_CAP_SHARE * row['cap']: fire('H2', f'{label} p90 is {_fmt(row["p90"])} min, at or above ' - f'{HARD_CAP_SHARE:.0%} of its {row["cap"]}-minute step cap.') + f'{HARD_CAP_SHARE:.0%} of its {row["cap"]}-minute step cap.', soft) if row.get('days_to_cap') is not None and row['days_to_cap'] <= HARD_PROJECTION_DAYS: fire('H4', f'{label} reaches its {row["cap"]}-minute step cap in ' - f'{_fmt(row["days_to_cap"])} days on a linear fit.') + f'{_fmt(row["days_to_cap"])} days on a linear fit.', soft) pr = report['runs'].get('pull_request', {}) mergeable = pr.get('mergeable', {}) diff --git a/util/python_versions.py b/util/python_versions.py index 0f57574c66..fe2b5f237f 100644 --- a/util/python_versions.py +++ b/util/python_versions.py @@ -16,8 +16,12 @@ ``test``, ``integration``, ``compatibility`` Both tiers. A KTLO leg carries ``ktlo: true``, and ``os`` where ``ubuntu-latest`` cannot provide it; ``test`` marks the ``latest`` leg - ``coverage: true``, and adds each experimental version as a leg carrying - ``experimental: true``. No other output holds an experimental version. + ``coverage: true``. +``experimental`` + Each experimental version as a leg carrying ``experimental: true``, for + experimental.yml's ``experimental`` job, a workflow of its own so that nothing + waits on it (#1732). + No other output holds an experimental version. ``pypcap-parity`` The versions below 3.12 that run on ``ubuntu-latest``. ``pypi`` @@ -207,13 +211,26 @@ def experimental_legs(data: 'dict[str, Any]') -> 'list[dict[str, Any]]': def test_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """unit-tests.yml's ``test``: both tiers, the one coverage leg on ``latest``, and the experimental legs.""" + """unit-tests.yml's ``test``: both tiers, and the one coverage leg on ``latest``.""" matrix = tiered(data) matrix['include'].insert(0, {'python-version': data['latest'], 'coverage': True}) - matrix['include'].extend(experimental_legs(data)) return matrix +def experimental_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': + """experimental.yml's ``experimental``: the experimental legs alone, out of ``test`` (#1732). + + ``Required checks passed`` and ``unittest-ordering`` wait on every leg of + ``test``, and every ``workflow_run`` consumer of "Unit Tests" on its run, so a + leg in that workflow that runs for two hours holds up every pull request. + The shape of every other matrix here, a ``python-version`` axis that + ``include`` marks, which keys each leg by its version. + + """ + found = experimental_legs(data) + return {'python-version': [leg['python-version'] for leg in found], 'include': found} + + def parity_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': """unit-tests.yml's ``pypcap-parity``: every version below :data:`PARITY_BELOW` it can run. @@ -244,6 +261,7 @@ def wheel_tag(version: 'str') -> 'str': OUTPUTS = { 'latest': lambda data: data['latest'], 'test': test_matrix, + 'experimental': experimental_matrix, 'integration': tiered, 'compatibility': tiered, 'pypcap-parity': parity_matrix, diff --git a/util/update_python_versions.py b/util/update_python_versions.py index 16d41b1ac3..c8325bc111 100644 --- a/util/update_python_versions.py +++ b/util/update_python_versions.py @@ -901,9 +901,9 @@ def counted(count: 'int', ends: 'Optional[tuple[Any, Any]]') -> 'Optional[tuple[ lambda f: counted(len(f['ktlo']), lines_of(f, 'ktlo'))), Claim(r'and the (\w+) ``Compat Python (3\.\d+)``-``(3\.\d+)`` legs', lambda f: counted(len(f['required']) - 1, span(f['supported']))), - Claim(r'``test`` \((\w+) Python legs: (\w+) supported, (\w+) KTLO, (\w+) experimental\)', - lambda f: (Word(f['legs']['test']), Word(len(f['supported'])), Word(len(f['ktlo'])), - Word(len(f['experimental'])))), + Claim(r'``test`` \((\w+) Python legs: (\w+) supported, (\w+) KTLO\)', + lambda f: (Word(f['legs']['test']), Word(len(f['supported'])), Word(len(f['ktlo'])))), + Claim(r'``experimental`` \((\w+)\)', lambda f: (Word(f['legs']['experimental']),)), Claim(r'``integration`` \((\w+)\)', lambda f: (Word(f['legs']['integration']),)), Claim(r'each looping over (\w+) Python versions', lambda f: (Word(f['engines']),)), Claim(r'``pypcap-parity`` \((\w+)\)', lambda f: (Word(f['legs']['pypcap-parity']),)), @@ -1007,7 +1007,9 @@ def make_facts(data: 'dict[str, Any]', values: 'dict[str, str]', workflows: 'dic 'experimental': [version for impl in data['versions'] for version in pv.tier(data, impl, 'experimental')], 'wheels': len(values['pypi-wheels'].split()), 'conda': len(json.loads(values['conda'])), - 'legs': {job_id: len(found) for (name, job_id), found in legs.items() if name == 'unit-tests.yml'}, + # #1732: experimental.yml's one job, which unit-tests.yml's prose counts with its own. + 'legs': {job_id: len(found) for (name, job_id), found in legs.items() + if name in ('unit-tests.yml', 'experimental.yml')}, 'engines': len(engines), 'required': required_contexts(workflows, values), 'tier_lines': tier_lines,