diff --git a/.github/coverage.toml b/.github/coverage.toml index c9e66a2e07..91aabab72e 100644 --- a/.github/coverage.toml +++ b/.github/coverage.toml @@ -1,5 +1,6 @@ -# Coverage configuration for the one coverage-measured leg of unit-tests.yml -# (`test`, Python 3.15), passed with `--rcfile`. Coverage reads a single config +# Coverage configuration for the coverage-measured legs of unit-tests.yml +# (`test`, Python 3.15) and each cell of `test-latest-tier`, whose parts the +# `coverage` job combines, passed with `--rcfile`. Coverage reads a single config # file, so this repeats pyproject.toml's [tool.coverage.*] settings and adds the # settings below on top. tests/project/test_coverage_rcfile.py asserts every # pyproject setting is repeated here unchanged. diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml index 00215d0841..73c60ebc78 100644 --- a/.github/workflows/coverage-comment.yml +++ b/.github/workflows/coverage-comment.yml @@ -1,7 +1,7 @@ name: Coverage Comment -# Posts the coverage the `test` job's `latest` leg measured in unit-tests.yml -# as a pull-request comment. Not from unit-tests.yml itself: that file is also a +# Posts the coverage unit-tests.yml's `coverage` job combined from its `latest` +# legs as a pull-request comment. Not from unit-tests.yml itself: that file is also a # reusable workflow whose callers grant `contents: read`, and a called workflow # can only narrow its caller's token, so `pull-requests: write` there would # break every caller, the release gate included. @@ -84,7 +84,7 @@ jobs: dir="$RUNNER_TEMP/coverage" if ! gh run download "$RUN_ID" --repo "$REPO" --name coverage-summary --dir "$dir"; then - echo "::notice::Run $RUN_ID uploaded no coverage-summary artifact (the coverage leg did not get that far)." + echo "::notice::Run $RUN_ID uploaded no coverage-summary artifact (the coverage job did not get that far)." exit 0 fi diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 72864ab6ca..2aadda5b09 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -40,11 +40,11 @@ jobs: # needs 3.11). Runs on every path, `gate-only` included, with no write # permission, since callers of this reusable workflow grant `contents: read`. # - # The other single-interpreter jobs (`unittest-ordering`, `changelog`, - # `project-tests`) run the script in a step of their own: setup-python takes - # an empty `python-version` as "the Python on PATH", with only a warning, so - # `unittest-ordering`'s `!cancelled()` would otherwise need a check that this - # job succeeded. + # The other single-interpreter jobs (`coverage`, `unittest-ordering`, + # `changelog`, `project-tests`) run the script in a step of their own: + # setup-python takes an empty `python-version` as "the Python on PATH", with + # only a warning, so the `!cancelled()` of `coverage` and `unittest-ordering` + # would otherwise need a check that this job succeeded. python-versions: name: Read the Python versions runs-on: ubuntu-latest @@ -100,17 +100,19 @@ jobs: # so nothing here waits on it (#1732). #1958, the owner's ruling E: PyPy's # legs run the same `ALL` tier as every CPython leg but `latest`. # - # `include` marks one leg `coverage: true`, `latest` (#1661), so coverage - # adds no leg to the queue. That leg runs slower than the rest: it runs the - # `LATEST` tier as well as `ALL`, under coverage. util/ci_timing.py's W3 - # tracks the gap. + # `include` marks one leg `coverage: true`, `latest` (#1661). It runs the + # `ALL` tier, as every leg does, under coverage; util/ci_timing.py's W3 + # weighs that against its paired legs. #2751: the `LATEST` tier runs in + # `test-latest-tier`, under coverage too, and `coverage` combines the two. # # #2243, the owner's option B: a test-only pull request runs the - # `latest` leg alone, the coverage leg, which runs the whole suite with - # `integration`'s `latest` leg and `project-tests`. See `required-checks`. + # `latest` leg alone, which with `test-latest-tier`, `integration`'s + # `latest` leg and `project-tests` runs the whole suite. See + # `required-checks`. matrix: ${{ fromJSON(needs.changes.outputs.test-only == 'true' && needs.python-versions.outputs.test-latest || needs.python-versions.outputs.test) }} - # `&test-steps`: `test-ktlo` runs these same steps, by YAML alias. + # `&test-steps`: `test-ktlo` and `test-latest-tier` run these same steps, + # by YAML alias. steps: &test-steps - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 id: checkout @@ -189,15 +191,16 @@ jobs: # # #1958, the test tiers: tests/_tiers.py's module_tier() puts every test # module in `ALL`, a curated list each stating `__tier__ = 'ALL'` (#1963), - # or else in `LATEST`, and `LATEST` runs on `latest` - # alone, which is also the coverage leg. Every other leg -- `test-ktlo`'s - # too -- passes each module latest_tier_paths() prints as an `--ignore`, - # which takes in the round-trip sweeps (#1732). + # or else in `LATEST`, and `LATEST` runs on `latest` alone. Every leg of + # `test` and `test-ktlo`, `latest`'s too, passes each module + # latest_tier_paths() prints as an `--ignore`, which takes in the + # round-trip sweeps (#1732). # - # #1954, the owner's option 3: the project pins, tests/_tiers.py's - # project_pin_paths(), run once per push in `project-tests`, so `latest` - # passes each as an `--ignore` too. They import no pcapkit, so coverage - # loses nothing. + # #2751: `test-latest-tier`'s cells, on `latest`, set `HALF`, and each + # runs the half latest_tier_half() prints by name instead. It leaves out + # the project pins, tests/_tiers.py's project_pin_paths(), which run + # once per push in `project-tests` (#1954, the owner's option 3). They + # import no pcapkit, so coverage loses nothing. # # #2005: on PyPy an xdist worker's RSS grows with every test it runs, so # the `ALL` tier took the workers past 20 GB. A smaller `-n` only hands @@ -212,12 +215,18 @@ jobs: # of its own, and `tests/protocols` then takes the remainder. env: COVERAGE: ${{ matrix.coverage }} - LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }} + HALF: ${{ matrix.half }} SHARDS: ${{ startsWith(matrix.python-version, 'pypy') && 'tests/protocols/internet tests/protocols/application tests/protocols/misc tests/protocols tests/corekit' || '' }} run: | - if [ "$LATEST" = "true" ]; then - if ! skipped=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then - echo "::${ANNOTATION_LEVEL} title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above." + skipped= + selected= + if [ -n "$HALF" ]; then + if ! selected=$(python -c "import sys; from tests._tiers import latest_tier_half; print(' '.join(latest_tier_half(int(sys.argv[1]))))" "$HALF"); then + echo "::${ANNOTATION_LEVEL} title=Could not compute the LATEST tier's half::tests._tiers.latest_tier_half($HALF) failed -- see the traceback above." + exit 1 + fi + if [ -z "$selected" ]; then + echo "::${ANNOTATION_LEVEL} title=No LATEST modules in half $HALF::tests._tiers.latest_tier_half($HALF) returned nothing. Refusing to run pytest with no module, which would run the whole suite." exit 1 fi elif ! skipped=$(python -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))"); then @@ -232,6 +241,9 @@ jobs: for module in $skipped; do set -- "$@" --ignore="$module" done + for module in $selected; do + set -- "$@" "$module" + done if [ "$COVERAGE" = "true" ]; then # Logs every measured process and its parent, for the check below. COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="$PWD/coverage-processes.log" \ @@ -243,20 +255,38 @@ jobs: python util/cap_file_size.py python -m pytest "$@" fi - # util/coverage_report.py (#1967) fails, posting no number, unless every - # measured process saved its data (#1387). coverage-comment.yml posts the - # PR comment from the artifact, since callers of this reusable workflow - # grant only `contents: read`. `!cancelled()` so a failing test still - # reports what it measured. #2030: its summary goes to the job's copy too. - - name: Report coverage + # #2751: a coverage leg measures one part of the suite -- `test`'s + # `latest` leg the `ALL` tier, each `test-latest-tier` cell a half of the + # `LATEST` one -- and the `coverage` job combines the parts. Each part's + # check, util/coverage_report.py's (#1967), fails its leg unless every + # measured process saved its data (#1387). `!cancelled()` so a failing + # test still hands on what it measured, with the tests step's outcome + # beside it. #2030: a failure's summary goes to the job's copy too. + - name: Check coverage data + id: coverage-check if: ${{ !cancelled() && matrix.coverage }} shell: bash env: - SHA: ${{ github.sha }} - PYTHON: ${{ matrix.python-version }} TESTS_OUTCOME: ${{ steps.tests.outcome }} JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md - run: python util/coverage_report.py + run: | + printf '%s\n' "$TESTS_OUTCOME" > coverage-tests-outcome + python util/coverage_report.py check + + # Only a part whose check passed, so `coverage` finds a lost one missing + # and posts no number, rather than reporting the rest as the whole. The + # data files are hidden ones, and `coverage` reads them in this run. + - name: Upload coverage data + if: ${{ !cancelled() && matrix.coverage && steps.coverage-check.outcome == 'success' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-data-${{ matrix.half && 'latest-half-' || 'all' }}${{ matrix.half }} + path: | + .coverage.* + coverage-tests-outcome + include-hidden-files: true + retention-days: 1 + if-no-files-found: error # #1423: this log is the only record of a measured process that saved no # data. Kept whenever the leg ran, not only on failure: a run that passed @@ -265,36 +295,21 @@ jobs: if: ${{ !cancelled() && matrix.coverage }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: coverage-processes-log + name: coverage-processes-log-${{ matrix.half && 'latest-half-' || 'all' }}${{ matrix.half }} path: coverage-processes.log retention-days: 30 if-no-files-found: warn - - name: Upload coverage HTML report - if: ${{ !cancelled() && matrix.coverage }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: coverage-html - path: coverage-html/ - retention-days: 30 - - - name: Upload coverage summary - if: ${{ !cancelled() && matrix.coverage }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: coverage-summary - path: coverage-comment/coverage-summary.json - retention-days: 7 - # #1538: ci-timing.yml reads these from the newest push runs. Diagnostic # only, so an artifact service outage must not fail a gating leg. The glob - # takes a PyPy leg's JUnit file per run too (#2005). + # takes a PyPy leg's JUnit file per run too (#2005), and a half of the + # `LATEST` tier's file is its own, read as `latest`'s (#2751). - name: Upload test timings if: ${{ !cancelled() && steps.install.outcome == 'success' }} continue-on-error: true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: junit-unit-${{ matrix.python-version }} + name: junit-unit-${{ matrix.python-version }}${{ matrix.half && '-latest-half-' || '' }}${{ matrix.half }} path: ${{ runner.temp }}/junit*.xml retention-days: 14 @@ -328,7 +343,7 @@ jobs: continue-on-error: true shell: bash env: - JOB_NAME: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + JOB_NAME: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }}${{ matrix.half && ' (LATEST tier, half ' || '' }}${{ matrix.half }}${{ matrix.half && ')' || '' }} run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi - name: Upload the job summary @@ -370,6 +385,120 @@ jobs: matrix: ${{ fromJSON(needs.python-versions.outputs.test-ktlo) }} steps: *test-steps + # #2751: the `LATEST` tier, on `latest` alone, under coverage, out of `test`'s + # `latest` leg, as `project` was cut out of one cell in #2254. That leg ran + # both tiers, and its `Run unit tests` took 30.8, 23.4, 28.6 and 27.4 min of + # the job's 45 on the push runs of f45c82b5d, 21ec0a6f1, ab7be2fb0 and + # 596dcfaf8: 6795, 5040, 6303 and 6027 test-seconds, of which the `LATEST` + # tier, less the pins, took 3942, 2894, 3598 and 3384. So each cell runs a + # half, tests/_tiers.py's latest_tier_half(), cut at LATEST_TIER_BOUNDARY, + # whose halves took 1830 and 1661 test-seconds there at the median, and the + # `latest` leg runs the `ALL` tier alone. The steps are `test`'s, by YAML + # alias, so each cell installs as that leg does and hands its coverage data + # to `coverage`. + test-latest-tier: + name: Python ${{ matrix.python-version }} (LATEST tier, half ${{ matrix.half }}) + if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} + needs: [changes, python-versions] + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ANNOTATION_LEVEL: error + strategy: + fail-fast: false + # `ktlo: false` names the key `test`'s steps read, which actionlint asks + # a matrix written out here to define. + matrix: + python-version: ['${{ needs.python-versions.outputs.latest }}'] + half: [1, 2] + include: + - coverage: true + ktlo: false + steps: *test-steps + + # #2751: the coverage legs' parts, combined into the one number + # coverage-comment.yml posts, which comes from this job's artifact since + # callers of this reusable workflow grant only `contents: read`. Each part + # was checked whole on its own leg; util/coverage_report.py's `combine` + # fails, posting no number, unless every part named below arrived, as + # `test`'s `latest` leg and each `half` of `test-latest-tier` upload them. + # `!cancelled()` so a failing test still reports what was measured, as + # partial. `required-checks` needs it, as it needed the leg that reported. + coverage: + name: Coverage + needs: [changes, test, test-latest-tier] + if: ${{ !cancelled() && inputs.gate-only != true && needs.changes.outputs.code != 'false' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # See `unittest-ordering` below. + - name: Read the Python versions + id: python-versions + run: python3 util/python_versions.py --github-output + + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ steps.python-versions.outputs.latest }} + + # The version the legs measured with, as `test`'s step of the same name. + - name: Install coverage + run: python -m pip install 'coverage>=7.10' + + - name: Download coverage data + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: coverage-data-* + path: coverage-data + + # #2030: its summary goes to the job's copy too. + - name: Report coverage + shell: bash + env: + SHA: ${{ github.sha }} + PYTHON: ${{ steps.python-versions.outputs.latest }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md + run: python util/coverage_report.py combine coverage-data/coverage-data-all coverage-data/coverage-data-latest-half-1 coverage-data/coverage-data-latest-half-2 + + - name: Upload coverage HTML report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-html + path: coverage-html/ + retention-days: 30 + + - name: Upload coverage summary + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-summary + path: coverage-comment/coverage-summary.json + retention-days: 7 + + # #2030: as `test`'s steps of the same names. + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Coverage + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 + integration: name: Integration Python ${{ matrix.python-version }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} @@ -1502,7 +1631,9 @@ jobs: # so depending on it would skip this job on every ordinary PR. `changelog` # is left out too: it is a drift check, and folding it in would widen what # merging requires, which is the maintainer's call. `project-tests` gates - # every diff, as the one run of the project pins (#1954). `engine-tests`'s + # every diff, as the one run of the project pins (#1954). `test-latest-tier` + # and `coverage` gate as `test`'s `latest` leg did when it ran the `LATEST` + # tier and reported coverage itself (#2751). `engine-tests`'s # `unsupported` and `not-installable` cells exit 0 when they decline as # expected, and its KTLO cells are non-blocking through step-level # `continue-on-error`; the KTLO legs of the matrix jobs are jobs of their @@ -1511,21 +1642,23 @@ jobs: # # The docs-only path. On a pull request whose diff touches nothing but # `docs/**` and `*.md` files outside the code directories, `test`, - # `integration`, `engine-tests` and `pypcap-parity` are skipped by their own - # `if:`, and `project-tests` runs the docs readers as well. A job-level `if:` - # rather than workflow-level `paths-ignore`, which would leave this required - # context pending forever. So `skipped` is accepted for those legs **only** - # when `changes` succeeded and said `code == 'false'`. + # `test-latest-tier`, `coverage`, `integration`, `engine-tests` and + # `pypcap-parity` are skipped by their own `if:`, and `project-tests` runs + # the docs readers as well. A job-level `if:` rather than workflow-level + # `paths-ignore`, which would leave this required context pending forever. + # So `skipped` is accepted for those legs **only** when `changes` succeeded + # and said `code == 'false'`. # # The test-only path (#2243). On one whose diff touches tests and docs alone, - # `test` and `integration` run their `latest` leg alone and must succeed, and - # `engine-tests` and `pypcap-parity` are skipped by their own `if:`. So - # `skipped` is accepted for those two **only** when `changes` succeeded and - # said `code == 'true'` and `test-only == 'true'`. `project-tests` must have - # succeeded on every path, and every other combination fails this gate. + # `test` and `integration` run their `latest` leg alone and must succeed, as + # must `test-latest-tier` and `coverage`, and `engine-tests` and + # `pypcap-parity` are skipped by their own `if:`. So `skipped` is accepted + # for those two **only** when `changes` succeeded and said `code == 'true'` + # and `test-only == 'true'`. `project-tests` must have succeeded on every + # path, and every other combination fails this gate. required-checks: name: Required checks passed - needs: [changes, test, integration, engine-tests, pypcap-parity, project-tests] + needs: [changes, test, test-latest-tier, coverage, integration, engine-tests, pypcap-parity, project-tests] if: ${{ always() && inputs.gate-only != true }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -1537,6 +1670,8 @@ jobs: CODE: ${{ needs.changes.outputs.code }} TEST_ONLY: ${{ needs.changes.outputs.test-only }} TEST: ${{ needs.test.result }} + TEST_LATEST_TIER: ${{ needs.test-latest-tier.result }} + COVERAGE: ${{ needs.coverage.result }} INTEGRATION: ${{ needs.integration.result }} ENGINE_TESTS: ${{ needs.engine-tests.result }} PYPCAP_PARITY: ${{ needs.pypcap-parity.result }} @@ -1545,6 +1680,8 @@ jobs: set -eu echo "changes: $CHANGES (code: '$CODE', test-only: '$TEST_ONLY')" echo "test: $TEST" + echo "test-latest-tier: $TEST_LATEST_TIER" + echo "coverage: $COVERAGE" echo "integration: $INTEGRATION" echo "engine-tests: $ENGINE_TESTS" echo "pypcap-parity: $PYPCAP_PARITY" @@ -1568,13 +1705,13 @@ jobs: # every leg must have succeeded. [ "$PROJECT_TESTS" = "success" ] || { echo "::error title=project-tests did not succeed::result was '$PROJECT_TESTS'; it runs the project pins, and on a docs-only diff the docs readers too, so it has to pass."; failed=1; } if [ "$CODE" = "false" ]; then - may_skip="test integration engine-tests pypcap-parity" + may_skip="test test-latest-tier coverage integration engine-tests pypcap-parity" elif [ "$TEST_ONLY" = "true" ]; then may_skip="engine-tests pypcap-parity" else may_skip="" fi - for pair in "test=$TEST" "integration=$INTEGRATION" "engine-tests=$ENGINE_TESTS" "pypcap-parity=$PYPCAP_PARITY"; do + for pair in "test=$TEST" "test-latest-tier=$TEST_LATEST_TIER" "coverage=$COVERAGE" "integration=$INTEGRATION" "engine-tests=$ENGINE_TESTS" "pypcap-parity=$PYPCAP_PARITY"; do job="${pair%%=*}" result="${pair#*=}" case " $may_skip " in diff --git a/docs/source/contributing/conventions/test-coverage.rst b/docs/source/contributing/conventions/test-coverage.rst index 76308591f9..3a916e5d44 100644 --- a/docs/source/contributing/conventions/test-coverage.rst +++ b/docs/source/contributing/conventions/test-coverage.rst @@ -127,10 +127,12 @@ undoes that work in the tree. git apply /tmp/fix.diff python -m pytest tests//test__unit.py -q # must pass -CI measures one leg, the ``test`` job's Python 3.15 cell in -:file:`.github/workflows/unit-tests.yml`, with branch coverage over :mod:`pcapkit` -configured by :file:`.github/coverage.toml`; :file:`.github/workflows/coverage-comment.yml` -posts the totals on the pull request. Locally, measure with ``coverage run`` rather than +CI measures the ``test`` job's Python 3.15 cell in +:file:`.github/workflows/unit-tests.yml`, which runs the ``ALL`` tier, and each half +of the ``LATEST`` tier in its ``test-latest-tier`` job, with branch coverage over +:mod:`pcapkit` configured by :file:`.github/coverage.toml`; its ``coverage`` job +combines them, and :file:`.github/workflows/coverage-comment.yml` posts the totals on +the pull request. Locally, measure with ``coverage run`` rather than a pytest plugin, which this project does not install, and narrow at report time -- ``[tool.coverage.run]``'s ``source`` in :file:`pyproject.toml` takes precedence over ``--include`` at run time: diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index a6ebc628d2..b13434dc48 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -454,13 +454,14 @@ uploads it, and a second dispatch is needed only if it did not. suite, on ``latest``) but also ``changelog`` (checks ``CHANGELOG.md`` against its source entry) and ``python-versions``, which reads ``gate``'s version (see `Python Versions`_ below). It skips the rest -- the matrix jobs, ``test``, -``test-ktlo``, ``integration``, ``integration-ktlo``, ``engine-tests`` (a job per -engine, each looping over its Python versions), ``pypcap-parity``, -``pypcap-parity-ktlo`` and ``unittest-ordering``, each of which has already run -once for this commit from Unit Tests' own ``push`` trigger (and, all but -``unittest-ordering``, which runs on ``main`` pushes only, from its +``test-ktlo``, ``test-latest-tier``, ``integration``, ``integration-ktlo``, +``engine-tests`` (a job per engine, each looping over its Python versions), +``pypcap-parity``, ``pypcap-parity-ktlo`` and ``unittest-ordering``, each of which +has already run once for this commit from Unit Tests' own ``push`` trigger (and, +all but ``unittest-ordering``, which runs on ``main`` pushes only, from its ``pull_request`` trigger), so running any of them again per caller would test -the same commit several times over. It also skips ``changes``, +the same commit several times over. It also skips ``changes``, ``coverage``, +which combines what ``test`` and ``test-latest-tier`` measured, ``project-tests``, whose project pins ``gate``'s full suite runs too (see `Project Pins`_ below), and ``required-checks`` (see `Required Status Checks`_ below), which are gated out by their own ``if:`` rather than by having @@ -647,10 +648,12 @@ writes its state as it then stands. Coverage (:issue:`1063`) ------------------------------------------------------------------------- -Coverage is measured inside an existing leg rather than by a workflow of its +Coverage is measured inside existing legs rather than by a workflow of its own: the ``Python 3.15`` leg of ``test`` (matrix key ``coverage: true``) runs -its unchanged pytest selection under ``coverage run ---rcfile=.github/coverage.toml``. That rcfile repeats +its unchanged pytest selection, the ``ALL`` tier, under ``coverage run +--rcfile=.github/coverage.toml``, and each cell of ``test-latest-tier`` runs its +half of the ``LATEST`` tier the same way (:issue:`2751`, see `Test Tiers`_ +below). That rcfile repeats :file:`pyproject.toml`'s ``[tool.coverage.*]`` settings (``tests/project/test_coverage_rcfile.py`` asserts it) and adds ``parallel`` and ``patch = ["subprocess"]``, without which only the xdist controller is @@ -660,14 +663,16 @@ and re-importing pcapkit, one module's peak RSS was 4.17 GiB under the default killed the runner. ``sigterm = true`` and a ``pytest_sessionfinish`` hook in :file:`tests/conftest.py` that saves each xdist worker's data early keep a worker that dies during shutdown from losing its file -(:issue:`1423`). The leg's -``Report coverage`` step then runs :file:`util/coverage_report.py`, which fails, -posting no total, unless the controller and every xdist worker saved a data file -(:issue:`1387`), and otherwise -writes the total and a per-package table to the job summary. The leg uploads the HTML -report as the ``coverage-html`` artifact (30 days), with the bare numbers as the -``coverage-summary`` artifact and the per-process debug log as -``coverage-processes-log``. +(:issue:`1423`). Each such leg's +``Check coverage data`` step then runs :file:`util/coverage_report.py`'s ``check``, +which fails the leg unless the controller and every xdist worker saved a data file +(:issue:`1387`), and only then uploads the leg's part as a ``coverage-data-*`` +artifact; the per-process debug log goes up as ``coverage-processes-log-*`` either +way. The ``coverage`` job combines the parts: its ``Report coverage`` step's +``combine`` fails, posting no total, unless every part arrived, and otherwise +writes the total and a per-package table to the job summary, partial where a +part's tests failed. It uploads the HTML report as the ``coverage-html`` artifact +(30 days), with the bare numbers as the ``coverage-summary`` artifact. The PR comment comes from **Coverage Comment** (:file:`coverage-comment.yml`), not from ``unit-tests.yml``. That file is also a reusable workflow, a called @@ -679,8 +684,9 @@ treats the artifact as untrusted: it never checks out pull-request code. It takes the PR number from the API, as the open PR whose head is the run's head sha, and renders its own Markdown from validated numbers. It edits one comment, found by a hidden marker, on each push. The first line says when the tests -failed. Neither workflow's coverage step is a required check. A push run has no -pull request, so its report goes to Discussions instead (see `CI Reports`_). +failed. ``coverage`` gates through ``Required checks passed``, as the coverage legs +do; Coverage Comment is not a required check. A push run has no pull request, so its +report goes to Discussions instead (see `CI Reports`_). CI Timing Review (:issue:`1538`) --------------------------------------------------------------------------------- @@ -715,11 +721,13 @@ thread (see `CI Reports`_). It is not a ``workflow_run`` on Unit Tests, which would add a job per run to a runner pool measured saturated at 20 concurrent jobs. -Per-test timings come from the ``test`` and ``integration`` legs, and from -:file:`experimental.yml`'s, which run pytest with ``--durations=30 +Per-test timings come from the ``test``, ``test-latest-tier`` and ``integration`` +legs, and from :file:`experimental.yml`'s, which run pytest with ``--durations=30 --durations-min=1.0`` and ``--junitxml`` and upload the file as ``junit-unit-``, ``junit-integration-`` or -``junit-experimental-`` (14 days). Each upload step is ``continue-on-error``, +``junit-experimental-`` (14 days); a half of the ``LATEST`` tier uploads +``junit-unit--latest-half-``, which the review reads as part of its +``latest`` leg's (:issue:`2751`). Each upload step is ``continue-on-error``, so an artifact-service outage cannot fail a leg. pytest writes one ```` per test however many subtests it runs -- about 0.6 MB per unit leg (estimate) -- but the suite's ``tests=`` count includes subtests, so the review reads each testcase's ``time``. The @@ -862,6 +870,9 @@ Anything else, and a diff the script cannot list, is code: ``code: true`` and * - ``unit-tests.yml``: ``test``, ``integration`` - skipped - the ``latest`` leg alone, the coverage leg included + * - ``unit-tests.yml``: ``test-latest-tier``, ``coverage`` + - skipped + - runs * - ``unit-tests.yml``: ``test-ktlo``, ``integration-ktlo``, ``engine-tests``, ``pypcap-parity``, ``pypcap-parity-ktlo`` - skipped @@ -890,11 +901,12 @@ Each job skips by its own ``if:``, so its check reads ``skipped``, never ``cance which GitHub counts against the commit. ``test`` and ``integration`` pick their matrix with ``fromJSON``, from :file:`util/python_versions.py`'s ``test-latest`` and ``integration-latest`` outputs on a test-only diff. ``Required checks passed`` -accepts ``skipped`` from ``test``, ``integration``, ``engine-tests`` and -``pypcap-parity`` only where ``changes`` succeeded and said docs-only, and from -``engine-tests`` and ``pypcap-parity`` only where it said test-only; a skipped -``test`` or ``integration`` fails a test-only diff, and ``project-tests`` must -succeed on every diff (see `Required Status Checks`_). +accepts ``skipped`` from ``test``, ``test-latest-tier``, ``coverage``, +``integration``, ``engine-tests`` and ``pypcap-parity`` only where ``changes`` +succeeded and said docs-only, and from ``engine-tests`` and ``pypcap-parity`` only +where it said test-only; a skipped ``test``, ``test-latest-tier``, ``coverage`` or +``integration`` fails a test-only diff, and ``project-tests`` must succeed on every +diff (see `Required Status Checks`_). The ``Compat Python`` legs are required contexts, so they cannot be skipped the same way. A conditionally skipped job reports ``Success`` (docs.github.com, @@ -979,9 +991,9 @@ without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, with ``Lint``, and nothing else. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- -``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus -``project-tests``, the one run of the project pins, and ``changes`` for the -docs-only path -- and checks each result explicitly, in its "Check every +``test``, ``test-latest-tier``, ``coverage``, ``integration``, ``engine-tests`` +and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, +and ``changes`` for the docs-only path -- and checks each result explicitly, in its "Check every required job succeeded" step. Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. @@ -1396,12 +1408,12 @@ Which legs run which tier: * - Leg - Tiers - * - ``test`` on ``latest``, the coverage leg - - ``ALL`` and ``LATEST``, less the project pins + * - ``test-latest-tier``, on ``latest``, a half per cell + - ``LATEST``, less the project pins * - ``project-tests`` - the project pins - * - ``test`` off ``latest``, PyPy's among them, ``test-ktlo``, ``experimental``, - ``windows`` + * - ``test``, ``latest``'s coverage leg and PyPy's among them, ``test-ktlo``, + ``experimental``, ``windows`` - ``ALL`` * - ``integration`` on ``latest`` - ``ALL`` and ``LATEST`` @@ -1412,8 +1424,9 @@ Which legs run which tier: - ``ALL`` and ``LATEST`` Each leg asks :file:`tests/_tiers.py` at run time, so no workflow lists a tier: a -``test`` leg off ``latest`` passes each module ``latest_tier_paths()`` prints as an -``--ignore``, and an ``integration`` or ``pypcap-parity`` leg off ``latest`` runs +``test`` leg passes each module ``latest_tier_paths()`` prints as an ``--ignore``, a +``test-latest-tier`` cell runs the modules ``latest_tier_half()`` prints for its +half, and an ``integration`` or ``pypcap-parity`` leg off ``latest`` runs ``fixture_tier_paths('ALL')``, which drops each ``LATEST`` path and node ID and replaces a directory holding a ``LATEST`` module by its ``ALL`` ones, since pytest applies no ignore to a path it was handed. :file:`tests/test_tier_guard.py` checks @@ -1421,10 +1434,18 @@ that every module resolves to exactly one tier and runs the steps to check what does with the answer, and :file:`tests/project/test_sweeps_on_latest_only_1732.py` does the same for every leg the tier file gives. -The project pins stay in a job of their own rather than joining the ``latest`` leg: -``project-tests`` runs in parallel with the coverage leg, the slowest of ``test``'s, -rather than lengthening it under coverage, and it is also the job that runs the docs -readers on a docs-only diff. +The ``LATEST`` tier is a job of its own, halved at ``LATEST_TIER_BOUNDARY`` in +:file:`tests/_tiers.py` (:issue:`2751`), as ``project`` was cut for +``unittest-ordering`` (:issue:`2254`): a half is every module sorting before the +boundary or the rest, so a new module runs in exactly one, and the boundary is chosen +so the halves take about the same CI time. Run with the ``ALL`` tier on one leg under +coverage, it held ``test``'s ``latest`` leg past half its cap. The workflow's comment +above ``test-latest-tier`` records the measurement; move the boundary to rebalance. + +The project pins stay in a job of their own rather than joining a coverage leg: +``project-tests`` runs in parallel with the coverage legs rather than lengthening +one under coverage, and it is also the job that runs the docs readers on a docs-only +diff. Latest CPython runs the whole suite, as the sanity check and the gate (the owner's ruling on :issue:`1958`), so it also installs what the whole suite gates on. In ``test`` and @@ -1432,6 +1453,8 @@ ruling on :issue:`1958`), so it also installs what the whole suite gates on. In the ``latest`` leg alone, after the install every leg runs, and adds the engines pip can install there (``Scapy``, ``PyShark``), html5lib through ``vendor``, Sphinx and the pinned sphinx-autodoc-typehints through ``docs``, and emoji through ``cli``. +Each cell of ``test-latest-tier``, which runs ``test``'s steps, is a ``latest`` leg +too, and runs that step as well. ``project-tests`` and ``gate``, which run on ``latest`` only, install the same set in their ``Install package and test dependencies`` and ``Install package, test and generator dependencies`` steps. The other legs keep their install lines and run the @@ -1458,17 +1481,18 @@ loader or a child process, counts as importing it, and so does a file that does not parse. Every pin is in the ``LATEST`` test tier (see `Test Tiers`_ above), and a ``tests/project`` module marked ``__tier__ = 'ALL'`` is no pin. The modules that import ``pcapkit`` are no pins either; those the tiers send to ``latest`` run -on ``test``'s ``latest`` leg, under coverage. +in a half of ``test-latest-tier``, under coverage. ``project-tests`` runs the pins under pytest, with ``test``'s flags, on ``latest``, on every push to ``main`` and every pull request but a docs-only one, which runs :file:`tests/project` under plain unittest instead (see `Light Pull Requests (#2243)`_), and ``required-checks`` needs it. Every ``test``, ``test-ktlo`` and ``experimental`` leg passes each pin as an -``--ignore``, ``latest``'s coverage leg included, which measures only ``pcapkit``. -``latest`` asks ``project_pin_paths()`` at run time, and every other leg -``latest_tier_paths()``, which holds each pin, as ``integration`` asks for its -selection, so no workflow lists the pins, and :file:`tests/test_tier_guard.py` runs +``--ignore``, through ``latest_tier_paths()``, which holds each pin, and +``latest_tier_half()`` leaves them out of ``test-latest-tier``, whose coverage +measures only ``pcapkit``, and ``project-tests`` runs what ``project_pin_paths()`` +prints. Each asks at run time, as ``integration`` asks for its selection, so no +workflow lists the pins, and :file:`tests/test_tier_guard.py` runs the steps to check what each does with the answer. Under ``gate-only``, ``gate``'s full suite runs the pins with everything else. ``unittest-ordering``'s ``project`` cells keep them, as they ask how a module runs in one interpreter, in file order and @@ -1488,7 +1512,8 @@ several gigabytes on every run. So every pytest and unittest command in :file:`.github/workflows/unit-tests.yml` runs as ``python util/cap_file_size.py python -m ...``: ``test``'s ``Run unit tests``; ``integration``'s, ``pypcap-parity``'s and ``gate``'s ``Run full test suite``; the KTLO twins of -``test``, ``integration`` and ``pypcap-parity``, which share their steps; the +``test``, ``integration`` and ``pypcap-parity``, and ``test-latest-tier``, which +share their steps; the engine cells that ``engine-tests``' ``Write the per-cell script`` writes; ``unittest-ordering``'s cells; and each run step of ``project-tests``. @@ -1536,7 +1561,9 @@ to a script. Each script's docstring says what it does; each has tests under ``experimental.yml`` and of ``python-compatibility.yml``. - No. * - :file:`util/coverage_report.py` - - ``Report coverage`` in ``unit-tests.yml``'s ``test``, on the coverage leg. + - ``Check coverage data`` in ``unit-tests.yml``'s ``test`` and + ``test-latest-tier``, on each coverage leg, as ``coverage_report.py check``, + and ``Report coverage`` in its ``coverage``, as ``coverage_report.py combine``. - No. * - :file:`util/pytest_shards.py` - ``Run unit tests`` in ``experimental.yml``'s ``experimental``, on GraalPy's leg diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index 0848df3007..d729161136 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -1305,7 +1305,8 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': # A KTLO job (#1738) writes its steps as a YAML alias of its gating # twin's -- `steps: *test-steps` -- so its slice holds no pytest line and # is skipped here: it installs and selects what the twin does, so it - # reaches no gate the twin does not. + # reaches no gate the twin does not. `test-latest-tier` (#2751) runs + # `test`'s steps the same way, a part of what that step selects. if 'python -m pytest' not in section: continue @@ -1385,9 +1386,11 @@ def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': answer it -- and :meth:`~tests.test_tier_guard.WorkflowAgreementTests\ .test_ignore_flags_match_the_fixture_tier_constants` - is what keeps that equivalence true. It is the ``latest`` leg's: the - others also leave out the ``LATEST`` test tier, - :func:`~tests._tiers.latest_tier_paths` (#1958), which changes no answer, + is what keeps that equivalence true. It is what the ``latest`` legs run + between them, ``test``'s, which runs the ``ALL`` tier, and the halves of the + ``LATEST`` one that ``test-latest-tier`` runs by ``test``'s steps (#2751); the + other legs also leave out the ``LATEST`` test tier, + :func:`~tests._tiers.latest_tier_paths` (#1958). That changes no answer, since every leg shares the job's one install line. The same holds for a ``'fixture-tier'`` job with a ``latest`` leg, whose other legs ask for the ``ALL`` tier alone. diff --git a/tests/_tiers.py b/tests/_tiers.py index b6ae761f8f..3d3e65242e 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -52,8 +52,10 @@ a module is in it only when it says so with ``__tier__ = 'ALL'`` (:func:`declared_tier`), and :data:`DEFAULT_TIER`, ``LATEST``, otherwise. :func:`all_tier_paths` and :func:`latest_tier_paths` list them; every - leg but ``latest`` leaves the ``LATEST`` one out. See "The curated ``ALL`` tier" below + leg of ``test`` leaves the ``LATEST`` one out. See "The curated ``ALL`` tier" below for which modules opt in. +* :func:`latest_tier_half` -- which half of the ``LATEST`` tier does each cell of + the ``test-latest-tier`` job run on ``latest`` (#2751)? * :func:`is_project_pin` and :func:`project_pin_paths` -- which :file:`tests/project` modules give one answer on every interpreter, and so run once per push, in the ``project-tests`` job, rather than on every leg (#1954)? @@ -202,6 +204,7 @@ 'InvalidTierError', 'DEFAULT_TIER', 'suite_modules', 'declared_tier', 'branches_on_interpreter', 'module_tier', 'PROBE_MODULE_NAME', 'is_probe_module', 'all_tier_paths', 'latest_tier_paths', + 'LATEST_TIER_HALVES', 'LATEST_TIER_BOUNDARY', 'LATEST_HALF_SELECTOR', 'latest_tier_half', 'PROJECT_PINS_DIR', 'PROJECT_PINS_SELECTOR', 'PROJECT_PIN_PACKAGE', 'LOADABLE_DIRS', 'varies_by_interpreter', 'is_project_pin', 'project_pin_paths', 'is_unit_tier', 'committed_captures', 'committed_capture_names', @@ -256,8 +259,8 @@ ) #: The test tiers (#1958), which say on which legs a module runs. Orthogonal to the #: unit and fixture-dependent tiers above, which say what a module may read: ``ALL`` -#: modules run on every leg of every job that collects them, ``LATEST`` ones on the -#: ``latest`` leg alone. See :func:`module_tier`. +#: modules run on every leg of every job that collects them, ``LATEST`` ones on +#: ``latest`` alone. See :func:`module_tier`. TIER_ALL = 'ALL' #: See :data:`TIER_ALL`. TIER_LATEST = 'LATEST' @@ -270,9 +273,9 @@ #: ``__tier__ = 'LATEST'``, read by AST in :func:`declared_tier`. A constant rather #: than a pytest marker, so plain :mod:`unittest` runs see the same module. TIER_MARKER = '__tier__' -#: The ``LATEST`` tier's selector, verbatim from the workflows: every leg of ``test`` -#: but ``latest``, and every ``experimental`` leg, passes each path it prints as an -#: ``--ignore``. Asked at run time, as :data:`FIXTURE_TIER_SELECTOR` is, so no +#: The ``LATEST`` tier's selector, verbatim from the workflows: every leg of ``test``, +#: ``latest``'s included, and every ``experimental`` leg, passes each path it prints as +#: an ``--ignore``. Asked at run time, as :data:`FIXTURE_TIER_SELECTOR` is, so no #: workflow lists the tier. See :func:`latest_tier_paths`. LATEST_TIER_SELECTOR = ( 'python -c "from tests._tiers import latest_tier_paths; ' @@ -291,15 +294,30 @@ #: :func:`is_project_pin`. PROJECT_PINS_DIR = 'project' #: The project pins' selector, verbatim from the workflows: the ``project-tests`` -#: job runs what it prints, and the ``latest`` leg of ``test`` passes each path as -#: an ``--ignore``. Every other ``test`` leg and every ``experimental`` leg leaves +#: job runs what it prints. Every ``test`` leg and every ``experimental`` leg leaves #: the pins out through :data:`LATEST_TIER_SELECTOR`, since every pin is -#: ``LATEST``. Asked at run time, as :data:`FIXTURE_TIER_SELECTOR` is, so no +#: ``LATEST``, and :func:`latest_tier_half` leaves them out of the ``LATEST`` tier's +#: own job. Asked at run time, as :data:`FIXTURE_TIER_SELECTOR` is, so no #: workflow can drift from :func:`is_project_pin`. PROJECT_PINS_SELECTOR = ( 'python -c "from tests._tiers import project_pin_paths; ' 'print(\' \'.join(project_pin_paths()))"' ) +#: The halves :func:`latest_tier_half` cuts the ``LATEST`` tier into (#2751), one per +#: cell of the ``test-latest-tier`` job of :file:`.github/workflows/unit-tests.yml`. +LATEST_TIER_HALVES = (1, 2) +#: The path at which :func:`latest_tier_half` cuts (#2751): half 1 is every module +#: sorting before it, half 2 the rest, so a new module lands in exactly one half. It +#: is chosen so the halves take about the same CI time; move it to rebalance, and the +#: cells follow. +LATEST_TIER_BOUNDARY = 'tests/protocols/internet/test_j' +#: The selector each cell of the ``test-latest-tier`` job asks for its half, verbatim +#: from the workflow, with the half as its one argument. Asked at run time, as +#: :data:`LATEST_TIER_SELECTOR` is. See :func:`latest_tier_half`. +LATEST_HALF_SELECTOR = ( + 'python -c "import sys; from tests._tiers import latest_tier_half; ' + 'print(\' \'.join(latest_tier_half(int(sys.argv[1]))))"' +) #: The package whose behaviour varies by interpreter, so a module that reaches it #: runs on every leg (#1954). PROJECT_PIN_PACKAGE = 'pcapkit' @@ -1075,9 +1093,9 @@ def module_tier(path: 'pathlib.Path | str') -> 'str': Its :func:`declared_tier` where it states one, :data:`DEFAULT_TIER` otherwise, so every test module has exactly one and ``ALL`` is opt-in: see "The curated ``ALL`` tier" in this module's docstring for which modules opt in. ``ALL`` - modules run on every leg; ``LATEST`` ones on ``latest`` alone, which is also - the coverage leg, the ``gate`` job's interpreter, and every - ``unittest-ordering`` cell's. Within a tier, the unit and fixture-dependent + modules run on every leg; ``LATEST`` ones on ``latest`` alone, in the + ``test-latest-tier`` job (:func:`latest_tier_half`), the ``gate`` job, and + every ``unittest-ordering`` cell. Within a tier, the unit and fixture-dependent tiers still decide which job collects a module. Args: @@ -1124,8 +1142,8 @@ def is_project_pin(path: 'pathlib.Path | str') -> 'bool': text or drives a ``util/`` script, so the ``project-tests`` job runs it once, on ``latest``. Every pin is in the :data:`TIER_LATEST` test tier (#1958): a module that marks itself :data:`TIER_ALL` is no pin, and one that marks itself - ``LATEST`` while it imports ``pcapkit`` is no pin either, and runs on the - ``latest`` leg of ``test``, under coverage. + ``LATEST`` while it imports ``pcapkit`` is no pin either, and runs in a half of + the ``test-latest-tier`` job, under coverage (:func:`latest_tier_half`). Args: path: Path to a module, absolute or relative to :data:`ROOT`. @@ -1149,8 +1167,8 @@ def project_pin_paths() -> 'tuple[str, ...]': """Every :func:`project pin `, relative to :data:`ROOT`, sorted. What :data:`PROJECT_PINS_SELECTOR` prints: the ``project-tests`` job's - selection, and the ``--ignore`` list of the ``latest`` leg of ``test``. Each is - in :func:`latest_tier_paths` too, which every other leg leaves out. + selection. Each is in :func:`latest_tier_paths` too, which every leg of ``test`` + leaves out, and none is in :func:`latest_tier_half`. """ return tuple(sorted(path.relative_to(ROOT).as_posix() @@ -1158,6 +1176,31 @@ def project_pin_paths() -> 'tuple[str, ...]': if is_project_pin(path))) +def latest_tier_half(half: 'int') -> 'tuple[str, ...]': + """Half ``half`` of the unit-tier ``LATEST`` modules but the project pins, sorted (#2751). + + What :data:`LATEST_HALF_SELECTOR` prints: the selection of one cell of the + ``test-latest-tier`` job, which runs it on ``latest`` under coverage, by name. + Half 1 is every such module of :func:`latest_tier_paths` sorting before + :data:`LATEST_TIER_BOUNDARY`, half 2 the rest. Both halves together are what the + ``latest`` leg of ``test``, which runs the ``ALL`` tier, leaves out, less the + :func:`project pins `, which ``project-tests`` runs, and the + fixture-dependent modules, which ``integration`` runs. + + Args: + half: One of :data:`LATEST_TIER_HALVES`. + + Raises: + ValueError: ``half`` is not one of :data:`LATEST_TIER_HALVES`. + + """ + if half not in LATEST_TIER_HALVES: + raise ValueError(f'the LATEST tier has the halves {LATEST_TIER_HALVES}, not {half!r}') + pins = frozenset(project_pin_paths()) + return tuple(path for path in latest_tier_paths() + if is_unit_tier(path) and path not in pins and (path < LATEST_TIER_BOUNDARY) == (half == 1)) + + def _git(*args: 'str') -> 'Optional[str]': """Run :program:`git` in :data:`ROOT` and return its standard output. diff --git a/tests/conftest.py b/tests/conftest.py index a3ba04b991..52790c2259 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -204,14 +204,14 @@ def pytest_sessionstart(session: 'pytest.Session') -> 'None': def pytest_sessionfinish(session: 'pytest.Session') -> 'None': """Write an xdist worker's coverage data out before the process can die (#1423). - The coverage leg of ``unit-tests.yml`` measures each worker through + Each coverage leg of ``unit-tests.yml`` measures each worker through coverage's ``patch = ["subprocess"]`` hook, and that measurement is saved by an :mod:`atexit` handler -- so it is written *after* the worker has already reported its results and told the controller it is finished. Anything that kills the process in that window takes the whole worker's data with it, and takes it silently: pytest has its results, so every test still passes, while ``coverage combine`` merges one file fewer and reports the rest as the whole. - That is #1387, and the guard in ``Report coverage`` turns it into a failed + That is #1387, and the guard in ``Check coverage data`` turns it into a failed step rather than a quietly low number -- four times on 2026-10-08. Saving here closes the window instead of policing it. :func:`cov.save` is @@ -220,7 +220,7 @@ def pytest_sessionfinish(session: 'pytest.Session') -> 'None': this point on and a worker that dies during shutdown costs at most the arcs executed after this hook. - A no-op outside that one leg: it returns immediately unless the process is + A no-op outside those legs: it returns immediately unless the process is an xdist worker (``workerinput``) whose coverage was auto-started by the subprocess patch. diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py index 20c4bea2d2..3d890938e0 100644 --- a/tests/project/test_ci_timing.py +++ b/tests/project/test_ci_timing.py @@ -52,7 +52,8 @@ import requests -from tests import _workflows +from tests import _tiers, _workflows +from tests.project.test_ktlo_legs_stay_green_1672 import interpolate from tests.project.test_python_versions_1660 import NO_TOMLLIB from tests.project.test_python_versions_1660 import legs as matrix_legs from tests.project.test_python_versions_1660 import matrix, tomllib @@ -1135,9 +1136,16 @@ def setUp(self) -> 'None': def test_every_unit_and_integration_leg_uploads_its_timings(self) -> 'None': blocks = step_blocks(self.text, 'Upload test timings') names = [re.search(r'(?m)^ name: (.+)$', block).group(1) for block in blocks] - self.assertEqual(names, ['junit-unit-${{ matrix.python-version }}', - 'junit-integration-${{ matrix.python-version }}']) - self.assertIn(ci_timing.JUNIT_ARTIFACT.format(leg='${{ matrix.python-version }}'), names) + # #2751: a half of the LATEST tier, which `test-latest-tier` runs by `test`'s steps, has its own. + unit = "junit-unit-${{ matrix.python-version }}${{ matrix.half && '-latest-half-' || '' }}${{ matrix.half }}" + self.assertEqual(names, [unit, 'junit-integration-${{ matrix.python-version }}']) + legs = [leg for job in ('test', 'test-latest-tier') for leg in matrix_legs(matrix('unit-tests.yml', job)) + if leg['python-version'] == ci_timing.LATEST_VERSION] + self.assertEqual(sorted(interpolate(unit, {'matrix': leg}) for leg in legs), + [ci_timing.JUNIT_ARTIFACT.format(leg=ci_timing.LATEST_VERSION)] + + [ci_timing.JUNIT_HALF_ARTIFACT.format(leg=ci_timing.LATEST_VERSION, half=half) + for half in ci_timing.JUNIT_HALVES]) + self.assertEqual(ci_timing.JUNIT_HALVES, _tiers.LATEST_TIER_HALVES) # #2005: a PyPy unit leg writes a JUnit file per pytest run, junit.xml among them. paths = ('junit*.xml', 'junit.xml') self.assertEqual(len(blocks), len(paths)) @@ -1152,6 +1160,43 @@ def test_every_unit_and_integration_leg_uploads_its_timings(self) -> 'None': # tests/_dependency_gates.py reads exactly one pytest step per job. self.assertNotIn('python -m pytest', block) + def test_the_latest_tiers_halves_are_read_as_the_latest_legs(self) -> 'None': + """#2751: a push run's ``latest`` leg and each half of the ``LATEST`` tier are one entry.""" + latest = ci_timing.LATEST_VERSION + names = [ci_timing.JUNIT_ARTIFACT.format(leg=latest)] + [ + ci_timing.JUNIT_HALF_ARTIFACT.format(leg=latest, half=half) for half in ci_timing.JUNIT_HALVES] + ids = {name: JUNIT_ARTIFACT_ID + 100 + index for index, name in enumerate(names)} + base = 'https://api.github.com/repos/JarryShaw/PyPCAPKit/actions/artifacts' + + class HalvesAPI(FakeAPI): + """The fixtures, with the ``latest`` leg's and its halves' files in the push run's artifacts.""" + + def __call__(self, url: 'str', accept: 'str') -> 'tuple[bytes, dict[str, str]]': + path = urllib.parse.urlsplit(url).path + if path.endswith(f'/actions/runs/{PUSH_RUN}/artifacts'): + self.requests.append((url, accept)) + listing = json.loads(fixture(f'artifacts-{PUSH_RUN}.json')) + listing['artifacts'] += [{'id': ident, 'name': name, 'expired': False, + 'archive_download_url': f'{base}/{ident}/zip'} + for name, ident in ids.items()] + listing['total_count'] = len(listing['artifacts']) + return json.dumps(listing).encode(), {'x-ratelimit-remaining': '4321'} + ident = next((ident for ident in ids.values() if path.endswith(f'/actions/artifacts/{ident}/zip')), None) + if ident is None: + return super().__call__(url, accept) + self.requests.append((url, accept)) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('junit.xml', f'') + return buffer.getvalue(), {} + + with mock.patch.object(ci_timing, 'MIN_SAMPLES', 1): + _, _, report, _ = run_main(api=HalvesAPI()) + leg = report['tests'][latest] + self.assertEqual(leg['runs'], 1) + self.assertEqual(sorted(name for name, _ in leg['top_modules']), sorted(f'tests.m{ident}' for ident in ids.values())) + def test_artifact_names_are_unique_in_a_run(self) -> 'None': names = [found.group(1) for found in (re.match(r'\n? +with:\n +name: ([^\n]+)', self.text[pin.end():]) diff --git a/tests/project/test_coverage_data_guard.py b/tests/project/test_coverage_data_guard.py index 9e9e45d877..2fbee72861 100644 --- a/tests/project/test_coverage_data_guard.py +++ b/tests/project/test_coverage_data_guard.py @@ -6,10 +6,13 @@ process that dies before coverage's atexit save leaves no ``.coverage.*`` file, and ``coverage combine`` reports what is left as the whole. So the ``Run unit tests`` step of :file:`.github/workflows/unit-tests.yml` logs every measured -process (``COVERAGE_DEBUG=pid,process``), and ``Report coverage`` runs -:file:`util/coverage_report.py` (#1967), whose check, before ``coverage combine``, -fails the step, posting no total, unless the controller and each of its direct -children -- the xdist workers -- saved a file. +process (``COVERAGE_DEBUG=pid,process``), and ``Check coverage data`` runs +:file:`util/coverage_report.py` (#1967), whose check fails the step unless the +controller and each of its direct children -- the xdist workers -- saved a file. + +#2751: each coverage leg measures a part of the suite and uploads it only once +that check passed; the ``coverage`` job's ``Report coverage`` combines the parts, +and its ``combine`` stage posts no total unless every part arrived. The script's ``check`` stage is run here, both on hand-written logs and on one written by a real ``coverage run``, so the format it parses is the one coverage @@ -19,6 +22,9 @@ from __future__ import annotations +import contextlib +import importlib.util +import io import os import pathlib import re @@ -45,10 +51,10 @@ CHILD = re.compile(r"^ *(\d+)\.\w+: New process: cmd: \['-c'\]", re.M) -def _report_step() -> str: - """The ``run:`` text of the ``Report coverage`` step.""" +def _report_step(name: str = 'Report coverage') -> str: + """The text of the step ``name``, ``Report coverage`` by default.""" text = WORKFLOW.read_text(encoding='utf-8') - return text.split('- name: Report coverage', 1)[1].split('\n - name:', 1)[0] + return text.split(f'- name: {name}\n', 1)[1].split('\n - name:', 1)[0] def _has_coverage() -> bool: @@ -92,8 +98,19 @@ def assert_fails(self, why: str) -> None: self.assertIn('**Coverage not reported:**', self.summary.read_text(encoding='utf-8')) def test_the_check_runs_before_combine(self) -> None: - """``combine`` deletes the data files it merges, so the check must come first.""" - self.assertIn('run: python util/coverage_report.py\n', _report_step()) + """``combine`` deletes the data files it merges, so the check must come first. + + #2751: on CI each leg checks its part, and uploads it only once the check + passed, before the ``coverage`` job combines the parts. + + """ + check = _report_step('Check coverage data') + self.assertIn('\n python util/coverage_report.py check\n', check) + self.assertIn('id: coverage-check\n', check) + upload = _report_step('Upload coverage data') + self.assertIn("if: ${{ !cancelled() && matrix.coverage && steps.coverage-check.outcome == 'success' }}\n", + upload) + self.assertIn('run: python util/coverage_report.py combine ', _report_step()) log = _entry(100, 1, CONTROLLER_CMD) + _entry(101, 100, "['-c']") + _entry(102, 100, "['-c']") self.write(log, [100, 101]) result = self.run_guard('all') @@ -184,6 +201,87 @@ def test_the_process_log_is_kept_as_an_artifact(self) -> None: self.assertIn(f'path: {LOG}', step) +class CoveragePartsTests(unittest.TestCase): + """#2751: the ``coverage`` job combines every leg's part, or posts no total.""" + + def setUp(self) -> None: + self.tempdir = pathlib.Path(tempfile.mkdtemp(prefix='coverage-parts-')) + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + self.summary = self.tempdir / 'summary.md' + self.summary.touch() + + def part(self, name: str, outcome: str, pids: 'list[int]') -> 'list[pathlib.Path]': + """A part as a leg uploads it: its data files, and its tests step's outcome.""" + part = self.tempdir / name + part.mkdir() + (part / 'coverage-tests-outcome').write_text(f'{outcome}\n', encoding='utf-8') + files = [part / f'.coverage.{name}.pid{pid}.Xabcdefgh' for pid in pids] + for path in files: + path.touch() + return files + + def gather(self, *parts: str) -> 'tuple[int, str, str]': + """``gather(parts)`` in the temporary directory: its status and outcome, and what it printed.""" + spec = importlib.util.spec_from_file_location('coverage_report_2751', SCRIPT) + module = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + sys.path.insert(0, str(SCRIPT.parent)) + try: + spec.loader.exec_module(module) # type: ignore[union-attr] + finally: + sys.path.remove(str(SCRIPT.parent)) + out = io.StringIO() + cwd = os.getcwd() + os.chdir(self.tempdir) + try: + with unittest.mock.patch.dict(os.environ, {'GITHUB_STEP_SUMMARY': str(self.summary)}), \ + contextlib.redirect_stdout(out): + status, outcome = module.gather(list(parts)) + finally: + os.chdir(cwd) + return status, outcome, out.getvalue() + + def test_every_part_is_moved_in_and_the_outcome_spans_them(self) -> None: + moved = self.part('all', 'success', [100, 101]) + self.part('latest-half-1', 'failure', [200]) + self.assertEqual(self.gather('all', 'latest-half-1')[:2], (0, 'failure')) + self.assertEqual(sorted(path.name for path in self.tempdir.glob('.coverage.*')), + sorted(path.name for path in moved)) + self.assertEqual(self.summary.read_text(encoding='utf-8'), '') + self.part('latest-half-2', 'success', [300]) + self.assertEqual(self.gather('latest-half-2')[:2], (0, 'success')) + + def test_a_missing_part_posts_no_total(self) -> None: + """A leg whose check failed uploads nothing, so its part is absent and nothing is combined.""" + kept = self.part('all', 'success', [100]) + status, _, out = self.gather('all', 'latest-half-1') + self.assertEqual(status, 1) + self.assertIn("::error::Coverage data is incomplete: no data from ['latest-half-1']", out) + self.assertIn('**Coverage not reported:**', self.summary.read_text(encoding='utf-8')) + self.assertTrue(all(path.exists() for path in kept)) + self.assertEqual(list(self.tempdir.glob('.coverage.*')), []) + + def test_the_parts_combined_are_the_ones_the_legs_upload(self) -> None: + """``all`` from ``test``'s ``latest`` leg, and one per half of ``test-latest-tier``.""" + # pylint: disable=import-outside-toplevel + from tests import _tiers, _workflows + from tests.project.test_ktlo_legs_stay_green_1672 import interpolate + from tests.project.test_python_matrix_tiers_1633 import legs + + jobs = _workflows.load(WORKFLOW)['jobs'] + upload = next(step for step in jobs['test']['steps'] if step.get('name') == 'Upload coverage data') + covered = [leg for job in ('test', 'test-latest-tier') for leg in legs('unit-tests.yml', job) + if leg.get('coverage')] + self.assertEqual(sorted(leg.get('half') for leg in covered if leg.get('half')), + [str(half) for half in _tiers.LATEST_TIER_HALVES]) + names = [interpolate(upload['with']['name'], {'matrix': leg}) for leg in covered] + self.assertEqual(names, ['coverage-data-all'] + [f'coverage-data-latest-half-{half}' + for half in _tiers.LATEST_TIER_HALVES]) + download = next(step for step in jobs['coverage']['steps'] if step.get('name') == 'Download coverage data') + self.assertEqual(download['with'], {'pattern': 'coverage-data-*', 'path': 'coverage-data'}) + report = next(step for step in jobs['coverage']['steps'] if step.get('name') == 'Report coverage') + self.assertEqual(report['run'], 'python util/coverage_report.py combine ' + + ' '.join(f'coverage-data/{name}' for name in names)) + + class WorkerSessionFinishSaveTests(unittest.TestCase): """#1423: an xdist worker writes its data at session finish, not only at exit.""" diff --git a/tests/project/test_ktlo_jobs_gate_nothing_1738.py b/tests/project/test_ktlo_jobs_gate_nothing_1738.py index 56558e8220..dfb91496de 100644 --- a/tests/project/test_ktlo_jobs_gate_nothing_1738.py +++ b/tests/project/test_ktlo_jobs_gate_nothing_1738.py @@ -181,12 +181,17 @@ def test_each_reads_its_own_output(self) -> None: self.assertEqual(outputs[twin], f'${{{{ steps.read.outputs.{twin} }}}}') def test_the_steps_are_one_yaml_node(self) -> None: - """An alias, not a copy, so the twins cannot drift: ``steps: &-steps`` and ``steps: *-steps``.""" + """An alias, not a copy, so the twins cannot drift: ``steps: &-steps`` and ``steps: *-steps``. + + #2751: ``test-latest-tier`` runs ``test``'s steps by the same alias. + + """ text = WORKFLOW.read_text(encoding='utf-8') for job_id, twin in TWINS.items(): with self.subTest(job=twin): self.assertEqual(len(re.findall(rf'(?m)^ steps: &{re.escape(job_id)}-steps$', text)), 1) - self.assertEqual(len(re.findall(rf'(?m)^ steps: \*{re.escape(job_id)}-steps$', text)), 1) + aliases = re.findall(rf'(?m)^ ([\w-]+):\n(?: .*\n|\n)*? steps: \*{re.escape(job_id)}-steps$', text) + self.assertEqual(aliases, [twin, 'test-latest-tier'] if job_id == 'test' else [twin]) def ci_timing() -> 'Any': diff --git a/tests/project/test_pip_install_1967.py b/tests/project/test_pip_install_1967.py index 3c74febd15..914461f88f 100644 --- a/tests/project/test_pip_install_1967.py +++ b/tests/project/test_pip_install_1967.py @@ -73,10 +73,13 @@ ('test', "Install the whole suite's dependencies on latest"), ('integration', "Install the whole suite's dependencies on latest"), ('test', 'Install coverage'), + ('coverage', 'Install coverage'), ('engine-tests', 'Write the per-cell script'), }) -#: The KTLO jobs, which run their twin's steps by alias (#1738). -TWINS = {'test-ktlo': 'test', 'integration-ktlo': 'integration', 'pypcap-parity-ktlo': 'pypcap-parity'} +#: The KTLO jobs, which run their twin's steps by alias (#1738), and #2751's +#: ``test-latest-tier``, which runs ``test``'s. +TWINS = {'test-ktlo': 'test', 'test-latest-tier': 'test', 'integration-ktlo': 'integration', + 'pypcap-parity-ktlo': 'pypcap-parity'} #: Text only the inline install holds. INLINE = ('pip-install.log', 'did not install on', "s/^ERROR: //p") #: The covered steps outside ``unit-tests.yml``, as ``(workflow, job)``, each step whole, diff --git a/tests/project/test_project_pins_once_1954.py b/tests/project/test_project_pins_once_1954.py index 635c02d33a..70555a4e1f 100644 --- a/tests/project/test_project_pins_once_1954.py +++ b/tests/project/test_project_pins_once_1954.py @@ -94,14 +94,17 @@ def test_the_ordering_cell_runs_every_pin(self) -> None: self.assertEqual(pins - modules, set()) def test_every_pin_is_in_the_latest_tier(self) -> None: - """#1958: the pins are ``LATEST`` modules, so the legs off ``latest`` leave them out with the rest of - the tier, and ``latest`` leaves out the pins alone.""" + """#1958: the pins are ``LATEST`` modules, so every ``test`` leg leaves them out with the rest of + the tier, and #2751: the ``LATEST`` tier's halves leave out the pins alone.""" pins = set(_tiers.project_pin_paths()) self.assertTrue(pins) self.assertEqual({_tiers.module_tier(pin) for pin in pins}, {_tiers.TIER_LATEST}) self.assertLess(pins, set(_tiers.latest_tier_paths())) + halves = {path for half in _tiers.LATEST_TIER_HALVES for path in _tiers.latest_tier_half(half)} + self.assertEqual(halves & pins, set()) + self.assertEqual(halves | pins, {path for path in _tiers.latest_tier_paths() if _tiers.is_unit_tier(path)}) run = str(step(('unit-tests.yml', 'test'), 'Run unit tests')['run']) - self.assertIn(_tiers.PROJECT_PINS_SELECTOR, run) + self.assertIn(_tiers.LATEST_HALF_SELECTOR, run) self.assertIn(_tiers.LATEST_TIER_SELECTOR, run) diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index 6c7ffbb108..a7ecc38390 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -208,7 +208,10 @@ def matrix(name: 'str', job_id: 'str', values: 'Optional[dict[str, str]]' = None found = resolve(name, job_id, raw, values) assert isinstance(found, dict), f'{name} {job_id} matrix is {raw!r}' return found - return {key: (resolve(name, job_id, value, values) if isinstance(value, str) else value) + # #2751: an axis may list a single output, as `test-latest-tier`'s version axis does. + return {key: (resolve(name, job_id, value, values) if isinstance(value, str) + else [resolve(name, job_id, item, values) if isinstance(item, str) and READ.fullmatch(item.strip()) + else item for item in value] if isinstance(value, list) else value) for key, value in raw.items()} diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 56d47ce559..ac5ea501e3 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -8,16 +8,18 @@ rulings there, paraphrased, keep the hand-picked round trips on every leg and send only the real sweeps, with the modules whose answer no interpreter changes, to ``latest``. So ``test``'s ``Run unit tests`` step passes each module -:func:`~tests._tiers.latest_tier_paths` prints as an ``--ignore`` on every leg whose -version is not ``latest`` -- the KTLO and older supported legs -- as -``experimental``'s does on each of its legs, and ``latest``, the coverage leg, keeps -the whole unit tier less the project pins (#1954). ``integration`` does the same +:func:`~tests._tiers.latest_tier_paths` prints as an ``--ignore`` on every leg, +``latest``'s too, as ``experimental``'s does on each of its legs, and #2751: +``test-latest-tier`` runs the tier by the same step on ``latest``, a half a cell, as +:func:`~tests._tiers.latest_tier_half` prints it, which leaves out the project pins +(#1954). The coverage legs together measure the whole unit tier less the pins. +``integration`` does the same with the fixture tier: off ``latest``, its ``Run full test suite`` asks for :func:`~tests._tiers.fixture_tier_paths`'s ``ALL`` part, and ``pypcap-parity``, none of whose legs is ``latest``, asks for it on each, :class:`TestTheIntegrationSubset`. :class:`TestTheStepSelectsByLeg` runs the step's script on each leg the tier file -gives either job, with a ``python`` on ``PATH`` that records its arguments, so what +gives each job, with a ``python`` on ``PATH`` that records its arguments, so what is checked is the selection bash builds, not what the text appears to say. :class:`TestTheIgnoresAreTheLatestTier` asks pytest itself what those ignores leave out. :class:`TestEveryXdistRunCountsLogicalCPUs` holds every xdist run in the @@ -60,6 +62,8 @@ #: The jobs and step that run the unit tier: ``test``, and the experimental legs' own (#1732). JOB = ('unit-tests.yml', 'test') EXPERIMENTAL = ('experimental.yml', 'experimental') +#: #2751: the ``LATEST`` tier's own job, which runs ``test``'s steps by YAML alias. +LATEST_TIER = ('unit-tests.yml', 'test-latest-tier') STEP = 'Run unit tests' #: The jobs and step that run the fixture tier, which the tiers cover too (#1732, #1958). INTEGRATION = ('unit-tests.yml', 'integration') @@ -73,7 +77,8 @@ #: main's, ``experimental``'s are sized for GraalPy (#2739: about 1.5 times its leg in job #: 125752017215, whose last run is estimated). #: The watcher's interpreter and its requests are capped too (#1938). -CAPS = {JOB: (45, {}), EXPERIMENTAL: (210, {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 190})} +CAPS = {JOB: (45, {}), LATEST_TIER: (45, {}), + EXPERIMENTAL: (210, {'python': 5, 'install': 10, 'watcher-python': 1, 'watcher': 1, 'tests': 190})} #: A pytest-xdist worker count on a command line: ``-n X``, ``-nX``, ``--numprocesses X`` or #: ``--numprocesses=X``, with or without ``--dist`` after it. WORKERS_FLAG = re.compile(r'(?[^\s;|&]+)') @@ -103,7 +108,7 @@ def step(where: 'tuple[str, str]' = JOB) -> 'dict[str, Any]': - name = STEP if where in (JOB, EXPERIMENTAL) else INTEGRATION_STEP + name = STEP if where in (JOB, EXPERIMENTAL, LATEST_TIER) else INTEGRATION_STEP found = [entry for entry in job(*where)['steps'] if entry.get('name') == name] if len(found) != 1: raise AssertionError(f'expected one {name!r} step in {where}, found {len(found)}') @@ -247,9 +252,14 @@ def ignored(argv: 'list[str]') -> 'list[str]': return [arg.split('=', 1)[1] for arg in pytest_args(argv) if arg.startswith('--ignore=') and arg not in TIER_FLAGS] +def selected(argv: 'list[str]') -> 'list[str]': + """The modules the arguments name for pytest to run, a ``test-latest-tier`` cell's half (#2751).""" + return [arg for arg in pytest_args(argv) if not arg.startswith('-') and arg.endswith('.py')] + + @unittest.skipIf(shutil.which('bash') is None, 'the step is a bash script') class TestTheStepSelectsByLeg(unittest.TestCase): - """Each leg of ``test`` and ``experimental``, from the tier file, through the step's own script.""" + """Each leg of ``test``, ``test-latest-tier`` and ``experimental``, from the tier file, through the step's own script.""" @classmethod def setUpClass(cls) -> None: @@ -260,34 +270,54 @@ def setUpClass(cls) -> None: else rest_unsharded(leg, where, run_step_each(leg, where)[-1])) for where in (JOB, EXPERIMENTAL) for leg in legs(*where)} assert len(cls.runs) == len(legs(*JOB)) + len(legs(*EXPERIMENTAL)), 'a version runs in both jobs' + # #2751: each half of the LATEST tier, on `latest`, by the same step. + cls.halves = {leg['half']: (leg, unsharded(leg, run_step_each(leg, LATEST_TIER))) for leg in legs(*LATEST_TIER)} def test_only_latest_runs_the_latest_tier(self) -> None: - """Every other leg ignores each ``LATEST`` module; ``latest`` ignores the project pins alone.""" + """Every leg of ``test``, ``latest``'s too, and of ``experimental`` ignores each ``LATEST`` module. + + #2751: ``test-latest-tier``'s cells, on ``latest``, run the tier less the project pins, a half each, by name. + + """ self.assertIn(self.latest, self.runs) self.assertTrue(any(is_experimental(leg) for leg, _ in self.runs.values())) tier, pins = list(_tiers.latest_tier_paths()), list(_tiers.project_pin_paths()) self.assertLess(set(pins), set(tier)) for version, (_, argv) in self.runs.items(): with self.subTest(python=version): - self.assertEqual(ignored(argv), pins if version == self.latest else tier) + self.assertEqual(ignored(argv), tier) + self.assertEqual(selected(argv), []) self.assertFalse([arg for arg in argv if 'roundtrip' in arg and arg.startswith('--ignore-glob')]) + self.assertEqual(sorted(self.halves), [str(half) for half in _tiers.LATEST_TIER_HALVES]) + for half, (leg, argv) in self.halves.items(): + with self.subTest(half=half): + self.assertEqual(leg['python-version'], self.latest) + self.assertEqual(ignored(argv), []) + self.assertEqual(selected(argv), list(_tiers.latest_tier_half(int(half)))) def test_latest_keeps_the_unit_tier_less_the_pins_under_coverage(self) -> None: """Coverage is measured over the whole unit tier, less the project pins, which - import no pcapkit and run in ``project-tests`` (#1954).""" - leg, argv = self.runs[self.latest] - self.assertIs(leg.get('coverage'), True) - self.assertEqual(argv[:4], ['-m', 'coverage', 'run', '--rcfile=.github/coverage.toml']) - flags = [arg for arg in pytest_args(argv) if arg.startswith('--ignore')] - pins = [f'--ignore={pin}' for pin in _tiers.project_pin_paths()] + import no pcapkit and run in ``project-tests`` (#1954): #2751, ``test``'s ``latest`` leg over + the ``ALL`` tier and each half of ``test-latest-tier`` over its part of the ``LATEST`` one.""" + covered = [self.runs[self.latest]] + [self.halves[half] for half in sorted(self.halves)] + for leg, argv in covered: + with self.subTest(leg=leg): + self.assertIs(leg.get('coverage'), True) + self.assertEqual(argv[:4], ['-m', 'coverage', 'run', '--rcfile=.github/coverage.toml']) + tier, pins = set(_tiers.latest_tier_paths()), set(_tiers.project_pin_paths()) self.assertTrue(pins) - self.assertEqual(flags, list(TIER_FLAGS) + pins) + unit = [module for module in _tiers.suite_modules() if _tiers.is_unit_tier(module)] + # The `latest` leg collects the unit tier less the LATEST one; each half names its modules. + ran = [module for module in unit if module not in tier] + for _, argv in covered[1:]: + self.assertEqual([arg for arg in pytest_args(argv) if arg.startswith('--ignore')], list(TIER_FLAGS)) + ran.extend(selected(argv)) + self.assertEqual(len(ran), len(set(ran))) + self.assertEqual(sorted(ran), sorted(module for module in unit if module not in pins)) def test_every_other_leg_runs_the_rest(self) -> None: - """The same selection as ``latest``, the ``LATEST`` tier ignored in place of the pins, and no coverage.""" - pins = {f'--ignore={pin}' for pin in _tiers.project_pin_paths()} - expected = ([arg for arg in pytest_args(self.runs[self.latest][1]) if arg not in pins] - + [f'--ignore={module}' for module in _tiers.latest_tier_paths()]) + """The same selection as ``latest``, the ``LATEST`` tier ignored, and no coverage.""" + expected = pytest_args(self.runs[self.latest][1]) for version, (leg, argv) in self.runs.items(): if version == self.latest: continue diff --git a/tests/project/test_unit_tests_push_reports_2030.py b/tests/project/test_unit_tests_push_reports_2030.py index 2db7b43184..c8afffef90 100644 --- a/tests/project/test_unit_tests_push_reports_2030.py +++ b/tests/project/test_unit_tests_push_reports_2030.py @@ -93,14 +93,15 @@ #: Each job that writes its summary, as ``(workflow, job)``: the KTLO summaries, #: the coverage table, the engine cells' table and the experimental summaries. SUMMARISED = ( - ('unit-tests.yml', 'test'), ('unit-tests.yml', 'test-ktlo'), ('unit-tests.yml', 'integration'), + ('unit-tests.yml', 'test'), ('unit-tests.yml', 'test-ktlo'), ('unit-tests.yml', 'test-latest-tier'), + ('unit-tests.yml', 'coverage'), ('unit-tests.yml', 'integration'), ('unit-tests.yml', 'integration-ktlo'), ('unit-tests.yml', 'engine-tests'), ('unit-tests.yml', 'pypcap-parity'), ('unit-tests.yml', 'pypcap-parity-ktlo'), ('experimental.yml', 'experimental'), ('experimental.yml', 'windows'), ('python-compatibility.yml', 'compatibility'), ) #: The steps that copy the summary through util/_actions.py's JOB_SUMMARY_COPY. -COPYING = ('Report coverage', 'KTLO summary', 'Experimental summary') +COPYING = ('Check coverage data', 'Report coverage', 'KTLO summary', 'Experimental summary') #: The shell's copy of a summary line. TEE = '| tee -a "$RUNNER_TEMP/job-summary.md" >> "$GITHUB_STEP_SUMMARY"' diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index 48851467c8..a32062da04 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -61,13 +61,14 @@ #: ``Classify the diff``'s command, as the job's checkout root runs it. CLASSIFY_RUN = ' run: python3 util/classify_changes.py\n' -#: The legs a docs-only pull request may skip. -LEGS = ('test', 'integration', 'engine-tests', 'pypcap-parity') +#: The legs a docs-only pull request may skip; #2751 split the ``LATEST`` tier and +#: the coverage report out of ``test``. +LEGS = ('test', 'test-latest-tier', 'coverage', 'integration', 'engine-tests', 'pypcap-parity') #: The legs a test-only pull request may skip (#2243); the others run ``latest`` alone. TEST_ONLY_SKIPS = ('engine-tests', 'pypcap-parity') #: The variable each leg's result reaches the gate's step in. -LEG_ENV = {'test': 'TEST', 'integration': 'INTEGRATION', 'engine-tests': 'ENGINE_TESTS', - 'pypcap-parity': 'PYPCAP_PARITY'} +LEG_ENV = {'test': 'TEST', 'test-latest-tier': 'TEST_LATEST_TIER', 'coverage': 'COVERAGE', + 'integration': 'INTEGRATION', 'engine-tests': 'ENGINE_TESTS', 'pypcap-parity': 'PYPCAP_PARITY'} def job(name: 'str') -> 'str': @@ -118,7 +119,7 @@ def setUp(self) -> None: def gate(self, code: 'str', legs: 'str', project: 'str', changes: 'str' = 'success', test_only: 'str' = 'false', **override: 'str') -> 'int': """The gate's exit status with every leg reporting ``legs`` unless overridden.""" - results = {'TEST': legs, 'INTEGRATION': legs, 'ENGINE_TESTS': legs, 'PYPCAP_PARITY': legs} + results = dict.fromkeys(LEG_ENV.values(), legs) results.update(override) env = dict(os.environ, CHANGES=changes, CODE=code, TEST_ONLY=test_only, PROJECT_TESTS=project, **results) @@ -159,7 +160,7 @@ def test_docs_only_still_rejects_a_failed_leg(self) -> None: def test_every_leg_is_checked_on_its_own(self) -> None: """One failing leg among passing ones fails the gate, on either path.""" - for env_name in ('TEST', 'INTEGRATION', 'ENGINE_TESTS', 'PYPCAP_PARITY'): + for env_name in LEG_ENV.values(): with self.subTest(leg=env_name): for code, legs, project, odd in (('true', 'success', 'success', 'failure'), ('true', 'success', 'success', 'skipped'), @@ -290,12 +291,13 @@ class TestJobGating(unittest.TestCase): """Which jobs run on which events, read off their job-level keys.""" def test_each_leg_skips_only_on_a_docs_only_diff(self) -> None: - """Each leg is gated on ``changes`` by a job-level ``if:``.""" + """Each leg is gated on ``changes`` by a job-level ``if:``; ``coverage`` waits on the legs it combines.""" for name in LEGS: with self.subTest(job=name): head = header(job(name)) self.assertIn("needs.changes.outputs.code != 'false'", head) - self.assertRegex(head, r'(?m)^ needs: \[changes, python-versions\]$') + needs = 'changes, test, test-latest-tier' if name == 'coverage' else 'changes, python-versions' + self.assertRegex(head, rf'(?m)^ needs: \[{needs}\]$') def test_project_tests_runs_on_every_diff_and_never_duplicates_test(self) -> None: """#1954: ``project-tests`` runs on every diff, but not under ``gate-only``, whose ``gate`` diff --git a/tests/project/test_workflow_scripts_1967.py b/tests/project/test_workflow_scripts_1967.py index 1721c74c19..3c27878ffb 100644 --- a/tests/project/test_workflow_scripts_1967.py +++ b/tests/project/test_workflow_scripts_1967.py @@ -222,14 +222,63 @@ def test_every_stage_on_a_real_coverage_run(self) -> None: self.assertEqual(json.loads((self.tmp / 'coverage-comment' / 'coverage-summary.json').read_text())['total'], 100.0) + def test_the_combine_stage_on_two_real_parts(self) -> None: + """#2751: two legs' parts, each from a real ``coverage run`` that its check passed, combined.""" + try: + import coverage # pylint: disable=import-outside-toplevel + except ImportError: # pragma: no cover + self.skipTest('needs coverage') + if tuple(int(x) for x in coverage.__version__.split('.')[:2]) < (7, 10): # pragma: no cover + self.skipTest('needs coverage 7.10+ (`[run] patch`)') + (self.tmp / '.github').mkdir() + shutil.copy(ROOT / '.github' / 'coverage.toml', self.tmp / '.github' / 'coverage.toml') + (self.tmp / 'pcapkit').mkdir() + (self.tmp / 'pcapkit' / '__init__.py').write_text('def f(x):\n if x:\n return 1\n return 2\n', + encoding='utf-8') + env = {k: v for k, v in os.environ.items() if not k.startswith('COVERAGE')} + env.update(COVERAGE_DEBUG='pid,process', COVERAGE_DEBUG_FILE=str(self.tmp / 'coverage-processes.log'), + PYTHONPATH=str(self.tmp)) + # Both parts' tests passed, so `TESTS_OUTCOME`'s failure in the environment is not read. + for part, argument in (('all', 1), ('latest-half-1', 0)): + (self.tmp / 'spawn.py').write_text( + 'import subprocess, sys\n' + f'subprocess.run([sys.executable, "-c", "import pcapkit; pcapkit.f({argument})"], check=True)\n', + encoding='utf-8') + subprocess.run([sys.executable, '-m', 'coverage', 'run', '--rcfile=.github/coverage.toml', # nosec: B603 + 'spawn.py'], cwd=self.tmp, env=env, check=True, timeout=120, capture_output=True) + checked = run('coverage_report.py', 'check', env=self.env(), cwd=self.tmp) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + # What `Upload coverage data` takes, as `Download coverage data` lays it out. + folder = self.tmp / 'coverage-data' / f'coverage-data-{part}' + folder.mkdir(parents=True) + (folder / 'coverage-tests-outcome').write_text('success\n', encoding='utf-8') + for path in self.tmp.glob('.coverage.*'): + path.rename(folder / path.name) + (self.tmp / 'coverage-processes.log').unlink() + result = run('coverage_report.py', 'combine', 'coverage-data/coverage-data-all', + 'coverage-data/coverage-data-latest-half-1', env=self.env(), cwd=self.tmp) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('coverage-data/coverage-data-latest-half-1: 2 data files, tests success', result.stdout) + self.assertTrue((self.tmp / 'coverage-html' / 'index.html').is_file()) + page = self.page.read_text(encoding='utf-8') + # Each part took one branch of `f`; only the two together cover it. + self.assertRegex(page, r'^earlier\n\*\*Coverage: 100\.00%\*\* \(unit tier, Python 3\.15, `012345678`\)\n') + self.assertEqual(json.loads((self.tmp / 'coverage-comment' / 'coverage-summary.json').read_text())['tests'], + 'success') + def test_the_step_runs_the_script(self) -> None: - step = next(step for step in steps('unit-tests.yml', 'test') - if step.get('name') == 'Report coverage') - self.assertEqual(step['run'], 'python util/coverage_report.py') - self.assertEqual(step['shell'], 'bash') + """#2751: each coverage leg checks its part, and ``coverage`` combines the parts.""" + check = next(step for step in steps('unit-tests.yml', 'test') if step.get('name') == 'Check coverage data') + self.assertEqual(check['run'], "printf '%s\\n' \"$TESTS_OUTCOME\" > coverage-tests-outcome\n" + 'python util/coverage_report.py check\n') # #2030: and the job's copy of its summary. - self.assertEqual(set(step['env']), {'SHA', 'PYTHON', 'TESTS_OUTCOME', 'JOB_SUMMARY_COPY'}) - self.assertTrue(re.search(r"steps\.tests\.outcome", step['env']['TESTS_OUTCOME'])) + self.assertEqual(set(check['env']), {'TESTS_OUTCOME', 'JOB_SUMMARY_COPY'}) + self.assertTrue(re.search(r"steps\.tests\.outcome", check['env']['TESTS_OUTCOME'])) + report = next(step for step in steps('unit-tests.yml', 'coverage') if step.get('name') == 'Report coverage') + self.assertTrue(report['run'].startswith('python util/coverage_report.py combine '), report['run']) + self.assertEqual(set(report['env']), {'SHA', 'PYTHON', 'JOB_SUMMARY_COPY'}) + for step in (check, report): + self.assertEqual(step['shell'], 'bash') if __name__ == '__main__': diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index a0333eb76c..6ab003926b 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -285,11 +285,13 @@ def test_integration_job_selects_positively_by_asking_tiers_for_it(self) -> None @unittest.skipIf(BASH is None, 'the steps are bash scripts') def test_the_legs_leave_out_exactly_the_latest_tier_or_the_pins(self) -> None: - """#1958: run as bash runs them, the ``test`` legs but ``latest`` ignore each ``LATEST`` module. + """#1958: run as bash runs them, the ``test`` legs, ``latest``'s too, ignore each ``LATEST`` module. - ``latest`` ignores each project pin (#1954), which ``project-tests`` runs, - and runs the rest of the ``LATEST`` tier itself. Every step asks - :data:`~tests._tiers.LATEST_TIER_SELECTOR` or + #2751: each cell of ``test-latest-tier`` runs its half of the ``LATEST`` tier + by name, and the halves leave out each project pin (#1954), which + ``project-tests`` runs. Every step asks + :data:`~tests._tiers.LATEST_TIER_SELECTOR`, + :data:`~tests._tiers.LATEST_HALF_SELECTOR` or :data:`~tests._tiers.PROJECT_PINS_SELECTOR` at run time, so none can drift from :func:`~tests._tiers.module_tier`; what can is how each uses the answer, which is what running them checks. experimental.yml asks in a step @@ -304,11 +306,20 @@ def test_the_legs_leave_out_exactly_the_latest_tier_or_the_pins(self) -> None: latest = list(_tiers.latest_tier_paths()) self.assertTrue(pins) self.assertLess(len(pins), len(latest)) - self.assertEqual(leg_ignores(text, 'true', self.selectors), pins) - self.assertEqual(leg_ignores(text, 'false', self.selectors), latest) + self.assertEqual(leg_ignores(text, cache=self.selectors), latest) + self.assertEqual(leg_selects(text, cache=self.selectors), []) + halves = [] # type: list[str] + for half in _tiers.LATEST_TIER_HALVES: + with self.subTest(half=half): + selected = leg_selects(text, str(half), self.selectors) + self.assertTrue(selected) + self.assertEqual(selected, list(_tiers.latest_tier_half(half))) + self.assertEqual(leg_ignores(text, str(half), self.selectors), []) + halves.extend(selected) + self.assertEqual(sorted(halves + pins), sorted(path for path in latest if _tiers.is_unit_tier(path))) self.assertEqual(project_pins_run(text, self.selectors), pins) for workflow, job, step, selector in ( - (WORKFLOW, 'test', 'Run unit tests', _tiers.PROJECT_PINS_SELECTOR), + (WORKFLOW, 'test', 'Run unit tests', _tiers.LATEST_HALF_SELECTOR), (WORKFLOW, 'test', 'Run unit tests', _tiers.LATEST_TIER_SELECTOR), (WORKFLOW, 'project-tests', 'Run the project pins', _tiers.PROJECT_PINS_SELECTOR), (EXPERIMENTAL_WORKFLOW, 'experimental', 'List the LATEST tier', @@ -316,7 +327,7 @@ def test_the_legs_leave_out_exactly_the_latest_tier_or_the_pins(self) -> None: with self.subTest(job=job, selector=selector): script = run_script(job_section(workflow.read_text(encoding='utf-8'), job), step) self.assertIn(selector, script) - # The experimental legs ignore what that step printed, as `test`'s legs but `latest` do. + # The experimental legs ignore what that step printed, as `test`'s legs do. tests = step_run_block(job_section(EXPERIMENTAL_WORKFLOW.read_text(encoding='utf-8'), 'experimental'), 'Run unit tests') self.assertIn('LATEST_TIER: ${{ steps.tier.outputs.paths }}', tests) @@ -330,24 +341,31 @@ def test_a_leg_that_stops_ignoring_the_latest_tier_is_caught(self) -> None: text = WORKFLOW.read_text(encoding='utf-8') loop = 'set -- "$@" --ignore="$module"' self.assertEqual(text.count(loop), 1, 'the loop this test removes has been reworded') - for latest in ('true', 'false'): - with self.subTest(latest=latest): - self.assertEqual(leg_ignores(text.replace(loop, ':'), latest, self.selectors), []) + self.assertEqual(leg_ignores(text.replace(loop, ':'), cache=self.selectors), []) + # #2751: and a cell that computes its half and then drops it, which would run the whole suite. + loop = 'set -- "$@" "$module"' + self.assertEqual(text.count(loop), 1, 'the loop this test removes has been reworded') + self.assertEqual(leg_selects(text.replace(loop, ':'), '1', self.selectors), []) @unittest.skipIf(BASH is None, 'the steps are bash scripts') def test_a_leg_that_asks_for_the_wrong_list_is_caught(self) -> None: - """Off ``latest``, the pins alone are not the ``LATEST`` tier; on it, the tier is not the pins.""" + """The pins alone are not the ``LATEST`` tier, and one half is not the other (#2751).""" if not WORKFLOW.is_file(): self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') text = WORKFLOW.read_text(encoding='utf-8') latest = _tiers.LATEST_TIER_SELECTOR.split('"', 1)[1] pins = _tiers.PROJECT_PINS_SELECTOR.split('"', 1)[1] self.assertEqual(text.count(latest), 1) - self.assertEqual(text.count(pins), 2) # the `test` step's and `project-tests`'s - self.assertNotEqual(leg_ignores(text.replace(latest, pins), 'false', self.selectors), + self.assertEqual(text.count(pins), 1) # `project-tests`'s + self.assertNotEqual(leg_ignores(text.replace(latest, pins), cache=self.selectors), list(_tiers.latest_tier_paths())) - swapped = text.replace(pins, latest, 1) - self.assertNotEqual(leg_ignores(swapped, 'true', self.selectors), list(_tiers.project_pin_paths())) + asked = 'latest_tier_half(int(sys.argv[1]))' + self.assertEqual(text.count(asked), 1) + first = _tiers.LATEST_TIER_HALVES[0] + for half in _tiers.LATEST_TIER_HALVES[1:]: + with self.subTest(half=half): + self.assertNotEqual(leg_selects(text.replace(asked, f'latest_tier_half({first})'), str(half), + self.selectors), list(_tiers.latest_tier_half(half))) #: The experimental legs' workflow (#1732), whose unit-tier step leaves the ``LATEST`` tier out too. @@ -453,20 +471,29 @@ def ask(code: 'str') -> 'subprocess.CompletedProcess[str]': self.assertEqual(real.read_text(encoding='utf-8').splitlines(), ['-c a', '-c b', '-c fail', '-c fail']) -def leg_ignores(text: 'str', latest: 'str', cache: 'str' = '') -> 'list[str]': - """The modules a ``test`` leg's ``Run unit tests`` step ``--ignore``\\ s, ``latest`` or not. +def leg_argv(text: 'str', half: 'str' = '', cache: 'str' = '') -> 'list[str]': + """What the ``Run unit tests`` step hands pytest on a ``test`` leg, or with ``half`` a ``test-latest-tier`` cell. ``cache`` is a directory for :data:`PYTHON_STUB`'s ``$SELECTOR_CACHE``; empty runs every selector. """ script = run_script(job_section(text, 'test'), 'Run unit tests') # #2005: a leg that runs pytest once, as a PyPy leg's own `SHARDS` would otherwise reach the stubbed step. - argv = stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'LATEST': latest, 'SHARDS': '', + return stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'HALF': half, 'SHARDS': '', 'SELECTOR_CACHE': cache}) - return [arg.split('=', 1)[1] for arg in argv + + +def leg_ignores(text: 'str', half: 'str' = '', cache: 'str' = '') -> 'list[str]': + """The modules :func:`leg_argv` ``--ignore``\\ s, less the fixture tier's directory.""" + return [arg.split('=', 1)[1] for arg in leg_argv(text, half, cache) if arg.startswith('--ignore=') and arg != f'--ignore=tests/{next(iter(_tiers.FIXTURE_TIER_DIRS))}'] +def leg_selects(text: 'str', half: 'str' = '', cache: 'str' = '') -> 'list[str]': + """The modules :func:`leg_argv` names for pytest to run, a ``test-latest-tier`` cell's half (#2751).""" + return [arg for arg in leg_argv(text, half, cache) if not arg.startswith('-') and arg.endswith('.py')] + + def project_pins_run(text: 'str', cache: 'str' = '') -> 'list[str]': """The modules ``project-tests``'s ``Run the project pins`` step hands pytest. diff --git a/util/check_suite_report.py b/util/check_suite_report.py index 3670991e3a..be222e9c8a 100644 --- a/util/check_suite_report.py +++ b/util/check_suite_report.py @@ -94,7 +94,8 @@ class Workflow(NamedTuple): #: pypy3.10 installs under #2040's cryptography cap. WORKFLOWS = { 'Unit Tests': Workflow( - 'unit-tests', re.compile(r'(?:Python |Integration Python |Engines \(|PyPCAP/PyPCAPFile parity Python )'), + 'unit-tests', re.compile(r'(?:Python |Integration Python |Engines \(|PyPCAP/PyPCAPFile parity Python ' + r'|Coverage$)'), {}), 'Experimental Tests': Workflow('experimental', re.compile(r'Python .* \(experimental\)$'), {}), 'Python Compatibility': Workflow('python-compatibility', re.compile(r'Compat Python '), {}), diff --git a/util/ci_timing.py b/util/ci_timing.py index b368fc3690..ddf1e07f91 100644 --- a/util/ci_timing.py +++ b/util/ci_timing.py @@ -142,10 +142,13 @@ def _coverage_legs() -> 'tuple[str, str, tuple[str, ...]]': #: 3.13 -- the newest such leg when W4's baseline was measured -- so that W4 compares #: one interpreter with itself week over week. MS_PER_TEST_LEG = '3.13' -#: The JUnit artifacts read, by leg, and from how many push runs. ``latest``'s holds -#: the ``LATEST`` tier, which no other leg runs (#2571). +#: The JUnit artifacts read, by leg, and from how many push runs. ``latest``'s are +#: read with the ``LATEST`` tier's, which no other leg runs (#2571): one per half of +#: the job that runs it (#2751), each read as part of ``latest``'s. JUNIT_LEGS = tuple(dict.fromkeys(('3.13', '3.14', LATEST_VERSION))) JUNIT_ARTIFACT = 'junit-unit-{leg}' +JUNIT_HALF_ARTIFACT = 'junit-unit-{leg}-latest-half-{half}' +JUNIT_HALVES = (1, 2) JUNIT_RUNS = 5 #: Steps reported on their own, by full-match pattern against the step's name. @@ -711,16 +714,24 @@ def say(message: 'str') -> 'None': client.paginate(f'{base}/runs/{run["id"]}/artifacts', 'artifacts') if not item.get('expired')} for leg in JUNIT_LEGS: - artifact = artifacts.get(JUNIT_ARTIFACT.format(leg=leg)) - if artifact is None: - continue - raw = client.fetch(artifact['archive_download_url'])[0] - say(f'JUnit: run {run["id"]}, {JUNIT_ARTIFACT.format(leg=leg)}, {len(raw) / 1e6:.2f} MB.') - archive = zipfile.ZipFile(io.BytesIO(raw)) - members = [name for name in archive.namelist() if name.endswith('.xml')] - if members: - data['junit'].append({'run_id': run['id'], 'created_at': run['created_at'], - 'leg': leg, **parse_junit(archive.read(members[0]))}) + # #2751: a run's halves of the LATEST tier are one entry with its `latest` leg's. + names = [JUNIT_ARTIFACT.format(leg=leg)] + ([JUNIT_HALF_ARTIFACT.format(leg=leg, half=half) + for half in JUNIT_HALVES] if leg == LATEST_VERSION else []) + parsed = [] # type: list[dict[str, Any]] + for name in names: + artifact = artifacts.get(name) + if artifact is None: + continue + raw = client.fetch(artifact['archive_download_url'])[0] + say(f'JUnit: run {run["id"]}, {name}, {len(raw) / 1e6:.2f} MB.') + archive = zipfile.ZipFile(io.BytesIO(raw)) + members = [member for member in archive.namelist() if member.endswith('.xml')] + if members: + parsed.append(parse_junit(archive.read(members[0]))) + if parsed: + data['junit'].append({'run_id': run['id'], 'created_at': run['created_at'], 'leg': leg, + 'cases': [case for entry in parsed for case in entry['cases']], + 'suite_tests': sum(entry['suite_tests'] for entry in parsed)}) if not data['junit']: data['notes'].append(f'No {JUNIT_ARTIFACT.format(leg="*")} artifact in the newest ' f'{len(pushes)} successful push runs, so there ' diff --git a/util/coverage_report.py b/util/coverage_report.py index 1ab0497743..4cf078ec8a 100644 --- a/util/coverage_report.py +++ b/util/coverage_report.py @@ -1,28 +1,38 @@ # -*- coding: utf-8 -*- -"""The coverage leg's report: check the data is whole, combine it, and summarise it (#1387). - -The ``Report coverage`` step of :file:`.github/workflows/unit-tests.yml`'s ``test`` -job runs ``python util/coverage_report.py`` on the coverage leg, after ``Run unit -tests`` measured the unit tier with ``COVERAGE_DEBUG=pid,process`` logging every -measured process to :data:`LOG`. In the checkout, in order: - -1. :func:`check`: a worker killed before its atexit save leaves no ``.coverage.*`` - file, and ``coverage combine`` then reports the rest as the whole. So unless - the controller and every xdist worker, its direct children in the log, left - one, it writes **Coverage not reported** to the job summary and an +"""The coverage legs' report: check the data is whole, combine it, and summarise it (#1387). + +Each coverage leg of :file:`.github/workflows/unit-tests.yml` -- ``test``'s +``latest`` leg, which runs the ``ALL`` tier, and each cell of ``test-latest-tier``, +a half of the ``LATEST`` one (#2751) -- measures its part of the unit tier in ``Run +unit tests``, with ``COVERAGE_DEBUG=pid,process`` logging every measured process to +:data:`LOG`. Its ``Check coverage data`` step runs ``python util/coverage_report.py +check``, and uploads the part only if that passed; the ``coverage`` job's ``Report +coverage`` step then runs ``python util/coverage_report.py combine PART...``. In the +checkout, in order: + +1. :func:`check`, on each leg: a worker killed before its atexit save leaves no + ``.coverage.*`` file, and ``coverage combine`` then reports the rest as the + whole. So unless the controller and every xdist worker, its direct children in + the log, left one, it writes **Coverage not reported** to the job summary and an ``::error`` annotation, and exits 1 before anything is combined. #1423: each missing process is named with its command line and whether coverage's atexit handler ran in it at all. -2. :data:`COMMANDS`: ``coverage combine``, ``report``, ``html`` into +2. :func:`gather`, in ``coverage``: each part's data moved into the checkout, and + the same failure, naming the part, for one that did not arrive. Its + :data:`OUTCOME` file holds its tests step's ``outcome``. +3. :data:`COMMANDS`: ``coverage combine``, ``report``, ``html`` into ``coverage-html/`` and ``json`` into ``coverage.json``, each under :data:`RCFILE`. The first to fail ends the script with its exit status. -3. :func:`summary`: writes ``coverage-comment/coverage-summary.json``, the +4. :func:`summary`: writes ``coverage-comment/coverage-summary.json``, the numbers coverage-comment.yml renders as a PR comment, and appends the - per-package table to ``$GITHUB_STEP_SUMMARY``. ``TESTS_OUTCOME``, the tests - step's ``outcome``, marks a total measured over failing tests as partial; - ``PYTHON`` and ``SHA`` name the leg and the commit. + per-package table to ``$GITHUB_STEP_SUMMARY``. The tests' outcome -- the parts' + outcomes under ``combine``, ``TESTS_OUTCOME`` otherwise -- marks a total + measured over failing tests as partial; ``PYTHON`` and ``SHA`` name the + version and the commit. -``python util/coverage_report.py check`` and ``... summary`` run one stage alone. +Run without a stage, it runs :func:`check`, :data:`COMMANDS` and :func:`summary` on +one leg's data in the checkout. ``python util/coverage_report.py check`` and +``... summary`` run one stage alone. """ @@ -34,6 +44,7 @@ import json import os import re +import shutil # coverage, with a fixed argument list and no shell. import subprocess # nosec B404 import sys @@ -47,10 +58,12 @@ if TYPE_CHECKING: from typing import Optional, TextIO -__all__ = ['COMMANDS', 'LOG', 'RCFILE', 'check', 'main', 'summary'] +__all__ = ['COMMANDS', 'LOG', 'OUTCOME', 'RCFILE', 'check', 'gather', 'main', 'summary'] #: The process log ``Run unit tests`` writes through ``COVERAGE_DEBUG_FILE``. LOG = 'coverage-processes.log' +#: The file a coverage leg writes beside its data: its tests step's ``outcome`` (#2751). +OUTCOME = 'coverage-tests-outcome' #: The coverage leg's configuration, relative to the checkout. RCFILE = '.github/coverage.toml' #: The coverage commands between :func:`check` and :func:`summary`, each run as ``python -m coverage``. @@ -91,21 +104,54 @@ def check(log: 'str' = LOG) -> 'int': print(f' pid {pid}: cmd {cmd.get(pid, "")}') print(f' pid {pid}: {atexit_.get(pid, "no atexit/sigterm entry -- died before saving")}') if len(root) != 1 or not workers or missing: - why = f'no data file from pid {missing}' if missing else 'the process log is missing or unreadable' - _actions.summary(f'**Coverage not reported:** {why} (#1387); a total would under-count.') - # #2047: deliberately unescaped. - _actions.annotate('error', f'Coverage data is incomplete: {why}.', escape=False) - return 1 + return _not_reported(f'no data file from pid {missing}' if missing + else 'the process log is missing or unreadable') return 0 +def _not_reported(why: 'str') -> 'int': + """Say no total is reported, and why, in the job summary and an error annotation; 1.""" + _actions.summary(f'**Coverage not reported:** {why} (#1387); a total would under-count.') + # #2047: deliberately unescaped. + _actions.annotate('error', f'Coverage data is incomplete: {why}.', escape=False) + return 1 + + +def gather(parts: 'list[str]') -> 'tuple[int, str]': + """Move each part's data files into the checkout, and the tests' outcome over every part (#2751). + + Each of ``parts`` is a directory a coverage leg uploaded once :func:`check` + passed on it, holding its ``.coverage.*`` files and :data:`OUTCOME`. A missing + one fails as an incomplete check does, before anything is moved. The outcome is + ``success`` when every part's is, else the first part's that is not. Returns + the exit status and the outcome. + + """ + absent = [part for part in parts if not os.path.isfile(os.path.join(part, OUTCOME))] + if not parts or absent: + return _not_reported(f'no data from {absent}' if absent else 'no part was named'), '' + outcomes = [] + for part in parts: + with open(os.path.join(part, OUTCOME), encoding='utf-8') as file: + outcomes.append(file.read().strip()) + files = glob.glob(os.path.join(part, '.coverage.*')) + print(f'{part}: {len(files)} data files, tests {outcomes[-1]}') + for path in files: + shutil.move(path, os.path.basename(path)) + return 0, next((outcome for outcome in outcomes if outcome != 'success'), 'success') + + def _percent(counts: 'collections.Counter[str]') -> 'float': total = counts['num_statements'] + counts['num_branches'] return 100.0 * (counts['covered_lines'] + counts['covered_branches']) / total if total else 100.0 -def summary(page: 'TextIO') -> 'int': - """Write ``coverage-comment/coverage-summary.json`` from ``coverage.json``, and the table to ``page``.""" +def summary(page: 'TextIO', tests: 'Optional[str]' = None) -> 'int': + """Write ``coverage-comment/coverage-summary.json`` from ``coverage.json``, and the table to ``page``. + + ``tests`` is the tests' outcome, ``TESTS_OUTCOME`` when not given. + + """ with open('coverage.json', encoding='utf-8') as file: data = json.load(file) packages = collections.defaultdict(collections.Counter) # type: dict[str, collections.Counter[str]] @@ -116,7 +162,7 @@ def summary(page: 'TextIO') -> 'int': rows = [[name, counts['num_statements'], counts['missing_lines'], counts['num_branches'], counts['num_partial_branches'], round(_percent(counts), 2)] for name, counts in sorted(packages.items())] total = round(data['totals']['percent_covered'], 2) - tests = os.environ['TESTS_OUTCOME'] + tests = os.environ['TESTS_OUTCOME'] if tests is None else tests os.makedirs('coverage-comment', exist_ok=True) with open('coverage-comment/coverage-summary.json', 'w', encoding='utf-8') as file: json.dump({'total': total, 'tests': tests, 'packages': rows, 'python': os.environ['PYTHON']}, file) @@ -133,22 +179,30 @@ def summary(page: 'TextIO') -> 'int': return 0 -def _summary_to_job_page() -> 'int': +def _summary_to_job_page(tests: 'Optional[str]' = None) -> 'int': with _actions.open_summary() as page: - return summary(page) + return summary(page, tests) def main(argv: 'Optional[list[str]]' = None) -> 'int': """Run ``stage``, or every stage in order; the exit status of the first to fail.""" parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) - parser.add_argument('stage', nargs='?', choices=('all', 'check', 'summary'), default='all', - help='the stage to run alone (default: every stage)') + parser.add_argument('stage', nargs='?', choices=('all', 'check', 'combine', 'summary'), default='all', + help='the stage to run alone, or `combine` to report the parts (default: every stage)') + parser.add_argument('parts', nargs='*', metavar='PART', + help='for `combine`: each directory a coverage leg uploaded (#2751)') args = parser.parse_args(argv) + if args.parts and args.stage != 'combine': + parser.error('only `combine` takes parts') if args.stage == 'check': return check() if args.stage == 'summary': return _summary_to_job_page() - status = check() + tests = None # type: Optional[str] + if args.stage == 'combine': + status, tests = gather(args.parts) + else: + status = check() if status: return status for command in COMMANDS: @@ -156,7 +210,7 @@ def main(argv: 'Optional[list[str]]' = None) -> 'int': status = subprocess.run([sys.executable, '-m', 'coverage', *command], check=False).returncode # nosec B603 if status: return status - return _summary_to_job_page() + return _summary_to_job_page(tests) if __name__ == '__main__':