From ddbd9a3b0152970cd6d0a24f7cc1808e63440234 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 18:45:01 -0400 Subject: [PATCH] ci(python): contain and sample the experimental legs so a lost runner becomes a warning (#1740) GraalPy's leg loses its runner 18-30 minutes into its tests, which fails the job, and so its check, past any `continue-on-error`. experimental.yml: - `Start the resource monitor`: a detached sampler, outside the tests' scope, writes `free -m`, `df -h`, /proc/loadavg, the OOM-kill count, the scope's memory and the top five RSS processes every 30 s, each line led by `[monitor]`; `Run unit tests` streams it with `tail -F`; the stop step and the `resource-monitor-` upload are `if: always()`. - `Run unit tests`: GraalPy alone gets GRAAL_PYTHON_VM_ARGS=--vm.Xmx2g and `-n 3`; pytest runs under `timeout --signal=INT --kill-after=3m 110m` and `prlimit --fsize=2147483648`, in a 12G, swapless, CPUWeight=20 systemd-run scope, probed with the same flags under a throwaway unit, else uncapped on the same argv. It writes the exit status, elapsed seconds, OOM kills and MemoryErrors to `tests-status`. - `Containment summary`, a new step before `Experimental summary`, warns of no scope, an OOM kill, a timeout (exit 124, or 137 with no OOM kill after 110 min) and a MemoryError, each with the last sample. `Experimental summary` is unchanged, the KTLO summary's script. - No cap, check name or job changes; nothing gates on it. Tests in tests/project/ pin and run each part; workflows.rst describes them. Refs #1740 --- .github/workflows/experimental.yml | 213 ++++++- docs/source/contributing/workflows.rst | 57 ++ .../project/test_ktlo_legs_stay_green_1672.py | 4 +- tests/project/test_resource_monitor_1740.py | 539 ++++++++++++++++++ .../test_sweeps_on_latest_only_1732.py | 18 +- 5 files changed, 824 insertions(+), 7 deletions(-) create mode 100644 tests/project/test_resource_monitor_1740.py diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 753fa308b..2a2b6bbb3 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -138,6 +138,12 @@ jobs: # 3.14 run and the 3.14, 3.15 and 3.14t legs of run 38581834697: 88-113 # minutes on GraalPy (estimate). 3.14t, which shares the caps, took 5.8 # minutes over the whole unit tier in that run. + # + # #1740 gives GraalPy 3 workers, not 4, so at 4/3 of that time its tests should + # take about 117-151 minutes (an inference from the 88-113, not measured), + # past the 110-minute `timeout` in `Run unit tests`. GraalPy's leg may then end + # on that timeout, which `Containment summary` reports as a warning: that is + # expected, blocks nothing, and leaves the caps as they are. experimental: name: Python ${{ matrix.python-version }} (experimental) if: ${{ needs.changes.outputs.code != 'false' }} @@ -191,23 +197,174 @@ jobs: nproc python -c "import os; print('cpu_count', os.cpu_count())" + # #1740: GraalPy's leg loses its runner 18-30 minutes into its tests ("The + # runner has received a shutdown signal"), under a 120-minute cap. So every + # 30 s this samples `free -m`, `df -h` of `/` and the workspace's volume, + # /proc/loadavg and the five processes with the most RSS, each line led by + # `[monitor]`, into $RUNNER_TEMP/resource-monitor.log. The sampler holds none + # of this step's output: the runner kills a step's process tree 5 s after + # its shell exits while a child still holds it (actions/runner + # src/Runner.Sdk/ProcessInvoker.cs:485-505, at 796cc18ff). `Run unit tests` + # follows the file into its own log instead, which is the evidence a lost + # runner leaves: once it has the shutdown signal, the runner evaluates no + # later step's condition, `always()` included (src/Runner.Worker/StepsRunner.cs:205-208, + # same commit), and job 116057255255 skipped every step after the tests. + # `set +eo pipefail`: `shell: bash` sets both, and `ps | head` can end + # `ps` with SIGPIPE. Started here, the sampler is outside the tests' memory + # scope (`Run unit tests`), so an OOM kill there leaves it running; each + # sample also holds the kernel's OOM-kill count and, while it exists, the + # scope's own memory use and events. + - name: Start the resource monitor + id: monitor + if: ${{ steps.install.outcome == 'success' }} + continue-on-error: true + shell: bash + run: | + log="$RUNNER_TEMP/resource-monitor.log" + : > "$log" + scope=/sys/fs/cgroup/system.slice/pcapkit-tests.scope + sample() { + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) sample $1" + free -m + df -h / "$GITHUB_WORKSPACE" + echo "loadavg $(cat /proc/loadavg)" + echo "oom_kill $(awk '$1 == "oom_kill" { print $2 }' /proc/vmstat)" + if [ -r "$scope/memory.events" ]; then + echo "scope memory.current $(cat "$scope/memory.current") $(tr '\n' ' ' < "$scope/memory.events")" + fi + ps -eo pid,rss,vsz,etime,args --sort=-rss | head -6 | cut -c1-300 + } + ( + set +eo pipefail + trap 'kill "$sleeper" 2>/dev/null; exit 0' TERM + count=0 + while :; do + count=$((count + 1)) + # One write a sample, opening on a newline: `Run unit tests` copies + # it into a log where pytest may have left a line unfinished. + printf '\n%s\n' "$(sample "$count" 2>&1 | sed 's/^/[monitor] /')" + sleep 30 & + sleeper=$! + wait "$sleeper" + done + ) >> "$log" 2>&1 < /dev/null & + echo "$!" > "$RUNNER_TEMP/resource-monitor.pid" + echo "Sampling every 30 s into $log, pid $!." + # unit-tests.yml `test`'s selection on every leg but `latest`: no # experimental version is `latest`, which util/python_versions.py holds to # a supported CPython, so the round-trip sweeps (#1732) are always left out - # here. + # here. #1740: `tail -F` copies the resource monitor's samples into this + # step's log as they are taken, and stops with the step. + # + # #1740, containment: a lost runner fails the job, and so the check, which + # no `continue-on-error` prevents, so the tests are held to what leaves the + # runner alive, and a breach fails this step instead: + # - GraalPy alone: a 2 GiB heap per process, read by every GraalPy + # process, xdist workers included (GRAAL_PYTHON_VM_ARGS), and 3 workers, + # which leaves a CPU of the 4 to the JIT and the runner. + # - `timeout` sends SIGINT at 110 minutes, under the 120-minute cap, so + # pytest still writes the junit XML, and exits 124; SIGKILL 3 minutes on. + # - `prlimit --fsize`: no file over 2 GiB. + # - A transient scope, 12G and no swap, at a fifth of the runner's CPU + # weight, where `sudo -n` can start one; otherwise uncapped. A scope's + # command "will thus inherit the execution environment of the caller" + # (systemd-run(1), systemd v255 man/systemd-run.xml:70-71), but sudo + # resets PATH to its secure_path and drops `LD_*` even under + # --preserve-env (sudo plugins/sudoers/env.c:133-141, at 43defa130), + # where setup-python puts its lib/ (src/find-python.ts:163-170, at + # ff73b2463), so `env` sets those again, and the heap too. systemd-run + # drops to the runner's uid and gid itself, with no setgroups + # (src/run/run.c:1645-1683, v255). The probe is `scoped` itself, every + # flag included, under a throwaway unit and running `true`, and either + # way the tests run the one argv, in the scope or not. + # `tests-status` holds the exit status, the scope and the probe's error, + # the seconds the tests took, the kernel's OOM kills while they ran + # (/proc/vmstat `oom_kill`) and the MemoryError count, for `Containment + # summary`. - name: Run unit tests id: tests if: ${{ steps.install.outcome == 'success' }} continue-on-error: true timeout-minutes: 120 + env: + GRAAL_PYTHON_VM_ARGS: ${{ startsWith(matrix.python-version, 'graalpy') && '--vm.Xmx2g' || '' }} + WORKERS: ${{ startsWith(matrix.python-version, 'graalpy') && '3' || 'auto' }} run: | - set -- -q -n auto --dist load \ + tail -n +1 -F "$RUNNER_TEMP/resource-monitor.log" 2>/dev/null & + monitor=$! + trap 'kill "$monitor" 2>/dev/null || true' EXIT + set -- -q -n "$WORKERS" --dist load \ --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ --ignore-glob='*_regression.py' \ --ignore-glob='*roundtrip*' - python -m pytest "$@" + set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m python -m pytest "$@" + scoped() { + unit=$1 + shift + sudo -n --preserve-env systemd-run --scope --quiet --unit="$unit" \ + -p MemoryMax=12G -p MemorySwapMax=0 -p CPUWeight=20 --uid="$(id -u)" --gid="$(id -g)" \ + -- env PATH="$PATH" HOME="$HOME" LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}" \ + GRAAL_PYTHON_VM_ARGS="$GRAAL_PYTHON_VM_ARGS" "$@" + } + if probe=$(scoped "pcapkit-probe-$$.scope" true 2>&1); then + scope=12G + probe= + set -- scoped pcapkit-tests.scope "$@" + else + scope=none + probe=$(echo "$probe" | tail -n 1) + # A log line, not an annotation: on a lost runner this log is all that is left. + echo "No 12G scope: the probe failed (${probe}), so the tests run uncapped." + fi + oom_kills() { awk '$1 == "oom_kill" { print $2 }' /proc/vmstat; } + before=$(oom_kills) + started=$SECONDS + set +e + "$@" 2>&1 | tee "$RUNNER_TEMP/pytest.log" + status=${PIPESTATUS[0]} + set -e + elapsed=$((SECONDS - started)) + after=$(oom_kills) + memory_errors=$(grep -c MemoryError "$RUNNER_TEMP/pytest.log" || true) + printf 'exit=%s\nscope=%s\nprobe=%s\nelapsed=%s\noom_kills=%s\nmemory_errors=%s\n' \ + "$status" "$scope" "$probe" "$elapsed" "$(( ${after:-0} - ${before:-0} ))" "$memory_errors" \ + > "$RUNNER_TEMP/tests-status" + exit "$status" + + # #1740: after a failed, timed-out or cancelled tests step too. A lost + # runner skips it, as above, and its orphan cleanup ends the sampler. + - name: Stop the resource monitor + if: ${{ always() }} + continue-on-error: true + shell: bash + run: | + pidfile="$RUNNER_TEMP/resource-monitor.pid" + if [ ! -s "$pidfile" ]; then + echo 'The resource monitor did not start on this leg.' + exit 0 + fi + pid=$(cat "$pidfile") + kill "$pid" 2>/dev/null || echo "The resource monitor, pid $pid, had already stopped." + echo "$(grep -c '^\[monitor\] .* sample [0-9]*$' "$RUNNER_TEMP/resource-monitor.log") samples taken." + + # #1740: the monitor's file, where the runner survives the tests. docs.github.com + # says `always()` runs a step "even when canceled" (Expressions, `always`) + # and nothing about a runner's shutdown, which runs no later step (above): + # job 116057255255 skipped even `Post Run actions/checkout@v7`, whose action + # sets no `post-if`, which "defaults to `always()`" (Metadata syntax, + # `runs.post-if`). So on a lost runner the samples are in the tests' log alone. + - name: Upload the resource monitor's log + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: resource-monitor-${{ matrix.python-version }} + path: ${{ runner.temp }}/resource-monitor.log + if-no-files-found: ignore + retention-days: 14 # As `test`'s, under a name of its own: ci-timing.yml reads `test`'s # `junit-unit-*` files alone, and this one is the record of GraalPy's @@ -221,6 +378,56 @@ jobs: path: ${{ runner.temp }}/junit.xml retention-days: 14 + # #1740: what `tests-status` and the monitor's last sample say about the + # tests, ahead of `Experimental summary`, whose script stays the KTLO + # summary's (tests/project/test_ktlo_legs_stay_green_1672.py) and which + # still reports a failed `tests` step as it reports any. Each cause gets a + # warning of its own: no 12G scope, an OOM kill, a MemoryError, and a + # timeout -- exit 124, or 137 with no OOM kill after 110 minutes, which is + # `--kill-after` ending a teardown that SIGINT did not. An OOM kill of + # pytest itself also exits 137, so the OOM count and the time tell them + # apart. Nothing here fails: a missing or partial status is reported as + # what it is. + - name: Containment summary + if: ${{ !cancelled() }} + continue-on-error: true + shell: bash + env: + LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) + run: | + status="$RUNNER_TEMP/tests-status" + if [ ! -s "$status" ]; then + echo "$LEG: the tests step wrote no tests-status." + exit 0 + fi + value() { sed -n "s/^$1=//p" "$status" | tail -n 1; } + last=$(awk '/^\[monitor\] .* sample [0-9]+$/ { at = $2; n = $4; mem = "" } + /^\[monitor\] Mem:/ { mem = "used " $4 " MiB of " $3 ", " $8 " available" } + END { if (n) print "last sample " n " at " at ": " (mem ? mem : "no free -m line") }' \ + "$RUNNER_TEMP/resource-monitor.log" 2>/dev/null || true) + last=${last:-no resource-monitor sample} + warn() { + echo "::warning title=${1} on ${LEG//,/%2C}::${2//%/%25} Experimental legs never block (#1660), so this job still succeeds." + echo "- ${LEG}: ${1}; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + } + code=$(value exit) scope=$(value scope) elapsed=$(value elapsed) + oom=$(value oom_kills) errors=$(value memory_errors) + if [ "$scope" = none ]; then + warn "No memory cap" "The probe, sudo -n systemd-run --scope with the tests' flags, failed ($(value probe)), so the tests ran without the 12G scope." + fi + if [ "$oom" -gt 0 ] 2>/dev/null; then + warn "tests lost a process to the OOM killer" "${oom} OOM kill(s) while the tests ran, scope ${scope}, exit ${code}, ${elapsed} s; ${last}." + fi + if [ "$code" = 124 ]; then + warn "tests timed out at 110 minutes" "timeout sent SIGINT (exit 124), scope ${scope}, ${elapsed} s; ${last}." + elif [ "$code" = 137 ] && [ "$oom" -eq 0 ] 2>/dev/null && [ "$elapsed" -ge 6600 ] 2>/dev/null; then + warn "tests timed out at 110 minutes" "SIGINT did not end them, so timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope ${scope}, ${elapsed} s; ${last}." + fi + if [ "$errors" -gt 0 ] 2>/dev/null; then + warn "tests raised MemoryError" "${errors} MemoryError line(s) in the pytest output, exit ${code}, scope ${scope}; ${last}." + fi + echo "$LEG: tests exited ${code} after ${elapsed} s, scope ${scope}, ${oom} OOM kill(s), ${errors} MemoryError line(s); ${last}." + # #1660: an experimental leg's report, as unit-tests.yml `test`'s `KTLO # summary` is a KTLO leg's, and for the same reasons. - name: Experimental summary diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 2916000de..60dd6684c 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -757,6 +757,63 @@ and 8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% 3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy 88-113 minutes (estimate). +GraalPy's leg still loses its runner 18-30 minutes into its tests, with ``The runner +has received a shutdown signal`` (`#1740 `__), +so each experimental leg samples its runner while the tests run. ``Start the resource +monitor`` writes ``free -m``, ``df -h`` of ``/`` and the workspace's volume, +``/proc/loadavg`` and the five processes with the most RSS every 30 seconds, each +line led by ``[monitor]``, to ``$RUNNER_TEMP/resource-monitor.log``; ``Run unit +tests`` copies the file into its own log with ``tail -F``; ``Stop the resource +monitor`` ends the sampler, ``if: always()``. The file is uploaded as +``resource-monitor-`` (14 days, ``if: always()``, ``continue-on-error``) +only where the runner survives the tests. `always() +`__ +runs a step "even when canceled", and docs.github.com says nothing of a runner's +shutdown, after which the runner evaluates no later step's condition (actions/runner +``src/Runner.Worker/StepsRunner.cs:205-208``, at ``796cc18ff``). Job 116057255255 +skipped every step after its tests, ``Post Run actions/checkout@v7`` too, whose +`post-if +`__ +"defaults to ``always()``". On a lost runner, then, the ``[monitor]`` lines in the +tests' log are the record. The monitor's three steps, start, stop and upload, are each +``continue-on-error`` with no cap of their own, so the caps above and the check names +are as they were. + +A lost runner fails the job, and so its check, whatever ``continue-on-error`` says, +so ``Run unit tests`` holds the tests to what leaves the runner alive, and a breach +fails the step, which ``Containment summary`` and ``Experimental summary`` report as +warnings: + +* GraalPy alone gets ``GRAAL_PYTHON_VM_ARGS=--vm.Xmx2g``, a 2 GiB heap for every + GraalPy process, xdist workers included, and ``-n 3``, a CPU of the runner's 4 left + to the JIT and the runner; 3.14t keeps ``-n auto``. +* ``timeout --signal=INT --kill-after=3m 110m`` ends pytest under the step's 120-minute + cap, so it still writes the junit XML, and exits 124; ``prlimit --fsize=2147483648`` + caps any file at 2 GiB. +* Where the scope starts ``true`` under a throwaway unit, ``pcapkit-probe-.scope``, + with every flag the tests get (``sudo -n --preserve-env``, ``MemoryMax=12G``, + ``MemorySwapMax=0``, ``CPUWeight=20``, ``--uid``, ``--gid``), the tests run in the + transient scope ``pcapkit-tests.scope`` as the runner's own user; otherwise they run + the same pytest command line uncapped, under a ``No memory cap`` warning. sudo + resets ``PATH`` and drops ``LD_*``, where setup-python puts its ``lib/``, so ``env`` + sets them, and the heap, again. The sampler started a step earlier, outside the scope, so an OOM kill there + leaves it running; each sample adds the kernel's ``oom_kill`` count and the scope's + ``memory.current`` and ``memory.events``. + +The step writes ``$RUNNER_TEMP/tests-status``: the exit status, the scope and the +probe's error, the seconds the tests took, the OOM kills while they ran +(``/proc/vmstat``) and the ``MemoryError`` lines in their output. From it and the +monitor's last sample, ``Containment summary`` warns ``No memory cap``, ``tests lost a +process to the OOM killer``, ``tests timed out at 110 minutes`` or ``tests raised +MemoryError``, each a warning of its own. A timeout is exit 124, or 137 with no OOM kill +after 110 minutes: ``--kill-after``'s SIGKILL to a teardown SIGINT did not end, where an +OOM kill of pytest exits 137 too. ``Experimental summary``, the job's last step and the +KTLO summary's script, still warns of every failed step, ``tests`` included. The 2 GiB, +``-n 3`` and 12G are estimates for a 16 GB, 4-CPU runner until a run shows them. At 3 +workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inference +from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a +warning: that is expected, blocks nothing, and leaves the caps as they are. + Python Versions --------------- diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index a833aa7ee..60677fda9 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -597,7 +597,9 @@ def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: def test_each_step_names_the_leg_by_its_tier(self) -> None: """``LEG`` in an annotation's title: ``(experimental, non-blocking)``.""" named = [step for step in self.spec['steps'] if 'LEG' in (step.get('env') or {})] - self.assertEqual([label(step) for step in named], ['Install package and test dependencies', EXPERIMENTAL_SUMMARY]) + # #1740's `Containment summary` titles the tests' causes by the leg, as the others do. + self.assertEqual([label(step) for step in named], + ['Install package and test dependencies', 'Containment summary', EXPERIMENTAL_SUMMARY]) for leg in legs(*EXPERIMENTAL_JOB): for step in named: with self.subTest(python=leg['python-version'], step=label(step)): diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py new file mode 100644 index 000000000..ae858cf85 --- /dev/null +++ b/tests/project/test_resource_monitor_1740.py @@ -0,0 +1,539 @@ +# -*- coding: utf-8 -*- +"""#1740: the experimental legs sample their runner while the tests run. + +GraalPy's leg loses its runner 18-30 minutes into ``Run unit tests``, under a +120-minute cap, so experimental.yml records what the runner has left: a sampler +started before the tests, stopped ``if: always()`` after them, whose samples +``Run unit tests`` copies into its own log, each line led by ``[monitor]``, and a +file uploaded ``if: always()`` with ``continue-on-error``. A lost runner runs no +later step, ``always()`` ones included, so the log lines are the evidence that +survives it; the upload is for a runner that survives the tests. + +:class:`TestTheStepsAreInPlace` reads the steps. :class:`TestTheSamplesReachTheTestsLog` +runs them, with ``sleep`` and ``python`` stubbed on ``PATH``, so what is checked is +what bash does with them. :class:`TestItGatesNothing` holds it out of every gating +path, as #1732's split requires. + +""" + +from __future__ import annotations + +import os +import pathlib +import re +import shutil +import subprocess # nosec B404 +import sys +import tempfile +import time +import unittest +from typing import TYPE_CHECKING + +from tests.project.test_python_matrix_tiers_1633 import job, legs, workflow, workflow_commands +from tests.project.test_python_versions_1660 import ROOT, prose + +if TYPE_CHECKING: + from typing import Any + +EXPERIMENTAL = ('experimental.yml', 'experimental') +START, TESTS, STOP = 'Start the resource monitor', 'Run unit tests', 'Stop the resource monitor' +UPLOAD = "Upload the resource monitor's log" +#: Where the sampler writes, and the pid it leaves for the stop step. +LOG = '$RUNNER_TEMP/resource-monitor.log' +PIDFILE = '$RUNNER_TEMP/resource-monitor.pid' +PREFIX = '[monitor] ' +#: What each sample holds, as the issue asked for it. +PROBES = ('free -m', 'df -h / "$GITHUB_WORKSPACE"', 'cat /proc/loadavg', + 'ps -eo pid,rss,vsz,etime,args --sort=-rss | head -6') +#: The experimental job's caps and its check name, which the monitor leaves alone. +CAPS = {'python': 5, 'install': 10, 'tests': 120} +CHECK_NAME = 'Python ${{ matrix.python-version }} (experimental)' +#: A sample's first line, ``[monitor] sample ``. +HEADER = re.compile(r'^\[monitor\] \d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ sample (\d+)$') + + +def steps() -> 'list[dict[str, Any]]': + return job(*EXPERIMENTAL)['steps'] + + +def step(name: 'str') -> 'dict[str, Any]': + found = [entry for entry in steps() if (entry.get('name') or entry.get('uses')) == name] + if len(found) != 1: + raise AssertionError(f'expected one {name!r} step in experimental.yml, found {len(found)}') + return found[0] + + +def position(name: 'str') -> 'int': + return steps().index(step(name)) + + +class TestTheStepsAreInPlace(unittest.TestCase): + """Started before the tests, stopped and uploaded ``if: always()`` after them.""" + + def test_the_monitor_starts_before_the_tests_on_the_legs_they_run(self) -> None: + start = step(START) + self.assertEqual(position(START) + 1, position(TESTS)) + self.assertEqual(start.get('id'), 'monitor') + self.assertEqual(start.get('if'), step(TESTS).get('if')) + self.assertIs(start.get('continue-on-error'), True) + self.assertEqual(start.get('shell'), 'bash') + script = start['run'] + for probe in PROBES: + with self.subTest(probe=probe): + self.assertIn(probe, script) + self.assertIn('sleep 30 &', script) + self.assertTrue(script.startswith(f'log="{LOG}"\n: > "$log"\n'), script[:200]) + self.assertIn(f"sed 's/^/{PREFIX}/'", script) + # Detached from the step's output, which the runner would end 5 s after the shell. + self.assertIn(') >> "$log" 2>&1 < /dev/null &', script) + self.assertIn(f'echo "$!" > "{PIDFILE}"', script) + + def test_the_tests_step_streams_the_samples_into_its_own_log(self) -> None: + script = step(TESTS)['run'] + self.assertTrue(script.startswith(f'tail -n +1 -F "{LOG}" 2>/dev/null &\n'), script[:200]) + self.assertIn('trap \'kill "$monitor" 2>/dev/null || true\' EXIT', script) + self.assertIn('python -m pytest "$@"', script) + self.assertTrue(script.rstrip().endswith('exit "$status"')) + + def test_the_monitor_is_stopped_if_always(self) -> None: + stop = step(STOP) + self.assertEqual(position(STOP), position(TESTS) + 1) + self.assertEqual(stop.get('if'), '${{ always() }}') + self.assertIs(stop.get('continue-on-error'), True) + self.assertIn(PIDFILE, stop['run']) + + def test_the_log_is_uploaded_if_always_and_cannot_fail_the_leg(self) -> None: + upload = step(UPLOAD) + self.assertGreater(position(UPLOAD), position(STOP)) + self.assertEqual(upload.get('if'), '${{ always() }}') + self.assertIs(upload.get('continue-on-error'), True) + self.assertTrue(upload['uses'].startswith('actions/upload-artifact@')) + self.assertEqual(upload['with'], { + 'name': 'resource-monitor-${{ matrix.python-version }}', + 'path': '${{ runner.temp }}/resource-monitor.log', + 'if-no-files-found': 'ignore', + 'retention-days': 14, + }) + + +def sh(script: 'str', env: 'dict[str, str]', shell: 'tuple[str, ...]') -> 'subprocess.CompletedProcess[str]': + """``script`` as the runner runs it: ``shell: bash`` or the default ``bash -e``.""" + with tempfile.NamedTemporaryFile('w', suffix='.sh', delete=False, encoding='utf-8') as file: + file.write(script) + try: + return subprocess.run([*shell, file.name], env=env, capture_output=True, text=True, # nosec B603 + check=False, timeout=60) + finally: + os.unlink(file.name) + + +BASH = ('bash', '--noprofile', '--norc', '-eo', 'pipefail') +DEFAULT = ('bash', '-e') + + +@unittest.skipUnless(sys.platform.startswith('linux') and all(map(shutil.which, ('bash', 'free', 'tail', 'ps'))), + 'the sampler reads /proc and procps, as on the Ubuntu runners') +class TestTheSamplesReachTheTestsLog(unittest.TestCase): + """The three scripts, run in order, with ``sleep 30`` made 0.2 s and pytest a stub.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1740-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + (self.tmp / 'temp').mkdir() + bindir = self.tmp / 'bin' + bindir.mkdir() + real_sleep = shutil.which('sleep') + assert real_sleep is not None + (bindir / 'sleep').write_text(f'#!/bin/sh\nexec {real_sleep} 0.2\n', encoding='utf-8') + # pytest leaves its progress line unfinished, as the runner's log shows it does. + (bindir / 'python').write_text(f'#!/bin/sh\nprintf uuu.u\n{real_sleep} 1.5\nprintf "uu [ 1%%]\\n"\nexit 3\n', + encoding='utf-8') + # No real sudo here: the probe fails, as where `sudo -n` cannot start a scope. + (bindir / 'sudo').write_text('#!/bin/sh\necho "sudo: a password is required" >&2\nexit 1\n', encoding='utf-8') + for stub in bindir.iterdir(): + stub.chmod(0o755) + self.env = dict(os.environ, RUNNER_TEMP=str(self.tmp / 'temp'), GITHUB_WORKSPACE=str(ROOT), + PATH=str(bindir) + os.pathsep + os.environ.get('PATH', '')) + + def pid(self) -> 'int': + return int((self.tmp / 'temp' / 'resource-monitor.pid').read_text(encoding='utf-8')) + + def test_samples_stream_into_the_tests_log_and_the_stop_step_ends_the_sampler(self) -> None: + start = sh(step(START)['run'], self.env, BASH) + self.assertEqual(start.returncode, 0, start.stderr) + pid = self.pid() + self.addCleanup(lambda: subprocess.run(['kill', str(pid)], capture_output=True, check=False)) + self.assertTrue(pathlib.Path(f'/proc/{pid}').exists(), 'the sampler did not outlive its step') + + tests = sh(step(TESTS)['run'], self.env, DEFAULT) + # The tests' own status, not the trap's or tail's. + self.assertEqual(tests.returncode, 3, tests.stderr) + lines = tests.stdout.splitlines() + samples = [int(match.group(1)) for match in map(HEADER.match, lines) if match] + self.assertGreaterEqual(len(samples), 2, tests.stdout) + self.assertEqual(samples, list(range(1, len(samples) + 1))) + # Each starts its own line, even after pytest's unfinished one. + self.assertEqual([line for line in lines if PREFIX.strip() in line and not line.startswith(PREFIX)], []) + for needle in ('Mem:', 'Filesystem', '[monitor] loadavg ', 'RSS'): + with self.subTest(holds=needle): + self.assertTrue(any(needle in line for line in lines if line.startswith(PREFIX)), tests.stdout) + + stop = sh(step(STOP)['run'], self.env, BASH) + self.assertEqual(stop.returncode, 0, stop.stderr) + self.assertRegex(stop.stdout, r'^\d+ samples taken\.$') + for _ in range(50): + if not pathlib.Path(f'/proc/{pid}').exists(): + break + time.sleep(0.1) + self.assertFalse(pathlib.Path(f'/proc/{pid}').exists(), 'the stop step left the sampler running') + + def test_the_stop_step_passes_on_a_leg_the_monitor_never_started(self) -> None: + stop = sh(step(STOP)['run'], self.env, BASH) + self.assertEqual(stop.returncode, 0, stop.stderr) + self.assertEqual(stop.stdout, 'The resource monitor did not start on this leg.\n') + + +#: Round 2's containment, as the issue's research sized it (estimates, until a run shows them). +HEAP, GRAAL_WORKERS, SCOPE_MEMORY = '--vm.Xmx2g', '3', '12G' +FSIZE = 2 * 1024 ** 3 +#: Round 3: the probe is the scope itself, every flag included, under a throwaway unit. +PROBE = 'if probe=$(scoped "pcapkit-probe-$$.scope" true 2>&1); then' +SCOPE_FLAGS = ('sudo -n --preserve-env systemd-run --scope --quiet --unit="$unit"', + f'-p MemoryMax={SCOPE_MEMORY}', '-p MemorySwapMax=0', '-p CPUWeight=20', + '--uid="$(id -u)"', '--gid="$(id -g)"', + '-- env PATH="$PATH" HOME="$HOME" LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}" \\\n', + 'GRAAL_PYTHON_VM_ARGS="$GRAAL_PYTHON_VM_ARGS" "$@"') +#: What ``sudo`` receives, past ``-n``, for the probe and for the tests, ``{unit}`` aside. +SUDO_ARGV = ('-n --preserve-env systemd-run --scope --quiet --unit={unit} ' + '-p MemoryMax=12G -p MemorySwapMax=0 -p CPUWeight=20 --uid={uid} --gid={gid} -- env PATH=') +SUMMARY = 'Containment summary' +#: ``${{ startsWith(matrix.python-version, 'graalpy') && 'a' || 'b' }}``, the one expression form the step's env uses. +BY_IMPL = re.compile(r"^\$\{\{ startsWith\(matrix\.python-version, 'graalpy'\) && '([^']*)' \|\| '([^']*)' \}\}$") + + +def by_impl(expression: 'str', version: 'str') -> 'str': + """``expression`` as GitHub evaluates it on the leg of ``version``.""" + match = BY_IMPL.match(expression) + if match is None: + raise AssertionError(f'not keyed on the GraalPy leg: {expression!r}') + # `a && 'x' || 'y'` is 'y' where 'x' is empty, as in GitHub's expressions. + return match.group(1) if version.lower().startswith('graalpy') and match.group(1) else match.group(2) + + +class TestTheTestsAreContained(unittest.TestCase): + """Round 2: the tests are held to what leaves the runner alive, so a breach fails the step.""" + + def legs(self) -> 'list[str]': + found = [leg['python-version'] for leg in legs(*EXPERIMENTAL)] + self.assertTrue(any(version.startswith('graalpy') for version in found), found) + self.assertTrue(any(not version.startswith('graalpy') for version in found), found) + return found + + def test_graalpy_alone_gets_a_2g_heap_and_3_workers(self) -> None: + env = step(TESTS).get('env', {}) + for version in self.legs(): + graalpy = version.startswith('graalpy') + with self.subTest(version=version): + self.assertEqual(by_impl(env['GRAAL_PYTHON_VM_ARGS'], version), HEAP if graalpy else '') + self.assertEqual(by_impl(env['WORKERS'], version), GRAAL_WORKERS if graalpy else 'auto') + script = step(TESTS)['run'] + self.assertIn('set -- -q -n "$WORKERS" --dist load', script) + self.assertNotIn('-n auto', script) + + def test_the_timeout_ends_the_tests_before_the_step_cap(self) -> None: + script = step(TESTS)['run'] + match = re.search(r'timeout --signal=INT --kill-after=(\d+)m (\d+)m python -m pytest "\$@"', script) + self.assertIsNotNone(match, script) + assert match is not None + kill_after, duration = map(int, match.groups()) + self.assertEqual((duration, kill_after), (110, 3)) + self.assertLess(duration + kill_after, step(TESTS)['timeout-minutes']) + + def test_no_file_grows_past_2_gib(self) -> None: + self.assertIn(f'set -- prlimit --fsize={FSIZE} timeout ', step(TESTS)['run']) + + def test_the_scope_is_probed_and_falls_back_with_a_warning(self) -> None: + script = step(TESTS)['run'] + self.assertIn(PROBE, script) + # One definition of the scope, so the probe cannot drift from the tests' flags. + self.assertEqual(script.count('systemd-run'), 1) + self.assertEqual(script.count('sudo '), 1) + for flag in SCOPE_FLAGS: + with self.subTest(flag=flag): + self.assertIn(flag, script) + self.assertIn('set -- scoped pcapkit-tests.scope "$@"', script) + fallback = script[script.index('else\n', script.index(PROBE)):] + self.assertIn('scope=none', fallback) + # The tests step writes no annotation and needs no `LEG`; `Containment summary` warns. + self.assertNotIn('::', script) + self.assertNotIn('LEG', step(TESTS).get('env', {})) + self.assertIn('title=${1} on ${LEG//,/%2C}', step(SUMMARY)['run']) + self.assertIn('warn "No memory cap"', step(SUMMARY)['run']) + # The probe, the scope and the fallback all come before the one run of the tests. + self.assertLess(script.index('scope=none'), script.index('"$@" 2>&1 | tee "$RUNNER_TEMP/pytest.log"')) + + def test_the_sampler_runs_outside_the_scope(self) -> None: + start, tests = step(START)['run'], step(TESTS)['run'] + for word in ('systemd-run', 'sudo', 'prlimit', 'timeout '): + with self.subTest(word=word): + self.assertNotIn(word, start) + self.assertLess(tests.index('tail -n +1 -F'), tests.index('systemd-run')) + # The scope runs the tests alone: its command line, after `--`, holds no `tail`. + self.assertNotIn('tail', tests[tests.index('-- env PATH='):].splitlines()[0]) + self.assertIn('/sys/fs/cgroup/system.slice/pcapkit-tests.scope', start) + self.assertIn('oom_kill', start) + + +@unittest.skipUnless(sys.platform.startswith('linux') and all(map(shutil.which, ('bash', 'timeout', 'prlimit', 'jq'))), + 'the tests step runs coreutils timeout and util-linux prlimit, as on the Ubuntu runners') +class TestTheContainmentRuns(unittest.TestCase): + """The tests step and ``Containment summary``, run with ``sudo``, ``python`` and /proc/vmstat stubbed.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1740-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.temp = self.tmp / 'temp' + self.temp.mkdir() + self.bindir = self.tmp / 'bin' + self.bindir.mkdir() + (self.temp / 'resource-monitor.log').write_text('', encoding='utf-8') + self.vmstat = self.tmp / 'vmstat' + self.vmstat.write_text('oom_kill 7\n', encoding='utf-8') + self.argv = self.tmp / 'argv' + self.env = dict(os.environ, RUNNER_TEMP=str(self.temp), GITHUB_WORKSPACE=str(ROOT), WORKERS='3', + GRAAL_PYTHON_VM_ARGS=HEAP, + LEG='Python graalpy-25.4 (experimental, non-blocking)', + GITHUB_STEP_SUMMARY=str(self.tmp / 'summary.md'), + PATH=str(self.bindir) + os.pathsep + os.environ.get('PATH', '')) + + def stub(self, name: 'str', body: 'str') -> 'None': + path = self.bindir / name + path.write_text('#!/bin/sh\n' + body, encoding='utf-8') + path.chmod(0o755) + + def scope(self) -> 'None': + """A ``sudo`` whose probe passes and whose scope runs what follows ``--``, recording each call.""" + self.stub('sudo', f'printf "%s\\n" "$*" >> {self.argv}\n' + 'while [ "$#" -gt 0 ] && [ "$1" != -- ]; do shift; done\n' + '[ "$#" -gt 0 ] || exit 0\nshift\nexec "$@"\n') + + def run_step(self, script: 'str | None' = None) -> 'subprocess.CompletedProcess[str]': + script = step(TESTS)['run'] if script is None else script + return sh(script.replace('/proc/vmstat', str(self.vmstat)), self.env, DEFAULT) + + def status(self) -> 'dict[str, str]': + text = (self.temp / 'tests-status').read_text(encoding='utf-8') + return dict(line.split('=', 1) for line in text.splitlines()) + + def summary(self, status: 'str') -> 'list[str]': + """``Containment summary``'s workflow commands, given ``tests-status``.""" + (self.temp / 'tests-status').write_text(status, encoding='utf-8') + result = sh(step(SUMMARY)['run'], self.env, BASH) + self.assertEqual(result.returncode, 0, result.stderr) + return workflow_commands(result.stdout) + + def no_real_sudo(self) -> 'None': + """Every run here resolves ``sudo`` to a stub in ``self.bindir``, never the real one.""" + found = shutil.which('sudo', path=self.env['PATH']) + self.assertEqual(found, str(self.bindir / 'sudo')) + + def test_the_scope_runs_the_tests_under_the_timeout_and_fsize_cap(self) -> None: + self.scope() + self.stub('python', 'echo "args: $*"\necho "heap $GRAAL_PYTHON_VM_ARGS"\n' + 'grep "^Max file size" /proc/$$/limits\nexit 1\n') + self.no_real_sudo() + result = self.run_step() + self.assertEqual(result.returncode, 1, result.stderr) + probe, run = self.argv.read_text(encoding='utf-8').splitlines() + # The probe passes sudo every flag the tests' scope does, under a throwaway unit. + unit = re.match(r'-n --preserve-env systemd-run --scope --quiet --unit=(pcapkit-probe-\d+\.scope) ', probe) + self.assertIsNotNone(unit, probe) + assert unit is not None + ids = {'uid': os.getuid(), 'gid': os.getgid()} + self.assertTrue(probe.startswith(SUDO_ARGV.format(unit=unit.group(1), **ids)), probe) + self.assertTrue(probe.endswith(f' GRAAL_PYTHON_VM_ARGS={HEAP} true'), probe) + self.assertTrue(run.startswith(SUDO_ARGV.format(unit='pcapkit-tests.scope', **ids)), run) + self.assertIn(f' prlimit --fsize={FSIZE} timeout --signal=INT --kill-after=3m 110m python -m pytest -q -n 3 ', + run) + self.assertIn('args: -m pytest -q -n 3 --dist load', result.stdout) + self.assertIn(f'heap {HEAP}\n', result.stdout) + self.assertIn(f' GRAAL_PYTHON_VM_ARGS={HEAP} prlimit ', run) + self.assertRegex(result.stdout, rf'Max file size +{FSIZE} +{FSIZE} +bytes') + self.assertEqual(workflow_commands(result.stdout), []) + status = self.status() + self.assertRegex(status.pop('elapsed'), r'^\d+$') + self.assertEqual(status, {'exit': '1', 'scope': '12G', 'probe': '', 'oom_kills': '0', 'memory_errors': '0'}) + + def test_without_a_scope_the_tests_run_uncapped_with_a_warning(self) -> None: + self.stub('sudo', 'echo "sudo: a password is required" >&2\nexit 1\n') + self.stub('python', 'exit 0\n') + self.no_real_sudo() + result = self.run_step() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(workflow_commands(result.stdout), []) + self.assertIn('No 12G scope: the probe failed (sudo: a password is required), so the tests run uncapped.\n', + result.stdout) + self.assertEqual(self.status()['scope'], 'none') + self.assertEqual(self.status()['probe'], 'sudo: a password is required') + self.assertEqual(self.summary((self.temp / 'tests-status').read_text(encoding='utf-8')), [ + '::warning title=No memory cap on Python graalpy-25.4 (experimental%2C non-blocking)::The probe, sudo -n ' + "systemd-run --scope with the tests' flags, failed (sudo: a password is required), so the tests ran " + 'without the 12G scope. Experimental legs never block (#1660), so this job still succeeds.']) + + def test_the_uncapped_tests_run_the_capped_argv(self) -> None: + """Round 3: the fallback drops the scope and nothing else, so the two paths run one pytest.""" + argvs = {} + for path, sudo in (('capped', None), ('uncapped', 'echo "sudo: a password is required" >&2\nexit 1\n')): + if sudo is None: + self.scope() + else: + self.stub('sudo', sudo) + self.stub('python', f'printf "%s\\n" "$@" > {self.tmp / path}\nexit 0\n') + self.no_real_sudo() + result = self.run_step() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.status()['scope'], '12G' if sudo is None else 'none') + argvs[path] = (self.tmp / path).read_text(encoding='utf-8').splitlines() + self.assertEqual(argvs['capped'], argvs['uncapped']) + argv = argvs['capped'] + self.assertEqual(argv[:6], ['-m', 'pytest', '-q', '-n', '3', '--dist']) + for arg in (f'--junitxml={self.temp}/junit.xml', '--ignore=tests/integration', '--ignore-glob=*_runtime.py', + '--ignore-glob=*_regression.py', '--ignore-glob=*roundtrip*', '--durations=30'): + with self.subTest(arg=arg): + self.assertIn(arg, argv) + # Only the capped path took the tests through sudo: the probe and the run. + self.assertEqual(len(self.argv.read_text(encoding='utf-8').splitlines()), 2) + + def test_a_timeout_exits_124(self) -> None: + self.scope() + self.stub('python', 'exec sleep 30\n') + script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=3s 1s') + self.assertNotEqual(script, step(TESTS)['run']) + result = self.run_step(script) + self.assertEqual(result.returncode, 124, result.stderr) + self.assertEqual(self.status()['exit'], '124') + + def test_oom_kills_and_memory_errors_are_counted(self) -> None: + self.scope() + self.stub('python', f'echo "oom_kill 9" > {self.vmstat}\necho "E MemoryError"\nexit 1\n') + result = self.run_step() + self.assertEqual(result.returncode, 1, result.stderr) + status = self.status() + self.assertEqual({key: status[key] for key in ('exit', 'scope', 'oom_kills', 'memory_errors')}, + {'exit': '1', 'scope': '12G', 'oom_kills': '2', 'memory_errors': '1'}) + + def titles(self, status: 'str') -> 'list[str]': + return [re.sub(r'^::warning title=(.*?) on Python .*$', r'\1', line) for line in self.summary(status)] + + def test_the_summary_reports_each_cause_distinctly(self) -> None: + (self.temp / 'resource-monitor.log').write_text( + '\n[monitor] 2026-10-14T20:00:00Z sample 1\n[monitor] Mem: 15990 9000 1000 10 6000 6990\n' + '\n[monitor] 2026-10-14T20:00:30Z sample 2\n[monitor] Mem: 15990 15000 100 10 890 990\n', + encoding='utf-8') + base = 'scope=12G\n' + self.assertEqual(self.titles(base + 'exit=124\noom_kills=0\nmemory_errors=0\n'), + ['tests timed out at 110 minutes']) + self.assertEqual(self.titles(base + 'exit=137\noom_kills=1\nmemory_errors=0\n'), + ['tests lost a process to the OOM killer']) + self.assertEqual(self.titles(base + 'exit=1\noom_kills=0\nmemory_errors=3\n'), ['tests raised MemoryError']) + # A failure with no cause of its own is `Experimental summary`'s to report, as any step's. + self.assertEqual(self.titles(base + 'exit=1\noom_kills=0\nmemory_errors=0\n'), []) + oom = self.summary(base + 'exit=137\noom_kills=1\nmemory_errors=0\n')[0] + self.assertIn('last sample 2 at 2026-10-14T20:00:30Z: used 15000 MiB of 15990, 990 available', oom) + self.assertIn('scope 12G', oom) + + def test_a_hung_teardown_killed_after_the_timeout_is_a_timeout(self) -> None: + """Round 3: ``--kill-after`` ends a teardown SIGINT did not with SIGKILL, exit 137, as an OOM kill does.""" + base = 'scope=12G\nmemory_errors=0\n' + timeout = ['tests timed out at 110 minutes'] + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=6781\n'), timeout) + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=6600\n'), timeout) + # Before 110 minutes, or with an OOM kill, 137 is not the timeout's. + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\nelapsed=6599\n'), []) + self.assertEqual(self.titles(base + 'exit=137\noom_kills=1\nelapsed=6781\n'), + ['tests lost a process to the OOM killer']) + self.assertEqual(self.titles(base + 'exit=137\noom_kills=0\n'), []) + killed = self.summary(base + 'exit=137\noom_kills=0\nelapsed=6781\n')[0] + self.assertIn('timeout sent SIGKILL 3 minutes on (exit 137, no OOM kill), scope 12G, 6781 s', killed) + + def test_a_tests_step_killed_after_the_timeout_records_137_and_its_time(self) -> None: + self.scope() + self.stub('python', "trap '' INT\nsleep 5\n") + script = step(TESTS)['run'].replace('--kill-after=3m 110m', '--kill-after=1s 1s') + self.assertNotEqual(script, step(TESTS)['run']) + self.no_real_sudo() + result = self.run_step(script) + self.assertEqual(result.returncode, 137, result.stderr) + self.assertEqual(self.status()['exit'], '137') + self.assertIn(self.status()['elapsed'], {'2', '3'}) + + def test_the_summary_is_quiet_without_a_status(self) -> None: + result = sh(step(SUMMARY)['run'], self.env, BASH) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(workflow_commands(result.stdout), []) + self.assertIn('the tests step wrote no tests-status', result.stdout) + + def test_experimental_summary_is_mains_and_runs_last(self) -> None: + """Round 3: the causes are a step of their own, so ``Experimental summary`` stays the KTLO summary's.""" + self.assertEqual(position(SUMMARY) + 1, position('Experimental summary')) + self.assertEqual(step(SUMMARY).get('if'), '${{ !cancelled() }}') + self.assertIs(step(SUMMARY).get('continue-on-error'), True) + self.assertEqual(step(SUMMARY).get('env'), + {'LEG': 'Python ${{ matrix.python-version }} (experimental, non-blocking)'}) + self.assertNotIn('tests-status', step('Experimental summary')['run']) + + +class TestItGatesNothing(unittest.TestCase): + """In experimental.yml alone, inside its caps and check names, and read by nothing.""" + + def test_only_the_experimental_job_runs_it(self) -> None: + for path in sorted((ROOT / '.github' / 'workflows').glob('*.yml')): + if path.name == EXPERIMENTAL[0]: + continue + with self.subTest(workflow=path.name): + self.assertNotIn('resource-monitor', path.read_text(encoding='utf-8')) + self.assertIs(job(*EXPERIMENTAL).get('continue-on-error'), True) + self.assertEqual(steps()[-1]['name'], 'Experimental summary') + + def test_the_caps_and_check_name_are_unchanged(self) -> None: + spec = job(*EXPERIMENTAL) + self.assertEqual(spec['name'], CHECK_NAME) + self.assertEqual(spec['timeout-minutes'], 140) + self.assertEqual({entry.get('id'): entry['timeout-minutes'] for entry in spec['steps'] + if 'timeout-minutes' in entry}, CAPS) + + def test_no_reader_takes_the_artifact_for_a_junit_one(self) -> None: + """CI Timing reads ``junit-*`` artifacts by name; nothing else reads this one.""" + from util import ci_timing # pylint: disable=import-outside-toplevel + name = step(UPLOAD)['with']['name'] + self.assertFalse(name.startswith('junit-')) + self.assertTrue(ci_timing.JUNIT_ARTIFACT.startswith('junit-')) + self.assertEqual([path.name for path in sorted((ROOT / 'util').glob('*.py')) + if 'resource-monitor' in path.read_text(encoding='utf-8')], []) + self.assertEqual(list(workflow(EXPERIMENTAL[0])['jobs']), ['python-versions', 'changes', 'experimental']) + + +class TestTheDocsSayIt(unittest.TestCase): + + def test_the_workflows_page_describes_the_monitor(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + for phrase in ('``Start the resource monitor``', '``[monitor]``', '``resource-monitor-``', + '``Stop the resource monitor``', 'src/Runner.Worker/StepsRunner.cs:205-208'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, page) + + def test_the_workflows_page_describes_the_containment(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + for phrase in (f'``GRAAL_PYTHON_VM_ARGS={HEAP}``', '``-n 3``', '``timeout --signal=INT --kill-after=3m 110m``', + f'``prlimit --fsize={FSIZE}``', '``Containment summary``', '``pcapkit-probe-.scope``', + '117-151 minutes', '137', f'``MemoryMax={SCOPE_MEMORY}``', + '``No memory cap``', '``$RUNNER_TEMP/tests-status``', '``tests timed out at 110 minutes``', + '``tests lost a process to the OOM killer``', '``tests raised MemoryError``'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, page) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index c77933120..0508a0adb 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -109,15 +109,23 @@ def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'list[str stub = pathlib.Path(temp, 'python') stub.write_text(STUB, encoding='utf-8') stub.chmod(0o755) + # #1740: experimental's tests step probes `sudo -n systemd-run`, which fails here, as it + # would where no scope can start, so the tests run uncapped: no scope on this machine. + sudo = pathlib.Path(temp, 'sudo') + sudo.write_text('#!/bin/sh\nexit 1\n', encoding='utf-8') + sudo.chmod(0o755) + # A directory for the files that step writes, named RUNNER_TEMP in what is returned. + runner_temp = pathlib.Path(temp, '_temp') + runner_temp.mkdir() record = pathlib.Path(temp, 'args') - env = dict(os.environ, **step_env(leg, where), ARGS_FILE=str(record), RUNNER_TEMP=RUNNER_TEMP, + env = dict(os.environ, **step_env(leg, where), ARGS_FILE=str(record), RUNNER_TEMP=str(runner_temp), REAL_PYTHON=sys.executable, PATH=temp + os.pathsep + os.environ.get('PATH', '')) # From the root, as the runner runs it: ``integration`` imports tests/_tiers.py. result = subprocess.run(['bash', '-e', '-c', step(where)['run']], env=env, cwd=ROOT, # nosec B603 B607 capture_output=True, text=True, check=False, timeout=60) if result.returncode: raise AssertionError(f'the step failed on {leg}: {result.stderr}') - return record.read_text(encoding='utf-8').splitlines() + return [arg.replace(str(runner_temp), RUNNER_TEMP) for arg in record.read_text(encoding='utf-8').splitlines()] def pytest_args(argv: 'list[str]') -> 'list[str]': @@ -160,10 +168,14 @@ def test_every_other_leg_runs_the_rest(self) -> None: for version, (leg, argv) in self.runs.items(): if version == self.latest: continue + want = list(expected) + if is_experimental(leg): + # #1740: the experimental step's own worker count, 3 on GraalPy and `auto` on the rest. + want[want.index('-n') + 1] = step_env(leg, EXPERIMENTAL)['WORKERS'] with self.subTest(python=version): self.assertNotIn('coverage', argv) self.assertEqual(argv[:2], ['-m', 'pytest']) - self.assertEqual(pytest_args(argv), expected) + self.assertEqual(pytest_args(argv), want) class TestTheGlobIsTheSweeps(unittest.TestCase):