diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml index 054de102..590285a0 100644 --- a/.github/workflows/test-jvm.yml +++ b/.github/workflows/test-jvm.yml @@ -78,16 +78,32 @@ on: # # A force-push to a PR branch left the OLD run going alongside the new one: the # stale run reports a failure that the newer commit already fixed, and it holds -# macOS/Windows runners the organisation shares with ArcadeData/arcadedb. +# macOS/Windows runners the organisation shares with ArcadeData/arcadedb. A push +# run of main used to get a group of its own (run_id), so eight merges in a day +# queued 21 runs and about 70 macOS jobs behind each other (#271). # -# cancel-in-progress is scoped to pull_request ONLY. Every other event gets a -# group of its own (run_id), so a push to main or a nightly run is never -# cancelled and never replaced while queued behind another run. The group names -# the workflow literally: github.workflow is the CALLER's name in a -# workflow_call, which would put this workflow and its sibling in one group. +# Per event: +# * pull_request: one group per PR, a newer push cancels the older run. +# * push to main: one group for the workflow, cancel-in-progress false. GitHub +# keeps at most one running and one pending run per group and lets a newer +# pending run replace the older pending one, so the queue never grows beyond +# two and the pending run is always the newest commit. Each merge was already +# gated green on its pull request. +# * schedule (nightly): its own group, a stuck old nightly is cancelled. +# * workflow_dispatch and workflow_call (the release): a group of their own per +# run, never cancelled. In a workflow_call github.event_name is the CALLER's +# (a tag push), which is why the push case also checks that the ref is main: +# a release run must never share a group with, or be replaced by, a push to main. +# +# The group names the workflow literally: github.workflow is the CALLER's name in +# a workflow_call, which would put this workflow and its sibling in one group. concurrency: - group: test-jvm-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: >- + ${{ github.event_name == 'pull_request' && format('test-jvm-pr-{0}', github.event.pull_request.number) + || github.event_name == 'push' && github.ref == 'refs/heads/main' && 'test-jvm-main' + || github.event_name == 'schedule' && 'test-jvm-schedule' + || format('test-jvm-run-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }} env: # The image download-jars takes the jars from. Unset (push, pull_request, schedule) diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index e5a8671a..aa3f14ec 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -89,10 +89,18 @@ on: # The group names the workflow literally: github.workflow is the CALLER's name in a workflow_call, and the group of release-native.yml # must differ from this one or the two would wait for each other. +# +# Per event: a pull request has one group per PR; a push to main and the nightly schedule each have their own group (a newer run +# supersedes an older one); a workflow_dispatch has a group of its own per run, so a manual run and a push can never cancel each other +# (they used to share the ref of main); a run for a tag (the release calls this workflow, and github.event_name is then the caller's +# push) is never cancelled, a half-done release is worse than a queued one. concurrency: - group: test-native-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} - # A dry run is superseded by a newer one; a run for a tag is never cancelled (a half-done release is worse than a queued one). - cancel-in-progress: ${{ github.ref_type != 'tag' }} + group: >- + ${{ github.event_name == 'pull_request' && format('test-native-{0}', github.event.pull_request.number) + || github.event_name == 'workflow_dispatch' && format('test-native-dispatch-{0}', github.run_id) + || github.event_name == 'schedule' && 'test-native-schedule' + || format('test-native-{0}', github.ref) }} + cancel-in-progress: ${{ github.ref_type != 'tag' && github.event_name != 'workflow_dispatch' }} permissions: contents: read diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index eb086997..49520838 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -80,22 +80,34 @@ on: # SUPERSEDE, do not race (same rule in test-jvm.yml). # -# A force-push to a PR branch left the OLD run going alongside the new one: on -# #6889 both 118b347ce and 75293c7a2 ran at once, and the stale one reported a -# failure that the newer commit had already fixed. Anyone reading the PR sees a -# red check against work that is done. The stale run also holds macOS/Windows -# runners the organisation shares with ArcadeData/arcadedb. +# A force-push to a PR branch left the OLD run going alongside the new one: the +# stale run reports a failure that the newer commit already fixed, and it holds +# macOS/Windows runners the organisation shares with ArcadeData/arcadedb. A push +# run of main used to get a group of its own (run_id), so eight merges in a day +# queued 21 runs and about 70 macOS jobs behind each other (#271). # -# cancel-in-progress is scoped to pull_request ONLY. On a push to a long-lived -# branch a superseded run is often the one you still want -- it is the record -# for that commit -- so those are left to finish, and they get a group of their -# own (run_id) so a queued push or nightly run is not replaced by a newer one -# either. The group names the workflow literally: github.workflow is the -# CALLER's name in a workflow_call, which would put this workflow and its -# sibling in one group. +# Per event: +# * pull_request: one group per PR, a newer push cancels the older run. +# * push to main: one group for the workflow, cancel-in-progress false. GitHub +# keeps at most one running and one pending run per group and lets a newer +# pending run replace the older pending one, so the queue never grows beyond +# two and the pending run is always the newest commit. Each merge was already +# gated green on its pull request. +# * schedule (nightly): its own group, a stuck old nightly is cancelled. +# * workflow_dispatch and workflow_call (the release): a group of their own per +# run, never cancelled. In a workflow_call github.event_name is the CALLER's +# (a tag push), which is why the push case also checks that the ref is main: +# a release run must never share a group with, or be replaced by, a push to main. +# +# The group names the workflow literally: github.workflow is the CALLER's name in +# a workflow_call, which would put this workflow and its sibling in one group. concurrency: - group: test-python-examples-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: >- + ${{ github.event_name == 'pull_request' && format('test-python-examples-pr-{0}', github.event.pull_request.number) + || github.event_name == 'push' && github.ref == 'refs/heads/main' && 'test-python-examples-main' + || github.event_name == 'schedule' && 'test-python-examples-schedule' + || format('test-python-examples-run-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }} env: # The image download-jars takes the jars from. Unset (push, pull_request, schedule) diff --git a/tests/test_workflow_concurrency.py b/tests/test_workflow_concurrency.py new file mode 100644 index 00000000..1822f2f3 --- /dev/null +++ b/tests/test_workflow_concurrency.py @@ -0,0 +1,117 @@ +"""Which runs of the test workflows cancel each other: superseded runs of a pull request, never a release or a dispatch.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = { + name: ROOT / ".github" / "workflows" / f"{name}.yml" + for name in ("test-jvm", "test-python-examples", "test-native") +} + + +def _concurrency(text: str) -> tuple[str, str]: + """The group and cancel-in-progress expressions of the top-level concurrency block, `${{ }}` stripped.""" + block = text.split("\nconcurrency:\n", 1)[1].split("\n\n", 1)[0] + group_text, cancel_text = block.split(" cancel-in-progress:", 1) + group = group_text.split("group:", 1)[1].replace(">-", "") + group = " ".join(line.strip() for line in group.strip().splitlines()) + cancel = cancel_text.strip() + for value in (group, cancel): + assert value.startswith("${{") and value.endswith("}}"), value + return group[3:-2].strip(), cancel[3:-2].strip() + + +def _evaluate(expression: str, context: dict[str, str]): + """GitHub's expression language restricted to what the blocks use: && || == != strings, format() and github.* names. + + `&&` and `||` return an operand and bind like Python's `and` and `or`, so the translation keeps the meaning. + """ + + def fmt(template: str, *values: str) -> str: + return template.format(*values) + + translated = re.sub( + r"github\.([a-z_.]+)", lambda m: repr(context.get(m.group(1), "")), expression + ) + translated = translated.replace("&&", " and ").replace("||", " or ") + # only literals, operators and format() are left; nothing else may run + assert re.fullmatch(r"[\s\w'\"(),.=!/{}\-]*", translated), translated + return eval(translated, {"__builtins__": {}, "format": fmt}) # nosec B307 + + +def _run(workflow: str, event: str, ref: str, run_id: str = "1", pr: str = ""): + group, cancel = _concurrency(WORKFLOWS[workflow].read_text(encoding="utf-8")) + context = { + "event_name": event, + "ref": ref, + "ref_type": "tag" if ref.startswith("refs/tags/") else "branch", + "run_id": run_id, + "event.pull_request.number": pr, + } + return _evaluate(group, context), bool(_evaluate(cancel, context)) + + +@pytest.mark.parametrize("workflow", sorted(WORKFLOWS)) +def test_a_new_push_to_a_pull_request_cancels_the_old_run(workflow): + old = _run(workflow, "pull_request", "refs/pull/7/merge", "1", "7") + new = _run(workflow, "pull_request", "refs/pull/7/merge", "2", "7") + other = _run(workflow, "pull_request", "refs/pull/8/merge", "3", "8") + assert old == new and new[1] is True + assert other[0] != new[0] + + +@pytest.mark.parametrize("workflow", sorted(WORKFLOWS)) +def test_pushes_to_main_share_one_group_so_no_backlog_builds(workflow): + first = _run(workflow, "push", "refs/heads/main", "1") + second = _run(workflow, "push", "refs/heads/main", "2") + assert first[0] == second[0] + if workflow == "test-native": + # the native workflow keeps its older rule: a newer push to main cancels the older run + assert first[1] is True + else: + # JVM and examples: one running and one pending run; a running run is never cancelled + assert first[1] is False + + +@pytest.mark.parametrize("workflow", sorted(WORKFLOWS)) +def test_the_nightly_run_has_its_own_group_and_replaces_a_stuck_one(workflow): + nightly = _run(workflow, "schedule", "refs/heads/main", "1") + push = _run(workflow, "push", "refs/heads/main", "2") + assert nightly[0] != push[0] + assert nightly[1] is True + assert nightly == _run(workflow, "schedule", "refs/heads/main", "3") + + +@pytest.mark.parametrize("workflow", sorted(WORKFLOWS)) +def test_a_dispatch_has_a_group_of_its_own_and_is_never_cancelled(workflow): + dispatch = _run(workflow, "workflow_dispatch", "refs/heads/main", "10") + push = _run(workflow, "push", "refs/heads/main", "11") + again = _run(workflow, "workflow_dispatch", "refs/heads/main", "12") + assert dispatch[1] is False + assert len({dispatch[0], push[0], again[0]}) == 3 + + +@pytest.mark.parametrize("workflow", sorted(WORKFLOWS)) +def test_a_release_run_is_never_cancelled_and_never_shares_the_group_of_main(workflow): + # a workflow_call from release-*.yml: github.event_name is the caller's, a push, and the ref is the tag + release = _run(workflow, "push", "refs/tags/26.11.1", "20") + main = _run(workflow, "push", "refs/heads/main", "21") + assert release[1] is False + assert release[0] != main[0] + # a newer push to main cannot replace it + assert _run(workflow, "push", "refs/heads/main", "22")[0] != release[0] + + +def test_the_release_workflows_never_cancel_a_running_release(): + for name in ("release-jvm", "release-native"): + text = (ROOT / ".github" / "workflows" / f"{name}.yml").read_text( + encoding="utf-8" + ) + block = text.split("\nconcurrency:\n", 1)[1].split("\n\n", 1)[0] + assert "cancel-in-progress: false" in block + assert "ref_name" in block