CI: cancel superseded runs, keep one pending run on main, never touch releases

.github/workflows/test-jvm.yml

@@ -78,16 +78,32 @@ on:
 #
 # A force-push to a PR branch left the OLD run going alongside the new one: the
 # stale run reports a failure that the newer commit already fixed, and it holds
-# macOS/Windows runners the organisation shares with ArcadeData/arcadedb.
+# macOS/Windows runners the organisation shares with ArcadeData/arcadedb. A push
+# run of main used to get a group of its own (run_id), so eight merges in a day
+# queued 21 runs and about 70 macOS jobs behind each other (#271).
 #
-# cancel-in-progress is scoped to pull_request ONLY. Every other event gets a
-# group of its own (run_id), so a push to main or a nightly run is never
-# cancelled and never replaced while queued behind another run. The group names
-# the workflow literally: github.workflow is the CALLER's name in a
-# workflow_call, which would put this workflow and its sibling in one group.
+# Per event:
+# * pull_request: one group per PR, a newer push cancels the older run.
+# * push to main: one group for the workflow, cancel-in-progress false. GitHub
+#   keeps at most one running and one pending run per group and lets a newer
+#   pending run replace the older pending one, so the queue never grows beyond
+#   two and the pending run is always the newest commit. Each merge was already
+#   gated green on its pull request.
+# * schedule (nightly): its own group, a stuck old nightly is cancelled.
+# * workflow_dispatch and workflow_call (the release): a group of their own per
+#   run, never cancelled. In a workflow_call github.event_name is the CALLER's
+#   (a tag push), which is why the push case also checks that the ref is main:
+#   a release run must never share a group with, or be replaced by, a push to main.
+#
+# The group names the workflow literally: github.workflow is the CALLER's name in
+# a workflow_call, which would put this workflow and its sibling in one group.
 concurrency:
-  group: test-jvm-${{ github.event.pull_request.number || github.run_id }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: >-
+    ${{ github.event_name == 'pull_request' && format('test-jvm-pr-{0}', github.event.pull_request.number)
+    || github.event_name == 'push' && github.ref == 'refs/heads/main' && 'test-jvm-main'
+    || github.event_name == 'schedule' && 'test-jvm-schedule'
+    || format('test-jvm-run-{0}', github.run_id) }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }}
 
 env:
   # The image download-jars takes the jars from. Unset (push, pull_request, schedule)

.github/workflows/test-native.yml

@@ -89,10 +89,18 @@ on:
 
 # The group names the workflow literally: github.workflow is the CALLER's name in a workflow_call, and the group of release-native.yml
 # must differ from this one or the two would wait for each other.
+#
+# Per event: a pull request has one group per PR; a push to main and the nightly schedule each have their own group (a newer run
+# supersedes an older one); a workflow_dispatch has a group of its own per run, so a manual run and a push can never cancel each other
+# (they used to share the ref of main); a run for a tag (the release calls this workflow, and github.event_name is then the caller's
+# push) is never cancelled, a half-done release is worse than a queued one.
 concurrency:
-  group: test-native-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
-  # A dry run is superseded by a newer one; a run for a tag is never cancelled (a half-done release is worse than a queued one).
-  cancel-in-progress: ${{ github.ref_type != 'tag' }}
+  group: >-
+    ${{ github.event_name == 'pull_request' && format('test-native-{0}', github.event.pull_request.number)
+    || github.event_name == 'workflow_dispatch' && format('test-native-dispatch-{0}', github.run_id)
+    || github.event_name == 'schedule' && 'test-native-schedule'
+    || format('test-native-{0}', github.ref) }}
+  cancel-in-progress: ${{ github.ref_type != 'tag' && github.event_name != 'workflow_dispatch' }}
 
 permissions:
   contents: read

.github/workflows/test-python-examples.yml

@@ -80,22 +80,34 @@ on:
 
 # SUPERSEDE, do not race (same rule in test-jvm.yml).
 #
-# A force-push to a PR branch left the OLD run going alongside the new one: on
-# #6889 both 118b347ce and 75293c7a2 ran at once, and the stale one reported a
-# failure that the newer commit had already fixed. Anyone reading the PR sees a
-# red check against work that is done. The stale run also holds macOS/Windows
-# runners the organisation shares with ArcadeData/arcadedb.
+# A force-push to a PR branch left the OLD run going alongside the new one: the
+# stale run reports a failure that the newer commit already fixed, and it holds
+# macOS/Windows runners the organisation shares with ArcadeData/arcadedb. A push
+# run of main used to get a group of its own (run_id), so eight merges in a day
+# queued 21 runs and about 70 macOS jobs behind each other (#271).
 #
-# cancel-in-progress is scoped to pull_request ONLY. On a push to a long-lived
-# branch a superseded run is often the one you still want -- it is the record
-# for that commit -- so those are left to finish, and they get a group of their
-# own (run_id) so a queued push or nightly run is not replaced by a newer one
-# either. The group names the workflow literally: github.workflow is the
-# CALLER's name in a workflow_call, which would put this workflow and its
-# sibling in one group.
+# Per event:
+# * pull_request: one group per PR, a newer push cancels the older run.
+# * push to main: one group for the workflow, cancel-in-progress false. GitHub
+#   keeps at most one running and one pending run per group and lets a newer
+#   pending run replace the older pending one, so the queue never grows beyond
+#   two and the pending run is always the newest commit. Each merge was already
+#   gated green on its pull request.
+# * schedule (nightly): its own group, a stuck old nightly is cancelled.
+# * workflow_dispatch and workflow_call (the release): a group of their own per
+#   run, never cancelled. In a workflow_call github.event_name is the CALLER's
+#   (a tag push), which is why the push case also checks that the ref is main:
+#   a release run must never share a group with, or be replaced by, a push to main.
+#
+# The group names the workflow literally: github.workflow is the CALLER's name in
+# a workflow_call, which would put this workflow and its sibling in one group.
 concurrency:
-  group: test-python-examples-${{ github.event.pull_request.number || github.run_id }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: >-
+    ${{ github.event_name == 'pull_request' && format('test-python-examples-pr-{0}', github.event.pull_request.number)
+    || github.event_name == 'push' && github.ref == 'refs/heads/main' && 'test-python-examples-main'
+    || github.event_name == 'schedule' && 'test-python-examples-schedule'
+    || format('test-python-examples-run-{0}', github.run_id) }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'schedule' }}
 
 env:
   # The image download-jars takes the jars from. Unset (push, pull_request, schedule)

tests/test_workflow_concurrency.py

@@ -0,0 +1,117 @@
+"""Which runs of the test workflows cancel each other: superseded runs of a pull request, never a release or a dispatch."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOWS = {
+    name: ROOT / ".github" / "workflows" / f"{name}.yml"
+    for name in ("test-jvm", "test-python-examples", "test-native")
+}
+
+
+def _concurrency(text: str) -> tuple[str, str]:
+    """The group and cancel-in-progress expressions of the top-level concurrency block, `${{ }}` stripped."""
+    block = text.split("\nconcurrency:\n", 1)[1].split("\n\n", 1)[0]
+    group_text, cancel_text = block.split("  cancel-in-progress:", 1)
+    group = group_text.split("group:", 1)[1].replace(">-", "")
+    group = " ".join(line.strip() for line in group.strip().splitlines())
+    cancel = cancel_text.strip()
+    for value in (group, cancel):
+        assert value.startswith("${{") and value.endswith("}}"), value
+    return group[3:-2].strip(), cancel[3:-2].strip()
+
+
+def _evaluate(expression: str, context: dict[str, str]):
+    """GitHub's expression language restricted to what the blocks use: && || == != strings, format() and github.* names.
+
+    `&&` and `||` return an operand and bind like Python's `and` and `or`, so the translation keeps the meaning.
+    """
+
+    def fmt(template: str, *values: str) -> str:
+        return template.format(*values)
+
+    translated = re.sub(
+        r"github\.([a-z_.]+)", lambda m: repr(context.get(m.group(1), "")), expression
+    )
+    translated = translated.replace("&&", " and ").replace("||", " or ")
+    # only literals, operators and format() are left; nothing else may run
+    assert re.fullmatch(r"[\s\w'\"(),.=!/{}\-]*", translated), translated
+    return eval(translated, {"__builtins__": {}, "format": fmt})  # nosec B307
+
+
+def _run(workflow: str, event: str, ref: str, run_id: str = "1", pr: str = ""):
+    group, cancel = _concurrency(WORKFLOWS[workflow].read_text(encoding="utf-8"))
+    context = {
+        "event_name": event,
+        "ref": ref,
+        "ref_type": "tag" if ref.startswith("refs/tags/") else "branch",
+        "run_id": run_id,
+        "event.pull_request.number": pr,
+    }
+    return _evaluate(group, context), bool(_evaluate(cancel, context))
+
+
+@pytest.mark.parametrize("workflow", sorted(WORKFLOWS))
+def test_a_new_push_to_a_pull_request_cancels_the_old_run(workflow):
+    old = _run(workflow, "pull_request", "refs/pull/7/merge", "1", "7")
+    new = _run(workflow, "pull_request", "refs/pull/7/merge", "2", "7")
+    other = _run(workflow, "pull_request", "refs/pull/8/merge", "3", "8")
+    assert old == new and new[1] is True
+    assert other[0] != new[0]
+
+
+@pytest.mark.parametrize("workflow", sorted(WORKFLOWS))
+def test_pushes_to_main_share_one_group_so_no_backlog_builds(workflow):
+    first = _run(workflow, "push", "refs/heads/main", "1")
+    second = _run(workflow, "push", "refs/heads/main", "2")
+    assert first[0] == second[0]
+    if workflow == "test-native":
+        # the native workflow keeps its older rule: a newer push to main cancels the older run
+        assert first[1] is True
+    else:
+        # JVM and examples: one running and one pending run; a running run is never cancelled
+        assert first[1] is False
+
+
+@pytest.mark.parametrize("workflow", sorted(WORKFLOWS))
+def test_the_nightly_run_has_its_own_group_and_replaces_a_stuck_one(workflow):
+    nightly = _run(workflow, "schedule", "refs/heads/main", "1")
+    push = _run(workflow, "push", "refs/heads/main", "2")
+    assert nightly[0] != push[0]
+    assert nightly[1] is True
+    assert nightly == _run(workflow, "schedule", "refs/heads/main", "3")
+
+
+@pytest.mark.parametrize("workflow", sorted(WORKFLOWS))
+def test_a_dispatch_has_a_group_of_its_own_and_is_never_cancelled(workflow):
+    dispatch = _run(workflow, "workflow_dispatch", "refs/heads/main", "10")
+    push = _run(workflow, "push", "refs/heads/main", "11")
+    again = _run(workflow, "workflow_dispatch", "refs/heads/main", "12")
+    assert dispatch[1] is False
+    assert len({dispatch[0], push[0], again[0]}) == 3
+
+
+@pytest.mark.parametrize("workflow", sorted(WORKFLOWS))
+def test_a_release_run_is_never_cancelled_and_never_shares_the_group_of_main(workflow):
+    # a workflow_call from release-*.yml: github.event_name is the caller's, a push, and the ref is the tag
+    release = _run(workflow, "push", "refs/tags/26.11.1", "20")
+    main = _run(workflow, "push", "refs/heads/main", "21")
+    assert release[1] is False
+    assert release[0] != main[0]
+    # a newer push to main cannot replace it
+    assert _run(workflow, "push", "refs/heads/main", "22")[0] != release[0]
+
+
+def test_the_release_workflows_never_cancel_a_running_release():
+    for name in ("release-jvm", "release-native"):
+        text = (ROOT / ".github" / "workflows" / f"{name}.yml").read_text(
+            encoding="utf-8"
+        )
+        block = text.split("\nconcurrency:\n", 1)[1].split("\n\n", 1)[0]
+        assert "cancel-in-progress: false" in block
+        assert "ref_name" in block