From 5da4758d36d5da8eb270d96c9f35a0753a3b2b6d Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 5 Nov 2026 09:43:25 +0900 Subject: [PATCH 1/2] CI: a tag release ends with a GitHub Release built from the version's milestone (#253) workflow_run on both release workflows, not a last job in each: the two do not wait for each other and either can need a publish-only re-run, so each firing checks both for the tag commit and the second to finish creates the release. A PyPI check for both versions is the last gate. Notes come from scripts/release_notes.py (merged PRs of the milestone named after the base, in the tag and not in the previous tag of the same base, grouped by label), devN is a pre-release, a final release closes its milestone. Only contents/issues write plus two read scopes, no secret. Co-Authored-By: Claude Opus 5.5 --- .github/ISSUE_TEMPLATE/release.md | 10 +- .github/workflows/release-notes.yml | 146 ++++++++ CLAUDE.md | 2 +- docs/modules/ROOT/pages/development/ci.adoc | 2 +- .../ROOT/pages/development/release.adoc | 25 +- scripts/release_notes.py | 327 ++++++++++++++++++ 6 files changed, 506 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/release-notes.yml create mode 100644 scripts/release_notes.py diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md index 74a82dfc..f68acca7 100644 --- a/.github/ISSUE_TEMPLATE/release.md +++ b/.github/ISSUE_TEMPLATE/release.md @@ -22,7 +22,11 @@ Every step says who does it: script (run the command, read the result), CI (a wo ``` - [ ] 2. owner: for a final or post release, the native readiness bar is met and the repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET` is `true` (the preflight line `(g)` shows it, or SKIPs when the login cannot read variables). -- [ ] 3. owner: the go. Nothing is published before this box is ticked. +- [ ] 3. owner: the go. Nothing is published before this box is ticked. Before it, preview the GitHub Release notes and check that every pull request of this release has the milestone `` (a pull request without it is not listed): + + ```bash + uv run --no-project python -I scripts/release_notes.py --dry-run + ``` ## The tag (the only publishing step) @@ -32,7 +36,7 @@ Every step says who does it: script (run the command, read the result), CI (a wo git fetch origin main --no-tags && git tag origin/main && git push origin ``` -- [ ] 5. CI: both release workflows finish and publish. Note the two run ids (the native build takes the longest). To retry a failed upload, re-run only the publish job, never the whole workflow. +- [ ] 5. CI: both release workflows finish and publish, then `release-notes.yml` creates the GitHub Release and, for a final release, closes the milestone. Note the two run ids (the native build takes the longest). To retry a failed upload, re-run only the publish job, never the whole workflow. ```bash gh run list -R ArcadeData/arcadedb-python --branch --limit 6 @@ -69,4 +73,4 @@ Every step says who does it: script (run the command, read the result), CI (a wo If the engine NOTICE changed, also refresh the kept copy with `git -C show :NOTICE > native/wheel/ARCADEDB_ENGINE_NOTICE`. - [ ] 9. CI, then owner: merge the bump PR when every check is green, and delete the branch and the worktree. Then remove the tests and skip-list lines that waited for the new engine: `git grep -n -i "native build links\|native image links"`. -- [ ] 10. owner: close this issue with the two run ids and the two PyPI links. +- [ ] 10. owner: check the GitHub Release (notes, pre-release flag) and, after a final release, that the milestone is closed and its open items moved to the next one. Close this issue with the two run ids and the two PyPI links. diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml new file mode 100644 index 00000000..e1abad8e --- /dev/null +++ b/.github/workflows/release-notes.yml @@ -0,0 +1,146 @@ +name: GitHub Release + +# A tag release ends here. When BOTH release workflows of a tag (release-native.yml for `arcadedb`, release-jvm.yml for `arcadedb-jvm`) +# have finished successfully, this workflow creates the GitHub Release of the tag with notes written by scripts/release_notes.py from the +# milestone named after the base version, and closes that milestone after a FINAL release (A.B.C, not A.B.C.devN, not A.B.C.postN). +# +# Why `workflow_run` and not a last job in each release workflow, or a poll of PyPI: the two release workflows do not wait for each other +# (the native build and the JVM build take different times and either can need a retry), and a job inside one of them cannot know the +# other finished. `workflow_run` fires when each one completes, including after a re-run of only a publish job. Each firing looks at both +# workflows for the same tag commit, and does nothing unless both concluded `success` (the first one to finish therefore exits quietly, +# the second one creates the release). A workflow only concludes `success` after its publish job uploaded to PyPI and, for the JVM +# package, after the install from PyPI on every platform. As a last check the gate asks PyPI for both versions. If both firings run at +# once, the release is created by one and edited by the other (the concurrency group below also serialises them), so the result is the same. +# +# A failed or half done release (for example a final tag whose native readiness bar is not met) creates nothing. When the missing part is +# fixed and its publish job re-run, this workflow fires again. `workflow_dispatch` is the manual way for a tag that is already published +# (the gate runs the same checks), for example to create the release of a tag that was published before this workflow existed. +# +# `workflow_run` workflows always run the file on the default branch, so a change here takes effect after it is merged. + +on: + workflow_run: + workflows: ["Release native (arcadedb)", "Release JVM (arcadedb-jvm)"] # the `name:` of the two release workflows + types: [completed] + workflow_dispatch: + inputs: + tag: + description: "The bare version of a tag that is already published, for example 26.11.1" + required: true + type: string + +permissions: + contents: read + +concurrency: + group: github-release-${{ github.event.workflow_run.head_branch || inputs.tag }} + cancel-in-progress: false + +jobs: + github-release: + name: GitHub Release + # A release workflow that did not succeed, or one that was not started by a tag push, never gets here. + if: >- + ${{ github.repository == 'ArcadeData/arcadedb-python' && + (github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push')) }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write # create the release on the existing tag + issues: write # close the milestone, and read the milestone's pull requests + pull-requests: read # merge commit of each pull request + actions: read # the runs of the two release workflows + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Both packages must be published before anything is created. Exit 0 with ready=false means "the other package is not finished + # yet"; exit 1 means the workflows say success but PyPI does not have the version, which a person must look at. + - name: Both release workflows succeeded and both packages are on PyPI + id: gate + env: + INPUT_TAG: ${{ inputs.tag }} + RUN_TAG: ${{ github.event.workflow_run.head_branch }} + run: | + set -euo pipefail + tag="${INPUT_TAG:-$RUN_TAG}" + if ! [[ "$tag" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then + echo "::error::'$tag' is not A.B.C, A.B.C.devN or A.B.C.postN (bare version, no v, no aN/bN/rcN)" + exit 1 + fi + base="${tag%%.dev*}" + base="${base%%.post*}" + if [[ "$tag" == *.dev* ]]; then kind=dev; elif [[ "$tag" == *.post* ]]; then kind=post; else kind=final; fi + sha="$(gh api "repos/$GH_REPO/commits/$tag" --jq .sha)" + for wf in release-native.yml release-jvm.yml; do + conclusion="$(gh api "repos/$GH_REPO/actions/workflows/$wf/runs?head_sha=$sha&event=push&per_page=1" --jq '.workflow_runs[0].conclusion // "none"')" + echo "$wf for $tag ($sha): $conclusion" + if [ "$conclusion" != success ]; then + echo "::notice::$wf has not succeeded for $tag ($conclusion): no GitHub Release yet. This workflow runs again when it finishes." + echo "ready=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + done + for dist in arcadedb arcadedb-jvm; do + code="$(curl -sS -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/$dist/$tag/json")" + if [ "$code" != 200 ]; then + echo "::error::$dist $tag is not on PyPI (HTTP $code) although its release workflow succeeded" + exit 1 + fi + done + { echo "ready=true"; echo "tag=$tag"; echo "base=$base"; echo "kind=$kind"; } >> "$GITHUB_OUTPUT" + + - name: Write the notes + if: steps.gate.outputs.ready == 'true' + env: + TAG: ${{ steps.gate.outputs.tag }} + run: python3 -I scripts/release_notes.py "$TAG" --output "$RUNNER_TEMP/notes.md" + + # A pre-release for A.B.C.devN, a normal release for A.B.C and A.B.C.postN. --verify-tag: the tag must exist, the release never + # creates one. When the release exists (the other firing, or an earlier run) its notes and its pre-release flag are brought up to date. + - name: Create or update the GitHub Release + if: steps.gate.outputs.ready == 'true' + env: + TAG: ${{ steps.gate.outputs.tag }} + KIND: ${{ steps.gate.outputs.kind }} + run: | + set -euo pipefail + if [ "$KIND" = dev ]; then pre=true; else pre=false; fi + if gh release view "$TAG" > /dev/null 2>&1; then + gh release edit "$TAG" --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md" --prerelease="$pre" + elif [ "$pre" = true ]; then + gh release create "$TAG" --verify-tag --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md" --prerelease + else + gh release create "$TAG" --verify-tag --title "$TAG" --notes-file "$RUNNER_TEMP/notes.md" --latest + fi + gh release view "$TAG" --json url,isPrerelease --jq '"release: " + .url + " (prerelease: " + (.isPrerelease|tostring) + ")"' + + # Only a final release closes its milestone. A dev tag leaves it open (the cycle goes on), a post tag belongs to a milestone that is + # already closed. Anything still open in the milestone is listed in the log first: closing does not move or close it. + - name: Close the milestone of a final release + if: steps.gate.outputs.ready == 'true' && steps.gate.outputs.kind == 'final' + env: + BASE: ${{ steps.gate.outputs.base }} + run: | + set -euo pipefail + row="$(gh api "repos/$GH_REPO/milestones?state=all&per_page=100" | jq -r --arg t "$BASE" '[.[] | select(.title == $t)][0] // empty | "\(.number) \(.state) \(.open_issues)"')" + if [ -z "$row" ]; then + echo "::warning::No milestone named $BASE: nothing to close" + exit 0 + fi + read -r number state open_issues <<< "$row" + if [ "$state" = closed ]; then + echo "Milestone $BASE (#$number) is already closed" + exit 0 + fi + if [ "$open_issues" != 0 ]; then + echo "::warning::Milestone $BASE still has $open_issues open items; they stay in it. Move them to the next milestone if they are not part of this release." + gh api "repos/$GH_REPO/issues?milestone=$number&state=open&per_page=100" --jq '.[] | " open: #\(.number) \(.title)"' + fi + gh api --method PATCH "repos/$GH_REPO/milestones/$number" -f state=closed --jq '"closed milestone " + .title + " (" + .html_url + ")"' diff --git a/CLAUDE.md b/CLAUDE.md index 332cf3b8..304df0f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,6 @@ Each package has one test and one release workflow: `test-jvm.yml` and `release- `.github/workflows/test-jvm.yml` builds ONE wheel per platform (linux/amd64, linux/arm64, darwin/arm64, windows/amd64), tagged `py3-none-` by `setup.py` (`bdist_wheel.get_tag`; `scripts/verify_wheel_tag.py` checks it), and tests that wheel on Python 3.11–3.14; also runs nightly against `arcadedata/arcadedb:latest`. `requires-python` is `>=3.11,<3.15` because the tag no longer keeps pip off 3.15 and JPype has no 3.15 wheel; drop the cap with it. The org shares a runner cap with upstream, so the unit tests run 10 legs on every event (linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14, the shape of test-native.yml), and the examples run all four platforms on 3.14 on a pull request (4 legs, 28 checks in all) and 16 legs on push to main, the nightly run, dispatch and the release (40 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit locks the dependencies and extras on a throwaway uv project with `--resolution lowest-direct` for each Python classifier and runs `uvx pip-audit` on the export, so raise lower bounds when a floor has an advisory. No tool is installed with pip anywhere in CI or the scripts: one-off tools run through `uvx tool==X`, the wheel is built with `uvx --from "build[uv]==1.6.1" pyproject-build --installer uv`, and the uv install URL is pinned (`0.12.0`). -`.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/release_preflight.py ` checks everything before the tag (read-only); `scripts/after_tag_verify.sh` (`PACKAGE=native` for `arcadedb`) checks the finished release; `scripts/bump_engine.py` moves the dev-engine pins after a release. The monthly steps are the "Release day" section of `development/release.adoc` and the issue template `.github/ISSUE_TEMPLATE/release.md`. The rule is stated once, on the docs page `development/release.adoc`. +`.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/release_preflight.py ` checks everything before the tag (read-only); `scripts/after_tag_verify.sh` (`PACKAGE=native` for `arcadedb`) checks the finished release; `scripts/bump_engine.py` moves the dev-engine pins after a release. The monthly steps are the "Release day" section of `development/release.adoc` and the issue template `.github/ISSUE_TEMPLATE/release.md`. `.github/workflows/release-notes.yml` (`workflow_run` on both release workflows, `workflow_dispatch` for a published tag) creates the GitHub Release once both packages are published, with notes from `scripts/release_notes.py` (merged pull requests of the milestone named after the base version, grouped by label; `--dry-run` previews), a pre-release for `devN`, and closes the milestone after a final release. The rule is stated once, on the docs page `development/release.adoc`. `test-native.yml` builds the GraalVM native image and runs the parity suite against the installed wheel (selective: scripts/select_native_matrix.py, 5 to 30 minutes per platform, do not add runs lightly). `.github/workflows/release-native.yml` publishes `arcadedb` on the same tag: it calls `test-native.yml` with `release: true` (4 platforms x Python 3.11 to 3.14), then the readiness gate (the repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET`, for a final or post version), then one publish job through the environment `pypi`. diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index 1610a65e..cfa63422 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -16,7 +16,7 @@ Each package has one test workflow and one release workflow: | `release-native.yml` |=== -The release workflows call the test workflows (`workflow_call`) and then publish; see xref:development/release.adoc[Release]. Two workflows test the `arcadedb-jvm` package: `test-jvm.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheel once for each platform, on a native runner of that platform (`Build arcadedb-jvm wheel ()` and `Build wheel for examples ()`), and install that one file into each Python they test. `test-native.yml` builds and tests the `arcadedb` (native) package; see <<_native_build,Native build>>. +The release workflows call the test workflows (`workflow_call`) and then publish; see xref:development/release.adoc[Release]. `release-notes.yml` runs after both of them have succeeded for a tag and creates the GitHub Release (and closes the milestone of a final release). Two workflows test the `arcadedb-jvm` package: `test-jvm.yml` builds a wheel and runs the unit tests, and `test-python-examples.yml` runs the scripts in `examples/`. Both build the wheel once for each platform, on a native runner of that platform (`Build arcadedb-jvm wheel ()` and `Build wheel for examples ()`), and install that one file into each Python they test. `test-native.yml` builds and tests the `arcadedb` (native) package; see <<_native_build,Native build>>. The workflow `docs.yml` builds these docs (the Antora component in `docs/`) as a site of its own with `docs/antora-playbook.yml`, on every pull request and push to `main` that changes `docs/`, and attaches the site as the `docs-site` artifact. Download it from the run to read a page as it will render. The test workflows ignore `docs/**` and `*.adoc`, so for a docs-only pull request this job is the only check. A warning fails it: a broken `xref:`, a misspelled `+{attribute}+`, an AsciiDoc syntax problem. Pages link into the main ArcadeDB manual as `+{arcadedb-docs-url}/+`, not with an `arcadedb::` xref, which the component on its own cannot resolve. To build the site locally, run `npx -p @antora/cli@3.2.1 -p @antora/site-generator@3.2.1 antora docs/antora-playbook.yml` from the repository root and open `docs/build/site/index.html`. diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index a116bc46..e205bcd7 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -133,7 +133,7 @@ Each step is a script, CI, or the owner's decision. The owner's tag push is the | owner (the only publishing step) | 4 -| `release-jvm.yml` and `release-native.yml` validate the tag, build and test the wheels, check them, and publish with trusted publishing. A failed upload is retried by re-running only the publish job. +| `release-jvm.yml` and `release-native.yml` validate the tag, build and test the wheels, check them, and publish with trusted publishing. A failed upload is retried by re-running only the publish job. When both have succeeded, `release-notes.yml` creates the GitHub Release and, for a final release, closes the milestone (<<_github_release,GitHub Release>>). | CI | 5 @@ -159,6 +159,9 @@ uv run --no-project python -I scripts/release_preflight.py # 3. the tag (owner) git fetch origin main --no-tags && git tag origin/main && git push origin +# 3a. preview the GitHub Release notes (read-only; a tag that does not exist yet is read as the tip of main) +uv run --no-project python -I scripts/release_notes.py --dry-run + # 5. after the workflows finished (run ids: gh run list -R ArcadeData/arcadedb-python --branch ) scripts/after_tag_verify.sh PACKAGE=native scripts/after_tag_verify.sh @@ -168,6 +171,26 @@ uv run --no-project python scripts/bump_engine.py --engine --dev-versi uv run --no-project python scripts/bump_engine.py --engine --dev-version ---- +[[_github_release]] +=== GitHub Release and milestone + +A release does not end on PyPI: it ends with a GitHub Release, as in the upstream repository. `.github/workflows/release-notes.yml` runs when `release-native.yml` or `release-jvm.yml` completes (`workflow_run`). It looks at both workflows for the commit of the tag and does nothing until both concluded `success`, so the workflow that finishes first exits quietly and the second one creates the release. Both workflows only succeed after their publish job uploaded to PyPI, and the gate also asks PyPI for both versions. A tag whose native build is held back by the readiness gate therefore gets no GitHub Release until that package is published and its publish job re-run. `workflow_dispatch` with the tag does the same by hand for a tag that is already published. It needs no secret: `GITHUB_TOKEN` with `contents: write` (the release), `issues: write` (the milestone), `pull-requests: read` and `actions: read`. + +* `A.B.C.devN` becomes a GitHub pre-release. `A.B.C` and `A.B.C.postN` become a normal release, marked as the latest. +* The notes come from `scripts/release_notes.py`. The milestone is named after the base `A.B.C` (`26.11.1` for `26.11.1`, `26.11.1.dev2` and `26.11.1.post1`). A pull request is listed when it is merged, has that milestone and its merge commit is in the tag, grouped by its first matching label: `bug` (Bug fixes), `performance` (Performance), `enhancement` (Features), `documentation` (Documentation), everything else under Other. Each line is the title and `(#N)`. The notes also have the install lines, the engine version and the PyPI pages of both packages. +* A final release lists the whole milestone. A `devN` and a `postN` list only what is not in the previous tag of the same base (`dev(N-1)`; for `postN` the final or `post(N-1)`). A pull request without the milestone is not in the notes, so check the milestone before the tag. +* After a final release (not `devN`, not `postN`) the milestone `A.B.C` is closed. Items still open in it stay in it and are listed in the log of the run; move them to the next milestone by hand. +* A run for a release that already exists edits its notes and flag, so a re-run is safe. + +Preview the notes before the tag, and read them again after the release: + +[source,bash] +---- +uv run --no-project python -I scripts/release_notes.py --dry-run +---- + +The script only reads (`gh api`, so it needs `gh auth login` or `GH_TOKEN`) and prints the notes. A tag that does not exist yet is read as the tip of `main`. + === Choosing the next dev version A dev version of a released base is refused, so after `` is out the dev line moves to the next base. The next base is the version of upstream `main`, which carries `-SNAPSHOT`: diff --git a/scripts/release_notes.py b/scripts/release_notes.py new file mode 100644 index 00000000..52abe448 --- /dev/null +++ b/scripts/release_notes.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Write the notes of the GitHub Release of one tag from the milestone of its version. + +The tag is a bare version (docs page "Release"): `A.B.C.devN`, `A.B.C` or `A.B.C.postN`. The milestone is named after the base +`A.B.C`. The notes list the pull requests of that milestone that are merged, grouped by label, then the install lines, the engine +version and the PyPI pages of the two packages. + +Which pull requests a tag lists: + +- `A.B.C` every merged pull request of the milestone that is in the tag (the first release of a base is the whole cycle) +- `A.B.C.devN` those that are in the tag and not in the previous tag of the same base (`A.B.C.dev(N-1)`), so a preview lists only + what is new since the last preview +- `A.B.C.postN` those that are in the tag and not in the previous tag of the same base (`A.B.C` or `A.B.C.post(N-1)`) + +"In a tag" means the merge commit of the pull request is an ancestor of the tag (the GitHub compare API), so a pull request that merges +after the tag, while the release still runs, is not listed. A tag that does not exist yet (a dry run before the release) is read as the +tip of the default branch. + +Only reads: every API call is a GET through `gh api` (the token is the GH_TOKEN of the caller). Nothing is created or changed here; +the release workflow does that with the file this script writes. + +usage: + release_notes.py TAG --dry-run # print the notes + release_notes.py TAG --output notes.md # write them to a file + release_notes.py TAG --repo OWNER/NAME # default: $GITHUB_REPOSITORY, else ArcadeData/arcadedb-python + +Exit codes: 0 notes written, 1 the tag is not a valid release tag, 2 an API call failed. +""" + +import argparse +import base64 +import json +import os +import re +import subprocess +import sys +from urllib.parse import quote + +DEFAULT_REPO = "ArcadeData/arcadedb-python" +UPSTREAM = "ArcadeData/arcadedb" +NATIVE_DIST = "arcadedb" +JVM_DIST = "arcadedb-jvm" + +TAG_RE = re.compile(r"^(?P\d+\.\d+\.\d+)(?:\.(?Pdev|post)(?P\d+))?$") + +# First matching label wins; a pull request with none of them goes to "Other". +SECTIONS = [ + ("Bug fixes", "bug"), + ("Performance", "performance"), + ("Features", "enhancement"), + ("Documentation", "documentation"), +] +OTHER = "Other" + + +class ApiError(Exception): + pass + + +def gh_api(path): + """GET one API path through `gh api` and return the decoded JSON (None for a 404).""" + cmd = ["gh", "api", path] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + if "HTTP 404" in proc.stderr: + return None + raise ApiError(f"gh api {path}: {proc.stderr.strip() or proc.stdout.strip()}") + return json.loads(proc.stdout) if proc.stdout.strip() else None + + +def gh_pages(path): + """All pages of a list endpoint (100 per page).""" + out = [] + sep = "&" if "?" in path else "?" + page = 1 + while True: + chunk = gh_api(f"{path}{sep}per_page=100&page={page}") or [] + out.extend(chunk) + if len(chunk) < 100: + return out + page += 1 + + +def parse_tag(tag): + m = TAG_RE.match(tag) + if not m: + raise ValueError( + f"'{tag}' is not A.B.C, A.B.C.devN or A.B.C.postN (bare version, no v, no aN/bN/rcN)" + ) + kind = m.group("kind") or "final" + n = int(m.group("n") or 0) + return m.group("base"), kind, n + + +def order_key(kind, n): + """Order inside one base: dev0 < dev1 < ... < final < post1 < post2 ...""" + return {"dev": (0, n), "final": (1, 0), "post": (2, n)}[kind] + + +def previous_tag(tag, existing): + """The tag a release's list starts after, or None (a final lists the whole milestone).""" + base, kind, n = parse_tag(tag) + if kind == "final": + return None + me = order_key(kind, n) + best = None + for other in existing: + m = TAG_RE.match(other) + if not m or m.group("base") != base: + continue + okind = m.group("kind") or "final" + key = order_key(okind, int(m.group("n") or 0)) + if key < me and (best is None or key > best[0]): + best = (key, other) + return best[1] if best else None + + +def section_of(labels): + names = set(labels) + for title, label in SECTIONS: + if label in names: + return title + return OTHER + + +def clean_title(title, number): + """The title without a trailing (#N) of the pull request itself, which the line adds.""" + return re.sub(rf"\s*\(#{number}\)\s*$", "", title.strip()) + + +def safe_title(title): + """No @mention in a release body: a zero width space after the @ stops the notification.""" + return re.sub(r"@(?=\w)", "@​", title.strip()) + + +def is_ancestor(repo, sha, ref): + """True when commit `sha` is `ref` itself or one of its ancestors.""" + data = gh_api(f"repos/{repo}/compare/{sha}...{quote(ref, safe='')}?per_page=1") + return bool(data) and data.get("status") in ("ahead", "identical") + + +def milestone_prs(repo, base): + """(milestone dict or None, merged pull requests of it as dicts with number, title, labels, merge sha).""" + milestones = gh_pages(f"repos/{repo}/milestones?state=all") + milestone = next((m for m in milestones if m["title"] == base), None) + if milestone is None: + return None, [] + items = gh_pages(f"repos/{repo}/issues?milestone={milestone['number']}&state=all") + prs = [] + for item in items: + pull = item.get("pull_request") + if not pull or not pull.get("merged_at"): + continue + detail = gh_api(f"repos/{repo}/pulls/{item['number']}") + if not detail or not detail.get("merge_commit_sha"): + continue + prs.append( + { + "number": item["number"], + "title": item["title"], + "labels": [lb["name"] for lb in item.get("labels", [])], + "sha": detail["merge_commit_sha"], + "merged_at": pull["merged_at"], + } + ) + return milestone, prs + + +def engine_of(repo, kind, base, ref): + """(engine version, how it is known). A final or post wheel links the engine it is named after.""" + if kind != "dev": + return base, "release" + data = gh_api( + f"repos/{repo}/contents/.github/workflows/test-native.yml?ref={quote(ref, safe='')}" + ) + if data and data.get("content"): + text = base64.b64decode(data["content"]).decode() + m = re.search(r'^\s*DEV_ENGINE:\s*"?([0-9][^"\s]*)"?', text, re.M) + if m: + return m.group(1), "dev" + return None, "dev" + + +def build_notes(repo, tag): + base, kind, n = parse_tag(tag) + default_branch = (gh_api(f"repos/{repo}") or {}).get("default_branch", "main") + tag_ref = gh_api(f"repos/{repo}/git/ref/tags/{quote(tag, safe='')}") + if tag_ref: + ref = tag + else: + ref = default_branch + print( + f"note: tag {tag} does not exist yet, reading the tip of {default_branch}", + file=sys.stderr, + ) + + tags = [t["name"] for t in gh_pages(f"repos/{repo}/tags")] + prev = previous_tag(tag, [t for t in tags if t != tag]) + milestone, prs = milestone_prs(repo, base) + + listed = [] + for pr in prs: + if not is_ancestor(repo, pr["sha"], ref): + continue + if prev and is_ancestor(repo, pr["sha"], prev): + continue + listed.append(pr) + listed.sort(key=lambda p: p["number"]) + + groups = {title: [] for title, _ in SECTIONS} + groups[OTHER] = [] + for pr in listed: + groups[section_of(pr["labels"])].append(pr) + + engine, engine_kind = engine_of(repo, kind, base, ref) + lines = [] + if kind == "dev": + lines.append( + f"Pre-release of `{base}`: a preview built from the engine release named below, for trying the next release early." + ) + elif kind == "post": + lines.append( + f"A fix in the Python code of `{base}`. The wheels still carry the `{base}` engine." + ) + lines.append("") + + if milestone is None: + lines.append( + f"No milestone named `{base}` exists, so this release has no change list." + ) + print(f"warning: no milestone named {base}", file=sys.stderr) + lines.append("") + else: + scope = f"since `{prev}`" if prev else f"in the milestone `{base}`" + lines.append(f"## What changed {scope}") + lines.append("") + if not listed: + lines.append("No merged pull request.") + lines.append("") + for title in [t for t, _ in SECTIONS] + [OTHER]: + if not groups[title]: + continue + lines.append(f"### {title}") + lines.append("") + for pr in groups[title]: + shown = safe_title(clean_title(pr["title"], pr["number"])) + lines.append(f"- {shown} (#{pr['number']})") + lines.append("") + + lines.append("## Install") + lines.append("") + lines.append("```bash") + if kind == "dev": + lines.append(f'uv add "{NATIVE_DIST}=={tag}" # native build') + lines.append(f'uv add "{JVM_DIST}=={tag}" # JVM build') + else: + lines.append(f"uv add {NATIVE_DIST} # native build") + lines.append(f"uv add {JVM_DIST} # JVM build") + lines.append("```") + lines.append("") + if kind == "dev": + lines.append( + "A pre-release is installed only when it is named exactly, as above, or with `--prerelease allow`." + ) + lines.append("") + + lines.append("## Engine") + lines.append("") + if engine: + lines.append( + f"ArcadeDB engine [{engine}](https://github.com/{UPSTREAM}/releases/tag/{engine})." + ) + else: + lines.append("ArcadeDB engine: not found in `test-native.yml` for this tag.") + if kind == "dev": + lines.append( + f"This is the engine the native `{NATIVE_DIST}` wheels link. The `{JVM_DIST}` preview wheels take the jars of the upstream image tag `latest`." + ) + lines.append("") + + lines.append("## Packages") + lines.append("") + lines.append( + f"- [{NATIVE_DIST} {tag}](https://pypi.org/project/{NATIVE_DIST}/{tag}/)" + ) + lines.append(f"- [{JVM_DIST} {tag}](https://pypi.org/project/{JVM_DIST}/{tag}/)") + lines.append("") + if prev: + lines.append( + f"Full change list: https://github.com/{repo}/compare/{prev}...{tag}" + ) + lines.append("") + return "\n".join(lines).strip() + "\n" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("tag", help="the bare version, for example 26.11.1") + ap.add_argument( + "--repo", default=os.environ.get("GITHUB_REPOSITORY") or DEFAULT_REPO + ) + ap.add_argument( + "--dry-run", action="store_true", help="print the notes (the default output)" + ) + ap.add_argument("--output", help="write the notes to this file instead of stdout") + args = ap.parse_args(argv) + try: + parse_tag(args.tag) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + try: + notes = build_notes(args.repo, args.tag) + except ApiError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + if args.output and not args.dry_run: + with open(args.output, "w", encoding="utf-8") as fh: + fh.write(notes) + print(f"wrote {args.output}", file=sys.stderr) + else: + sys.stdout.write(notes) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From cba263d8c59af2945560f65e5debc42f5082f6c2 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 5 Nov 2026 09:48:09 +0900 Subject: [PATCH 2/2] Release notes: postN lists the commit range, a final release moves open items to the next milestone (#253) The final release closes the base milestone, so a postN cannot depend on it: its notes now come from the PRs whose merge commit lies between the previous tag of the same base and the tag, whatever milestone they carry. Closing a milestone with open items would strand them, so a final release first moves them to the open milestone due next; with none it creates one named after the next upstream engine version (due 30 days later), and when that cannot be read it moves and closes nothing and fails with a message, because guessing a name (A.B.C.post1) would be wrong under the no-milestone rule for post releases. Every move is logged. Co-Authored-By: Claude Opus 5.5 --- .github/ISSUE_TEMPLATE/release.md | 4 +- .github/workflows/release-notes.yml | 43 +++++++++-- CLAUDE.md | 2 +- .../ROOT/pages/development/release.adoc | 5 +- scripts/release_notes.py | 73 +++++++++++++++---- 5 files changed, 99 insertions(+), 28 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md index f68acca7..4850e9d3 100644 --- a/.github/ISSUE_TEMPLATE/release.md +++ b/.github/ISSUE_TEMPLATE/release.md @@ -22,7 +22,7 @@ Every step says who does it: script (run the command, read the result), CI (a wo ``` - [ ] 2. owner: for a final or post release, the native readiness bar is met and the repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET` is `true` (the preflight line `(g)` shows it, or SKIPs when the login cannot read variables). -- [ ] 3. owner: the go. Nothing is published before this box is ticked. Before it, preview the GitHub Release notes and check that every pull request of this release has the milestone `` (a pull request without it is not listed): +- [ ] 3. owner: the go. Nothing is published before this box is ticked. Before it, preview the GitHub Release notes and check that every pull request of this release has the milestone `` (a pull request without it is not listed in a `devN` or final release; a `postN` lists the commit range instead and needs no milestone): ```bash uv run --no-project python -I scripts/release_notes.py --dry-run @@ -73,4 +73,4 @@ Every step says who does it: script (run the command, read the result), CI (a wo If the engine NOTICE changed, also refresh the kept copy with `git -C show :NOTICE > native/wheel/ARCADEDB_ENGINE_NOTICE`. - [ ] 9. CI, then owner: merge the bump PR when every check is green, and delete the branch and the worktree. Then remove the tests and skip-list lines that waited for the new engine: `git grep -n -i "native build links\|native image links"`. -- [ ] 10. owner: check the GitHub Release (notes, pre-release flag) and, after a final release, that the milestone is closed and its open items moved to the next one. Close this issue with the two run ids and the two PyPI links. +- [ ] 10. owner: check the GitHub Release (notes, pre-release flag) and, after a final release, that the milestone is closed and what was still open in it moved to the next milestone (the open one due next, or one the workflow created named after the next upstream engine version). If the run failed with a message about the next milestone, create it with a due date and re-run `GitHub Release` (Actions, `workflow_dispatch`, the tag). Close this issue with the two run ids and the two PyPI links. diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml index e1abad8e..766498ba 100644 --- a/.github/workflows/release-notes.yml +++ b/.github/workflows/release-notes.yml @@ -2,7 +2,8 @@ name: GitHub Release # A tag release ends here. When BOTH release workflows of a tag (release-native.yml for `arcadedb`, release-jvm.yml for `arcadedb-jvm`) # have finished successfully, this workflow creates the GitHub Release of the tag with notes written by scripts/release_notes.py from the -# milestone named after the base version, and closes that milestone after a FINAL release (A.B.C, not A.B.C.devN, not A.B.C.postN). +# milestone named after the base version (a `postN` uses the commit range since the previous tag instead), and closes that milestone after +# a FINAL release (A.B.C, not A.B.C.devN, not A.B.C.postN), after moving what is still open in it to the next milestone. # # Why `workflow_run` and not a last job in each release workflow, or a poll of PyPI: the two release workflows do not wait for each other # (the native build and the JVM build take different times and either can need a retry), and a job inside one of them cannot know the @@ -121,26 +122,52 @@ jobs: fi gh release view "$TAG" --json url,isPrerelease --jq '"release: " + .url + " (prerelease: " + (.isPrerelease|tostring) + ")"' - # Only a final release closes its milestone. A dev tag leaves it open (the cycle goes on), a post tag belongs to a milestone that is - # already closed. Anything still open in the milestone is listed in the log first: closing does not move or close it. - - name: Close the milestone of a final release + # Only a final release closes its milestone. A dev tag leaves it open (the cycle goes on); a post tag has no milestone of its own + # (its notes come from the commit range, not from a milestone). Items still open in the milestone move to the next one first, and + # every move is logged. The next one is the open milestone with the earliest due date after this one. When there is none, the + # milestone is created, named after the next upstream engine version (due 30 days after this one; the base of the upstream `main` version, as in + # development/release.adoc), when that can be read. Otherwise nothing is moved or closed and the job fails, so the maintainer + # creates the next milestone and re-runs this workflow (the release step is a no-op edit the second time). + - name: Move the open items and close the milestone of a final release if: steps.gate.outputs.ready == 'true' && steps.gate.outputs.kind == 'final' env: BASE: ${{ steps.gate.outputs.base }} run: | set -euo pipefail - row="$(gh api "repos/$GH_REPO/milestones?state=all&per_page=100" | jq -r --arg t "$BASE" '[.[] | select(.title == $t)][0] // empty | "\(.number) \(.state) \(.open_issues)"')" + milestones="$(gh api "repos/$GH_REPO/milestones?state=all&per_page=100")" + row="$(jq -r --arg t "$BASE" '[.[] | select(.title == $t)][0] // empty | "\(.number) \(.state) \(.open_issues) \(.due_on // "")"' <<< "$milestones")" if [ -z "$row" ]; then echo "::warning::No milestone named $BASE: nothing to close" exit 0 fi - read -r number state open_issues <<< "$row" + read -r number state open_issues due <<< "$row" if [ "$state" = closed ]; then echo "Milestone $BASE (#$number) is already closed" exit 0 fi if [ "$open_issues" != 0 ]; then - echo "::warning::Milestone $BASE still has $open_issues open items; they stay in it. Move them to the next milestone if they are not part of this release." - gh api "repos/$GH_REPO/issues?milestone=$number&state=open&per_page=100" --jq '.[] | " open: #\(.number) \(.title)"' + next="$(jq -r --arg n "$number" --arg due "$due" '[.[] | select(.state == "open" and (.number | tostring) != $n and .due_on != null and ($due == "" or .due_on > $due))] | sort_by(.due_on) | .[0].number // empty' <<< "$milestones")" + if [ -z "$next" ]; then + upstream="$(curl -fsS https://raw.githubusercontent.com/ArcadeData/arcadedb/main/pom.xml | sed -n '/arcadedb-parent<\/artifactId>/,/<\/version>/ s/.*\(.*\)-SNAPSHOT<\/version>.*/\1/p' | head -n 1 || true)" + if [[ "$upstream" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$upstream" != "$BASE" ]]; then + found="$(jq -r --arg t "$upstream" '[.[] | select(.title == $t)][0] // empty | "\(.number) \(.state)"' <<< "$milestones")" + if [ -z "$found" ]; then + # due 30 days after this one (releases are monthly), so a pick-by-due-date finds it + args=(-f title="$upstream") + [ -z "$due" ] || args+=(-f due_on="$(date -u -d "$due + 30 days" +%Y-%m-%dT%H:%M:%SZ)") + next="$(gh api --method POST "repos/$GH_REPO/milestones" "${args[@]}" --jq .number)" + echo "Created milestone $upstream (#$next), named after the next upstream engine version" + elif [ "${found#* }" = open ]; then + next="${found%% *}" + fi + fi + fi + if [ -z "$next" ]; then + echo "::error::Milestone $BASE has $open_issues open items and there is no next open milestone with a due date, and the next upstream version could not be used (read '${upstream:-nothing}'). Nothing was moved or closed. Create the next milestone with a due date, then re-run this workflow." + exit 1 + fi + for item in $(gh api --paginate "repos/$GH_REPO/issues?milestone=$number&state=open&per_page=100" --jq '.[].number'); do + gh api --method PATCH "repos/$GH_REPO/issues/$item" -F milestone="$next" --jq '"moved #\(.number) \(.title) to milestone \(.milestone.title)"' + done fi gh api --method PATCH "repos/$GH_REPO/milestones/$number" -f state=closed --jq '"closed milestone " + .title + " (" + .html_url + ")"' diff --git a/CLAUDE.md b/CLAUDE.md index 304df0f8..75a69798 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,6 @@ Each package has one test and one release workflow: `test-jvm.yml` and `release- `.github/workflows/test-jvm.yml` builds ONE wheel per platform (linux/amd64, linux/arm64, darwin/arm64, windows/amd64), tagged `py3-none-` by `setup.py` (`bdist_wheel.get_tag`; `scripts/verify_wheel_tag.py` checks it), and tests that wheel on Python 3.11–3.14; also runs nightly against `arcadedata/arcadedb:latest`. `requires-python` is `>=3.11,<3.15` because the tag no longer keeps pip off 3.15 and JPype has no 3.15 wheel; drop the cap with it. The org shares a runner cap with upstream, so the unit tests run 10 legs on every event (linux/amd64 on all four Pythons, the other three platforms on 3.11 and 3.14, the shape of test-native.yml), and the examples run all four platforms on 3.14 on a pull request (4 legs, 28 checks in all) and 16 legs on push to main, the nightly run, dispatch and the release (40 checks). `scripts/select_matrix.py` holds both lists (`PULL_REQUEST_LEGS`, keyed `jvm` and `examples`), the first job of each workflow runs it, and `tests/test_select_matrix.py` pins the full matrix; docs page `development/ci.adoc`. Keep the job names `Test arcadedb (, Python )` unchanged. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit locks the dependencies and extras on a throwaway uv project with `--resolution lowest-direct` for each Python classifier and runs `uvx pip-audit` on the export, so raise lower bounds when a floor has an advisory. No tool is installed with pip anywhere in CI or the scripts: one-off tools run through `uvx tool==X`, the wheel is built with `uvx --from "build[uv]==1.6.1" pyproject-build --installer uv`, and the uv install URL is pinned (`0.12.0`). -`.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/release_preflight.py ` checks everything before the tag (read-only); `scripts/after_tag_verify.sh` (`PACKAGE=native` for `arcadedb`) checks the finished release; `scripts/bump_engine.py` moves the dev-engine pins after a release. The monthly steps are the "Release day" section of `development/release.adoc` and the issue template `.github/ISSUE_TEMPLATE/release.md`. `.github/workflows/release-notes.yml` (`workflow_run` on both release workflows, `workflow_dispatch` for a published tag) creates the GitHub Release once both packages are published, with notes from `scripts/release_notes.py` (merged pull requests of the milestone named after the base version, grouped by label; `--dry-run` previews), a pre-release for `devN`, and closes the milestone after a final release. The rule is stated once, on the docs page `development/release.adoc`. +`.github/workflows/release-jvm.yml` publishes `arcadedb-jvm` on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/release_preflight.py ` checks everything before the tag (read-only); `scripts/after_tag_verify.sh` (`PACKAGE=native` for `arcadedb`) checks the finished release; `scripts/bump_engine.py` moves the dev-engine pins after a release. The monthly steps are the "Release day" section of `development/release.adoc` and the issue template `.github/ISSUE_TEMPLATE/release.md`. `.github/workflows/release-notes.yml` (`workflow_run` on both release workflows, `workflow_dispatch` for a published tag) creates the GitHub Release once both packages are published, with notes from `scripts/release_notes.py` (merged pull requests of the milestone named after the base version for a final and a `devN` release, the commit range since the previous tag for `postN`, grouped by label; `--dry-run` previews), a pre-release for `devN`, and after a final release moves the open items to the next milestone (created from the next upstream engine version when none exists) and closes the milestone. The rule is stated once, on the docs page `development/release.adoc`. `test-native.yml` builds the GraalVM native image and runs the parity suite against the installed wheel (selective: scripts/select_native_matrix.py, 5 to 30 minutes per platform, do not add runs lightly). `.github/workflows/release-native.yml` publishes `arcadedb` on the same tag: it calls `test-native.yml` with `release: true` (4 platforms x Python 3.11 to 3.14), then the readiness gate (the repository variable `ARCADEDB_NATIVE_READINESS_BAR_MET`, for a final or post version), then one publish job through the environment `pypi`. diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index e205bcd7..a4861351 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -178,8 +178,9 @@ A release does not end on PyPI: it ends with a GitHub Release, as in the upstrea * `A.B.C.devN` becomes a GitHub pre-release. `A.B.C` and `A.B.C.postN` become a normal release, marked as the latest. * The notes come from `scripts/release_notes.py`. The milestone is named after the base `A.B.C` (`26.11.1` for `26.11.1`, `26.11.1.dev2` and `26.11.1.post1`). A pull request is listed when it is merged, has that milestone and its merge commit is in the tag, grouped by its first matching label: `bug` (Bug fixes), `performance` (Performance), `enhancement` (Features), `documentation` (Documentation), everything else under Other. Each line is the title and `(#N)`. The notes also have the install lines, the engine version and the PyPI pages of both packages. -* A final release lists the whole milestone. A `devN` and a `postN` list only what is not in the previous tag of the same base (`dev(N-1)`; for `postN` the final or `post(N-1)`). A pull request without the milestone is not in the notes, so check the milestone before the tag. -* After a final release (not `devN`, not `postN`) the milestone `A.B.C` is closed. Items still open in it stay in it and are listed in the log of the run; move them to the next milestone by hand. +* A final release lists the whole milestone. A `devN` lists only what is not in the previous dev tag of the same base. A pull request without the milestone is not in the notes of a final or dev release, so check the milestone before the tag. +* A `postN` does not use a milestone, because the final release closed it. Its notes list every pull request merged between the previous tag of the same base (the final or `post(N-1)`) and the tag, found from the commits of that range, whatever milestone the pull request has or none. +* After a final release (not `devN`, not `postN`) the milestone `A.B.C` is closed, after what is still open in it moves to the next milestone. The next milestone is the open one with the earliest due date after this one. When there is none, the workflow creates one named after the next upstream engine version (the base of the upstream `main` version, the same command as in <<_choosing_the_next_dev_version,Choosing the next dev version>>) and due 30 days after the closed one. When that version cannot be read, or is the same version, nothing is moved or closed and the run fails with a message: create the next milestone with a due date and re-run the workflow. Every move is logged. The milestone auto-assignment of merged pull requests, where it is set up, then picks the next milestone, because the closed one is no longer open. * A run for a release that already exists edits its notes and flag, so a re-run is safe. Preview the notes before the tag, and read them again after the release: diff --git a/scripts/release_notes.py b/scripts/release_notes.py index 52abe448..fff5a95f 100644 --- a/scripts/release_notes.py +++ b/scripts/release_notes.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 """Write the notes of the GitHub Release of one tag from the milestone of its version. -The tag is a bare version (docs page "Release"): `A.B.C.devN`, `A.B.C` or `A.B.C.postN`. The milestone is named after the base -`A.B.C`. The notes list the pull requests of that milestone that are merged, grouped by label, then the install lines, the engine -version and the PyPI pages of the two packages. +The tag is a bare version (docs page "Release"): `A.B.C.devN`, `A.B.C` or `A.B.C.postN`. The notes list merged pull requests grouped +by label, then the install lines, the engine version and the PyPI pages of the two packages. The pull requests of a `devN` and of a +final release come from the milestone named after the base `A.B.C`; a `postN` does not use a milestone (see below). Which pull requests a tag lists: - `A.B.C` every merged pull request of the milestone that is in the tag (the first release of a base is the whole cycle) - `A.B.C.devN` those that are in the tag and not in the previous tag of the same base (`A.B.C.dev(N-1)`), so a preview lists only what is new since the last preview -- `A.B.C.postN` those that are in the tag and not in the previous tag of the same base (`A.B.C` or `A.B.C.post(N-1)`) +- `A.B.C.postN` every pull request merged between the previous tag of the same base (`A.B.C` or `A.B.C.post(N-1)`) and the tag, found + from the commits of that range, with whatever milestone it has or none: the final release closes the milestone + `A.B.C`, so a fix made after it is not in it -"In a tag" means the merge commit of the pull request is an ancestor of the tag (the GitHub compare API), so a pull request that merges +For `devN` and the final release "in a tag" means the merge commit of the pull request is an ancestor of the tag (the GitHub compare API), so a pull request that merges after the tag, while the release still runs, is not listed. A tag that does not exist yet (a dry run before the release) is read as the tip of the default branch. @@ -130,7 +132,7 @@ def clean_title(title, number): def safe_title(title): """No @mention in a release body: a zero width space after the @ stops the notification.""" - return re.sub(r"@(?=\w)", "@​", title.strip()) + return re.sub(r"@(?=\w)", "@\u200b", title.strip()) def is_ancestor(repo, sha, ref): @@ -166,6 +168,39 @@ def milestone_prs(repo, base): return milestone, prs +def compare_commits(repo, base, head): + """The commits after `base` up to `head` (all pages of the compare API).""" + out = [] + page = 1 + while True: + data = gh_api( + f"repos/{repo}/compare/{quote(base, safe='')}...{quote(head, safe='')}" + f"?per_page=100&page={page}" + ) + commits = (data or {}).get("commits", []) + out.extend(commits) + if len(commits) < 100: + return out + page += 1 + + +def range_prs(repo, prev, ref): + """Merged pull requests whose merge commit lies between `prev` (excluded) and `ref`, whatever their milestone.""" + found = {} + for commit in compare_commits(repo, prev, ref): + for pr in gh_api(f"repos/{repo}/commits/{commit['sha']}/pulls") or []: + if not pr.get("merged_at") or pr.get("merge_commit_sha") != commit["sha"]: + continue + found[pr["number"]] = { + "number": pr["number"], + "title": pr["title"], + "labels": [lb["name"] for lb in pr.get("labels", [])], + "sha": commit["sha"], + "merged_at": pr["merged_at"], + } + return list(found.values()) + + def engine_of(repo, kind, base, ref): """(engine version, how it is known). A final or post wheel links the engine it is named after.""" if kind != "dev": @@ -196,15 +231,21 @@ def build_notes(repo, tag): tags = [t["name"] for t in gh_pages(f"repos/{repo}/tags")] prev = previous_tag(tag, [t for t in tags if t != tag]) - milestone, prs = milestone_prs(repo, base) - + milestone = None listed = [] - for pr in prs: - if not is_ancestor(repo, pr["sha"], ref): - continue - if prev and is_ancestor(repo, pr["sha"], prev): - continue - listed.append(pr) + if kind == "post": + if prev: + listed = range_prs(repo, prev, ref) + else: + print(f"warning: no earlier tag of {base} for {tag}", file=sys.stderr) + else: + milestone, prs = milestone_prs(repo, base) + for pr in prs: + if not is_ancestor(repo, pr["sha"], ref): + continue + if prev and is_ancestor(repo, pr["sha"], prev): + continue + listed.append(pr) listed.sort(key=lambda p: p["number"]) groups = {title: [] for title, _ in SECTIONS} @@ -224,7 +265,7 @@ def build_notes(repo, tag): ) lines.append("") - if milestone is None: + if kind != "post" and milestone is None: lines.append( f"No milestone named `{base}` exists, so this release has no change list." ) @@ -232,6 +273,8 @@ def build_notes(repo, tag): lines.append("") else: scope = f"since `{prev}`" if prev else f"in the milestone `{base}`" + if kind == "post" and not prev: + scope = "since the previous release" lines.append(f"## What changed {scope}") lines.append("") if not listed: