diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 67fc5db0..f3e4c9d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,9 @@ name: Release to PyPI # Pushing a tag that is the bare version (26.11.1, 26.11.1.post1, 26.11.1.dev0; no "v", same as # ArcadeData/arcadedb) builds the wheels, tests them, checks them and publishes them to PyPI. -# Versions run A.B.C.devN -> A.B.C -> A.B.C.postN; never aN, bN or rcN. +# Versions run A.B.C.devN -> A.B.C -> A.B.C.postN; never aN, bN or rcN. devN starts at 0 and +# restarts at 0 on a new base A.B.C, postN starts at 1; scripts/check_version.py enforces the +# counting in the first job (a later native job passes --tag-prefix native-v for its own tags). # # PyPI keys a trusted publisher on this FILENAME and on the job's environment, so do not rename # either without changing the publisher on PyPI at the same time: @@ -39,9 +41,9 @@ env: DIST_NAME: arcadedb jobs: - # The tag must be a bare PEP 440 version of the allowed kinds. The jar-derived version is - # compared with the tag in verify-wheels, because the jars only exist after the test workflow - # downloaded them. + # The tag must be a bare PEP 440 version of the allowed kinds, and the next number of its base. + # The jar-derived version is compared with the tag in verify-wheels, because the jars only exist + # after the test workflow downloaded them. validate-version: name: Validate tag runs-on: ubuntu-24.04 @@ -76,6 +78,23 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Release $TAG (engine $BASE, image tag $IMAGE_TAG)" + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The counting rule (docs, Release page): devN from 0, postN from 1, never aN/bN/rcN, and the + # tag is the next number after the versions already in the git tags and on PyPI. The script + # only checks, and it fails closed: exit 1 is a tag that breaks the rule, exit 2 is a source + # (git tags, PyPI) it could not read, and either stops the release before anything is built. + # The tag being released is left out of the git tags; running the whole workflow again for + # a version that is already on PyPI fails here, so re-run only the publish job instead. + # Its tests are in tests/test_check_version.py. + - name: Tag must be the next version + env: + TAG: ${{ github.ref_name }} + run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME" + # Builds a wheel per platform and Python version and runs the unit tests on each. The called # workflow uploads the wheels as wheel--py artifacts. test: diff --git a/CLAUDE.md b/CLAUDE.md index 43b84b29..2a7cfb75 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,3 +70,5 @@ python -m bandit -c pyproject.toml -r examples --severity-level medium --confide ## CI `.github/workflows/test-python-bindings.yml` builds wheels for linux/amd64, linux/arm64, darwin/arm64, windows/amd64 and tests Python 3.10–3.14; also runs nightly against `arcadedata/arcadedb:latest`. Both workflows accept an `arcadedb-tag` input to pin the image. `test-python-examples.yml` runs `examples/`. A dependency-floor audit resolves extras with `--resolution lowest-direct` for each Python classifier and runs pip-audit, so raise lower bounds when a floor has an advisory. + +`.github/workflows/release.yml` publishes on a bare-version tag (`26.11.1.dev0`, `26.11.1`, `26.11.1.post1`; no `v`, never `aN`/`bN`/`rcN`). Its first job runs `scripts/check_version.py` (the devN/postN counting rule, read-only, fails closed); `scripts/after_tag_verify.sh` checks the finished release. The rule is stated once, on the docs page `development/release.adoc`. diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 9dccd62c..dff0f2fa 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -5,3 +5,5 @@ * How-to ** xref:how-to/databases-and-queries.adoc[Databases and queries] ** xref:how-to/concurrency.adoc[Concurrency, processes and crashes] +* Development +** xref:development/release.adoc[Release] diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc new file mode 100644 index 00000000..cada48b7 --- /dev/null +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -0,0 +1,65 @@ += Release +:description: How a release of the arcadedb package is numbered, tagged, checked and published to PyPI. + +A release is a Git tag. Pushing the tag starts the `release.yml` workflow. It builds the wheels, tests them, checks them and publishes them to PyPI with trusted publishing. The workflow stores no token. + +== Version numbering + +The tag is the bare version, with no `v`. The tag, the wheel version and the PyPI version are the same string. + +[cols="2,3"] +|=== +| Version | Meaning + +| `26.11.1.dev0`, `26.11.1.dev1`, ... +| A preview of engine release `26.11.1`, built from the upstream snapshot jars. `devN` starts at 0. + +| `26.11.1` +| The release. The wheels ship the `26.11.1` engine jar. + +| `26.11.1.post1`, `26.11.1.post2`, ... +| A fix in the Python code of the released `26.11.1`. The wheels still ship the `26.11.1` engine jar. `postN` starts at 1. +|=== + +The rule has five parts: + +* The order for one base `A.B.C` is `A.B.C.dev0`, `.dev1`, ..., then `A.B.C`, then `A.B.C.post1`, `.post2`, .... +* `aN`, `bN` and `rcN` are never used, previews included. +* `devN` starts again at 0 when the base `A.B.C` changes. +* A number is never reused and never skipped. PyPI refuses a version it has seen, even after a delete. The next number is one above the highest published number of that base. +* A `devN` of a released base is refused, because it would sort before the release. A `postN` of a base that is not released is refused. + +The base is the version of the engine. The workflow compares the tag with the version of the engine jar in the wheels, and it refuses a `devN` tag for a release jar and a release tag for a snapshot jar. + +== How the rule is enforced + +`scripts/check_version.py` checks the rule. It reads the published versions from the Git tags of the repository and from PyPI, and it never changes anything: + +[source,bash] +---- +python3 scripts/check_version.py 26.11.1.dev0 --git-remote origin --pypi arcadedb +---- + +It exits 0 when the version is the correct next one, 1 when it is not, and 2 when a source cannot be read. Exit 2 is a failure, not a pass: the check fails closed. + +The first job of `release.yml` runs the same command on the pushed tag. When it fails, the workflow builds and publishes nothing. Run it by hand before you push the tag. Running the whole workflow again for a version that is already on PyPI fails the check on purpose. To retry a failed upload, re-run only the publish job. + +The tests of the script are in `tests/test_check_version.py`. + +== After the tag push + +`scripts/after_tag_verify.sh` checks the finished release. It only reads from GitHub and PyPI: + +[source,bash] +---- +scripts/after_tag_verify.sh 26.11.1 26.10.1 +---- + +The second argument is the previous release. Use `-` for the first release of the package. The script checks: + +* The workflow run, and the conclusion of the publish job and of its upload step. +* The files on PyPI: one wheel per supported Python version and platform, each under 100 MB, none yanked. +* The sha256 of every file on PyPI against the wheels the workflow checked. +* A `pip download` for each of the four platforms, and a fresh install on Python 3.12 and 3.14 that opens a database and runs one SQL and one openCypher query. + +It prints `PASS` or `FAIL` for each step, and its exit code is the number of failures. diff --git a/scripts/after_tag_verify.sh b/scripts/after_tag_verify.sh new file mode 100755 index 00000000..675032b0 --- /dev/null +++ b/scripts/after_tag_verify.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# After the tag push: verify the release end to end. Read-only against GitHub and PyPI (no writes). +# Ported from humemai/arcadedb-embedded-python#479 and adapted to .github/workflows/release.yml. +# +# usage: after_tag_verify.sh [] [ ] +# example: after_tag_verify.sh 26.11.1 26.10.1 +# +# is the release whose PyPI file set this one must match in shape. Pass - when +# there is none (the first release of the distribution): the file set is then checked against +# pyproject.toml, one wheel per supported Python version and platform. +# +# Run it under bash (not zsh). Every step prints PASS or FAIL; the exit code is the number of FAILs. +# Set PIN=4-5 to pin the smoke runs to those cores. It downloads about 4 GB (the 20 checked wheels of the run and four from PyPI): TMPDIR or WORK should point at a disk, not a tmpfs. Needs: gh, curl, python3, uv. +set -uo pipefail +V="${1:?version}" +PREV="${2:?previous version}" +RUN="${3:-}" +ENG_SHA="${4:-}" +ENG_BUILD="${5:-}" +R=ArcadeData/arcadedb-python +DIST=arcadedb +W="${WORK:-$(mktemp -d "${TMPDIR:-$HOME}/after-tag-XXXXXX")}" +FAILS=0 +pass() { printf 'PASS %s\n' "$*"; } +fail() { + printf 'FAIL %s\n' "$*" + FAILS=$((FAILS + 1)) +} +mkdir -p "$W" +printf 'work dir: %s\n' "$W" + +# (a) the release workflow, and the publish JOB's own conclusion (not only the run's: read what the upload step did) +if [ -n "$RUN" ]; then + gh run view "$RUN" -R "$R" --json status,conclusion,headBranch,headSha,createdAt,updatedAt > "$W/run.json" 2>&1 || true + python3 - "$W/run.json" "$V" << 'EOF' || FAILS=$((FAILS + 1)) +import json,sys +d=json.load(open(sys.argv[1])) +ok = d.get("status")=="completed" and d.get("conclusion")=="success" and d.get("headBranch")==sys.argv[2] +print(("PASS" if ok else "FAIL"), " release run:", d.get("status"), d.get("conclusion"), "ref", d.get("headBranch"), "sha", (d.get("headSha") or "")[:12], d.get("createdAt"), "->", d.get("updatedAt")) +sys.exit(0 if ok else 1) +EOF + gh api "repos/$R/actions/runs/$RUN/jobs" --paginate --jq '.jobs[] | select(.name|startswith("Publish")) | [.name,.status,.conclusion,(.steps|map(select(.name=="Publish to PyPI"))|.[0].conclusion)]|@tsv' > "$W/publish-job.tsv" 2>&1 + cat "$W/publish-job.tsv" + if awk -F'\t' '$2=="completed" && $3=="success" && $4=="success"{ok=1} END{exit ok?0:1}' "$W/publish-job.tsv"; then pass "publish job and its 'Publish to PyPI' step both succeeded"; else fail "publish job or its upload step did not succeed"; fi +fi + +# (b) PyPI file set against the previous release: the same files as the previous release with the new version, minus wheels for Python versions the release dropped, each under 100 MB (decimal). With "-" as the previous version: one wheel per Python version of the pyproject.toml classifiers and per platform +curl -fsS "https://pypi.org/pypi/$DIST/$V/json" -o "$W/pypi-new.json" || fail "PyPI has no $V yet" +if [ "$PREV" = "-" ]; then + echo '{"urls": []}' > "$W/pypi-prev.json" +else + curl -fsS "https://pypi.org/pypi/$DIST/$PREV/json" -o "$W/pypi-prev.json" || fail "cannot read PyPI $PREV" +fi +python3 - "$W/pypi-new.json" "$W/pypi-prev.json" "$V" "$PREV" "$(dirname "$0")/../pyproject.toml" << 'EOF' || FAILS=$((FAILS + 1)) +import json,re,sys,tomllib +new,prev,v,pv=json.load(open(sys.argv[1])),json.load(open(sys.argv[2])),sys.argv[3],sys.argv[4] +project=tomllib.load(open(sys.argv[5],"rb"))["project"] +# A previous release may carry wheels for an interpreter this release dropped: they are not expected again. +floor=int(re.search(r">=\s*3\.(\d+)",project["requires-python"]).group(1)) +def kept(f): + m=re.search(r"-cp3(\d+)-",f) + return m is None or int(m.group(1))>=floor +want={u["filename"].replace(pv,v) for u in prev["urls"] if kept(u["filename"])} +have={u["filename"]:u for u in new["urls"]} +gaps=0 +if pv=="-": + # No release to compare with: one wheel per supported Python version and platform, as release.yml counts them. + pythons=[c.rsplit(".",1)[1] for c in project["classifiers"] if c.startswith("Programming Language :: Python :: 3.")] + plats=[r"manylinux.*x86_64",r"manylinux.*aarch64",r"macosx.*arm64",r"win_amd64"] + for py in pythons: + for plat in plats: + hits=[f for f in have if re.search(r"-cp3%s-"%py,f) and re.search(plat,f)] + want.update(hits) + if len(hits)!=1: + print("EXPECTED one wheel for Python 3.%s on %s, got %d"%(py,plat,len(hits))); gaps+=1 +missing=sorted(want-set(have)); extra=sorted(set(have)-want) +print("files on PyPI for %s: %d (previous release had %d)"%(v,len(have),len(prev["urls"]))) +for m in missing: print("MISSING",m) +for e in extra: print("EXTRA ",e) +big=[f for f,u in have.items() if u["size"]>=100_000_000] +yanked=[f for f,u in have.items() if u["yanked"]] +for f,u in sorted(have.items()): print(" %-62s %9.2f MB %s"%(f,u["size"]/1e6,u["digests"]["sha256"][:16])) +ok=not missing and not extra and not gaps and not big and not yanked and new["info"]["version"]==v +print(("PASS" if ok else "FAIL"),"file set as expected (previous release, or pyproject.toml without one), all under 100 MB, none yanked") +sys.exit(0 if ok else 1) +EOF + +# (c) sha256 of every file on PyPI against the checked wheels the workflow's verify job uploaded and the publish job sent +if [ -n "$RUN" ] && [ -s "$W/pypi-new.json" ]; then + mkdir -p "$W/artifacts" + gh run download "$RUN" -R "$R" -n release-dist -D "$W/artifacts" > /dev/null 2>&1 || fail "could not download the release-dist artifact of run $RUN (retention is 7 days)" + python3 - "$W/pypi-new.json" "$W/artifacts" << 'EOF' || FAILS=$((FAILS + 1)) +import json,sys,hashlib,pathlib +d=json.load(open(sys.argv[1])); art={} +for p in pathlib.Path(sys.argv[2]).rglob("*.whl"): + art[p.name]=hashlib.sha256(p.read_bytes()).hexdigest() +bad=0 +for u in d["urls"]: + a=art.get(u["filename"]) + if a is None: print("NO ARTIFACT for",u["filename"]); bad+=1 + elif a!=u["digests"]["sha256"]: print("SHA MISMATCH",u["filename"]); bad+=1 +print(("PASS" if not bad else "FAIL"),"%d PyPI files compared with %d workflow artifacts"%(len(d["urls"]),len(art))) +sys.exit(1 if bad else 0) +EOF +fi + +# (d) every platform resolves, and a fresh environment installs and works (3.12 and 3.14) +for plat in manylinux_2_34_x86_64 manylinux_2_34_aarch64 macosx_11_0_arm64 win_amd64; do + if uvx pip download --no-deps --only-binary=:all: --platform "$plat" --python-version 3.12 "$DIST==$V" -d "$W/dl-$plat" > "$W/dl-$plat.log" 2>&1; then pass "pip download $plat (cp312)"; else fail "pip download $plat: $(tail -1 "$W/dl-$plat.log")"; fi +done +for py in 3.12 3.14; do + [ -n "${SKIP_SMOKE:-}" ] && { + printf "SKIP smoke on Python %s (SKIP_SMOKE set)\n" "$py" + continue + } + # A throwaway `uv run` environment with the PyPI release; the fresh cache directory makes the + # install download from PyPI (what `--no-cache` did), and the smoke run below reuses it. + UVRUN=(uv run --no-project --cache-dir "$W/uv-cache-$py" --python "$py" --with "$DIST==$V") + "${UVRUN[@]}" python -c "import arcadedb" > "$W/install-$py.log" 2>&1 || { + fail "uv run --with $DIST==$V on $py: $(tail -1 "$W/install-$py.log")" + continue + } + ENG_SHA="$ENG_SHA" ENG_BUILD="$ENG_BUILD" V="$V" ${PIN:+taskset -c "$PIN"} "${UVRUN[@]}" python - << 'EOF' > "$W/smoke-$py.log" 2>&1 +import hashlib, os, pathlib, re, tempfile, zipfile +import arcadedb as adb +assert adb.__version__ == os.environ["V"], adb.__version__ +jars = pathlib.Path(adb.__file__).parent / "jars" +eng = sorted(jars.glob("arcadedb-engine-*.jar")) +# The engine jar is named after the engine, not the wheel: a Python-only hotfix X.Y.Z.postN ships the +# X.Y.Z jar, and a snapshot build X.Y.Z.devN ships the X.Y.Z-SNAPSHOT jar (docs, Release page). +engine = re.sub(r"\.dev[0-9]+$", "-SNAPSHOT", re.sub(r"\.post[0-9]+$", "", os.environ["V"])) +assert len(eng) == 1 and eng[0].name == "arcadedb-engine-%s.jar" % engine, eng +sha = hashlib.sha256(eng[0].read_bytes()).hexdigest() +props = zipfile.ZipFile(eng[0]).read("com/arcadedb/arcadedb.properties").decode() +build = [l.split("=", 1)[1].strip() for l in props.splitlines() if l.startswith("buildNumber")][0] +print("engine jar", eng[0].name, sha, "buildNumber", build) +if os.environ["ENG_SHA"]: assert sha == os.environ["ENG_SHA"], "engine jar is not the official one" +if os.environ["ENG_BUILD"]: assert build == os.environ["ENG_BUILD"], "buildNumber differs" +with tempfile.TemporaryDirectory() as d: + with adb.create_database(os.path.join(d, "smoke"), jvm_kwargs={"heap_size": "1g"}) as db: + db.command("sql", "CREATE VERTEX TYPE P") + with db.transaction(): + db.command("sql", "CREATE VERTEX P SET name = 'Ada'") + assert db.query("sql", "SELECT name FROM P").to_list()[0]["name"] == "Ada" + assert db.query("opencypher", "MATCH (n:P) RETURN n.name AS name").to_list()[0]["name"] == "Ada" +print("SMOKE OK", adb.__version__) +EOF + if [ $? -eq 0 ] && grep -q 'SMOKE OK' "$W/smoke-$py.log"; then + pass "fresh environment Python $py: install, import, SQL, openCypher, official engine jar" + grep 'engine jar' "$W/smoke-$py.log" + else + fail "smoke on Python $py (see $W/smoke-$py.log)" + tail -3 "$W/smoke-$py.log" + fi +done + +# There is no docs step: the Antora component is unversioned and builds in ArcadeData/arcadedb-docs, so a release changes no docs URL to check. + +printf '\n%d FAIL(s)\n' "$FAILS" +exit "$FAILS" diff --git a/scripts/check_version.py b/scripts/check_version.py new file mode 100755 index 00000000..45b17154 --- /dev/null +++ b/scripts/check_version.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Check that a release version follows the counting rule, and is the next one. + +The rule (docs page "Release", section "Version numbering"): + +- a version is `X.Y.Z`, `X.Y.Z.devN` or `X.Y.Z.postN`, written without leading zeros +- `devN` counts from 0, `postN` counts from 1 +- `aN`, `bN`, `rcN` and every other pre-release form are never used, previews included +- the order for one base is `X.Y.Z.dev0`, `.dev1`, ... then `X.Y.Z`, then `X.Y.Z.post1`, `.post2`, ... +- a number is never reused and never skipped: the next devN is one above the highest published + devN of that base (0 when there is none), and likewise for postN (1 when there is none) + +The same scheme applies to the native wheels (`26.10.2.dev0` is a valid native preview). + +This script only checks. It never tags, uploads or edits anything, and it fails closed: any +problem reading a published-versions source exits 2, not 0. + +usage: + check_version.py VERSION # syntax only + check_version.py VERSION --published-file FILE # one version per line, "-" is stdin + check_version.py VERSION --pypi arcadedb # versions on PyPI + check_version.py VERSION --git-remote origin [--tag-prefix native-v] # versions in git tags + +The sources can be combined. VERSION itself is left out of the git-tag list (the release +workflow runs after its own tag was pushed), but it counts as published when a file or PyPI +lists it. + +Exit codes: 0 correct next version, 1 not the correct next version, 2 a source could not be read. +Output is ASCII only. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess # nosec B404 - runs `git ls-remote` with a fixed argument list +import sys +import urllib.error +import urllib.request +from typing import Iterable, NamedTuple + +VERSION_RE = re.compile( + r"^(?P(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))" + r"(?:\.(?Pdev|post)(?P0|[1-9][0-9]*))?$" +) + + +class Version(NamedTuple): + base: str + kind: str # "dev", "post" or "" for the stable release + n: int # 0 for the stable release + + def __str__(self) -> str: + return f"{self.base}.{self.kind}{self.n}" if self.kind else self.base + + +class VersionError(Exception): + """The version breaks the rule.""" + + +def parse_version(text: str) -> Version: + """Parse a version, or raise VersionError naming the broken part of the rule.""" + match = VERSION_RE.match(text) + if not match: + raise VersionError( + f"{text!r} is not X.Y.Z, X.Y.Z.devN or X.Y.Z.postN " + "(no aN, bN, rcN, v prefix, leading zeros or other suffix)" + ) + kind = match.group("kind") or "" + n = int(match.group("n")) if kind else 0 + if kind == "post" and n < 1: + raise VersionError(f"{text!r}: postN starts at 1, not 0") + return Version(match.group("base"), kind, n) + + +def _parse_published(published: Iterable[str]) -> list[Version]: + parsed = [] + for text in published: + try: + parsed.append(parse_version(text)) + except VersionError: + # an old or foreign entry cannot take part in the counting + continue + return parsed + + +def check_next(version: str, published: Iterable[str]) -> Version: + """Return the parsed version when it is the correct next one, else raise VersionError.""" + wanted = parse_version(version) + seen = _parse_published(published) + if wanted in seen: + raise VersionError( + f"{wanted} is already published and a number is never reused" + ) + + of_base = [v for v in seen if v.base == wanted.base] + stable = any(v.kind == "" for v in of_base) + + if wanted.kind == "dev": + if stable: + raise VersionError( + f"{wanted}: {wanted.base} is already released, so a dev version of it would sort before it" + ) + highest = max((v.n for v in of_base if v.kind == "dev"), default=-1) + if wanted.n != highest + 1: + raise VersionError( + f"{wanted}: the next dev version of {wanted.base} is dev{highest + 1}" + ) + elif wanted.kind == "post": + if not stable: + raise VersionError( + f"{wanted}: {wanted.base} has not been released, so it cannot have a post version" + ) + highest = max((v.n for v in of_base if v.kind == "post"), default=0) + if wanted.n != highest + 1: + raise VersionError( + f"{wanted}: the next post version of {wanted.base} is post{highest + 1}" + ) + # a stable release has no counter; "already published" above is its only extra rule + return wanted + + +def pypi_versions(project: str) -> list[str]: + """Versions on PyPI, including ones with only yanked files. A project that does not exist yet has none.""" + url = f"https://pypi.org/pypi/{project}/json" + if not url.startswith("https://pypi.org/pypi/"): + raise OSError(f"refusing URL {url}") + try: + with urllib.request.urlopen( + url, timeout=30 + ) as response: # nosec B310 - fixed https PyPI URL + data = json.load(response) + except urllib.error.HTTPError as err: + if err.code == 404: + return [] + raise OSError(f"PyPI answered {err.code} for {project}") from err + except (urllib.error.URLError, TimeoutError, ValueError) as err: + raise OSError(f"cannot read PyPI data for {project}: {err}") from err + return list(data["releases"]) + + +def git_tag_versions(remote: str, prefix: str = "") -> list[str]: + """Tag names on a remote that start with prefix, with the prefix removed.""" + try: + out = subprocess.run( # nosec B603 B607 - fixed argument list + ["git", "ls-remote", "--tags", "--refs", remote], + check=True, + capture_output=True, + text=True, + timeout=60, + ).stdout + except (subprocess.SubprocessError, OSError) as err: + raise OSError(f"cannot list tags of {remote}: {err}") from err + names = [ + line.split("\t", 1)[1].removeprefix("refs/tags/") + for line in out.splitlines() + if "\t" in line + ] + return [name.removeprefix(prefix) for name in names if name.startswith(prefix)] + + +def read_lines(path: str) -> list[str]: + try: + text = ( + sys.stdin.read() if path == "-" else open(path, encoding="utf-8").read() + ) # noqa: SIM115 + except OSError as err: + raise OSError(f"cannot read {path}: {err}") from err + return [line.strip() for line in text.splitlines() if line.strip()] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("version") + parser.add_argument("--published-file", action="append", default=[], metavar="FILE") + parser.add_argument("--pypi", action="append", default=[], metavar="PROJECT") + parser.add_argument("--git-remote", metavar="REMOTE") + parser.add_argument( + "--tag-prefix", default="", help="tag prefix of the series, e.g. native-v" + ) + args = parser.parse_args(argv) + + try: + parse_version(args.version) + except VersionError as err: + print(f"FAIL: {err}") + return 1 + + published: list[str] = [] + try: + for path in args.published_file: + published += read_lines(path) + for project in args.pypi: + published += pypi_versions(project) + if args.git_remote: + # the tag being released is already pushed: it is not "published" before this check + published += [ + v + for v in git_tag_versions(args.git_remote, args.tag_prefix) + if v != args.version + ] + except OSError as err: + print(f"ERROR: {err}") + return 2 + + try: + wanted = check_next(args.version, published) + except VersionError as err: + print(f"FAIL: {err}") + return 1 + print( + f"OK: {wanted} is the correct next version ({len(published)} published versions read)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_check_version.py b/tests/test_check_version.py new file mode 100644 index 00000000..6d9d27ec --- /dev/null +++ b/tests/test_check_version.py @@ -0,0 +1,263 @@ +"""The version counting rule: X.Y.Z.devN from 0, X.Y.Z, X.Y.Z.postN from 1, never aN/bN/rcN (humemai/arcadedb-embedded-python#471).""" + +from __future__ import annotations + +import importlib.util +import re +import shutil +import subprocess # nosec B404 - test-controlled child processes, no untrusted input +import sys +from pathlib import Path + +import pytest + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "check_version.py" +PYPROJECT_FIELD = re.compile(r'^version = "([^"]+)"', re.MULTILINE) +RELEASE_WORKFLOW = SCRIPT_PATH.parents[1] / ".github" / "workflows" / "release.yml" + + +def _load(): + spec = importlib.util.spec_from_file_location("check_version", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules["check_version"] = module + spec.loader.exec_module(module) + return module + + +cv = _load() + + +@pytest.mark.parametrize( + "text", + [ + "26.11.1", + "26.11.1.dev0", + "26.11.1.dev12", + "26.5.1.post1", + "26.10.2.post7", + "26.10.2.dev0", + ], +) +def test_valid_forms(text): + assert str(cv.parse_version(text)) == text + + +@pytest.mark.parametrize( + "text", + [ + "26.10.2a1", + "26.10.2.a1", + "26.10.2b1", + "26.10.2rc1", + "26.10.2.rc1", + "26.10.2-rc1", + "26.10.2.post0", + "26.10.2.dev01", + "26.10.2.post01", + "26.10.2.dev", + "26.10.2.post", + "26.10.2.dev1.post1", + "26.10.2.post1.dev0", + "26.10", + "26.10.2.1", + "v26.10.2", + "026.10.2", + "26.10.2.DEV0", + "26.10.2+local", + "", + ], +) +def test_forbidden_forms(text): + with pytest.raises(cv.VersionError): + cv.parse_version(text) + + +def test_first_dev_is_zero(): + assert str(cv.check_next("26.11.1.dev0", [])) == "26.11.1.dev0" + with pytest.raises(cv.VersionError, match="dev0"): + cv.check_next("26.11.1.dev1", []) + + +def test_dev_increments_without_skipping(): + published = ["26.11.1.dev0", "26.11.1.dev1"] + assert str(cv.check_next("26.11.1.dev2", published)) == "26.11.1.dev2" + with pytest.raises(cv.VersionError, match="dev2"): + cv.check_next("26.11.1.dev3", published) + with pytest.raises(cv.VersionError, match="already published"): + cv.check_next("26.11.1.dev1", published) + + +def test_dev_after_a_gap_continues_from_the_highest(): + # the real history has gaps (26.8.1.dev16 and dev19 were never published); only the highest counts + assert ( + str(cv.check_next("26.8.1.dev28", ["26.8.1.dev25", "26.8.1.dev27"])) + == "26.8.1.dev28" + ) + + +def test_counters_are_per_base(): + published = ["26.8.1.dev9", "26.10.2.post3"] + assert str(cv.check_next("26.11.1.dev0", published)) == "26.11.1.dev0" + + +def test_stable_release(): + assert str(cv.check_next("26.11.1", ["26.11.1.dev0", "26.11.1.dev1"])) == "26.11.1" + with pytest.raises(cv.VersionError, match="already published"): + cv.check_next("26.11.1", ["26.11.1"]) + + +def test_dev_after_the_release_is_refused(): + with pytest.raises(cv.VersionError, match="already released"): + cv.check_next("26.11.1.dev5", ["26.11.1", "26.11.1.dev4"]) + + +def test_post_starts_at_one_and_needs_the_release(): + assert str(cv.check_next("26.10.2.post1", ["26.10.2"])) == "26.10.2.post1" + with pytest.raises(cv.VersionError, match="post1"): + cv.check_next("26.10.2.post2", ["26.10.2"]) + with pytest.raises(cv.VersionError, match="not been released"): + cv.check_next("26.10.2.post1", ["26.10.2.dev0"]) + + +def test_post_increments_without_skipping(): + published = ["26.10.2", "26.10.2.post1"] + assert str(cv.check_next("26.10.2.post2", published)) == "26.10.2.post2" + with pytest.raises(cv.VersionError, match="post2"): + cv.check_next("26.10.2.post3", published) + + +def test_native_series_restarts_at_dev0_on_a_new_base(): + # native 26.10.2.dev0 and dev1 are published under the old base (a naming mistake, they stay); + # the engine moved to 26.11.1-SNAPSHOT, so the next native preview is 26.11.1.dev0 + old = ["26.10.2.dev0", "26.10.2.dev1"] + assert str(cv.check_next("26.11.1.dev0", old)) == "26.11.1.dev0" + with pytest.raises(cv.VersionError, match="dev0"): + cv.check_next("26.11.1.dev1", old) + with pytest.raises(cv.VersionError, match="dev0"): + cv.check_next("26.11.1.dev2", old) + for form in ("26.11.1a1", "26.11.1b1", "26.11.1rc1", "26.11.1.post0"): + with pytest.raises(cv.VersionError): + cv.check_next(form, old) + + +def test_full_sequence_of_a_base(): + published = ["26.10.2.dev0", "26.10.2.dev1"] + for version in ("26.11.1.dev0", "26.11.1.dev1", "26.11.1", "26.11.1.post1"): + cv.check_next(version, published) + published.append(version) + with pytest.raises(cv.VersionError): + cv.check_next("26.11.1.post3", published) + + +def test_foreign_published_entries_are_ignored(): + assert ( + str(cv.check_next("26.11.1.dev0", ["latest", "26.11.1rc1", "26.11.1a1"])) + == "26.11.1.dev0" + ) + + +def test_existing_versions_pass_in_order(): + history = [ + "26.5.1", + "26.5.1.post1", + "26.7.2.dev0", + "26.7.2.dev1", + "26.7.2.dev2", + "26.8.1.dev0", + "26.10.1", + "26.10.2", + "26.10.2.post1", + ] + published: list[str] = [] + for version in history: + cv.check_next(version, published) + published.append(version) + + +def test_current_pyproject_version_is_syntactically_valid(): + # pyproject.toml carries a placeholder; build.sh and the release workflow write the real version + text = (SCRIPT_PATH.parents[1] / "pyproject.toml").read_text(encoding="utf-8") + cv.parse_version(PYPROJECT_FIELD.search(text).group(1)) + + +def _run(*args, stdin=None): + return subprocess.run( # nosec B603 - fixed argv (sys.executable + repo script), no shell + [sys.executable, str(SCRIPT_PATH), *args], + input=stdin, + capture_output=True, + text=True, + check=False, + ) + + +def test_cli_exit_codes(tmp_path): + listing = tmp_path / "published.txt" + listing.write_text("26.11.1.dev0\n26.11.1.dev1\n", encoding="utf-8") + assert _run("26.11.1.dev2", "--published-file", str(listing)).returncode == 0 + assert _run("26.11.1.dev3", "--published-file", str(listing)).returncode == 1 + assert _run("26.11.1rc1").returncode == 1 + assert _run("26.11.1.dev0").returncode == 0 + assert ( + _run("26.11.1.dev0", "--published-file", "-", stdin="26.11.1.dev0\n").returncode + == 1 + ) + + +def test_cli_fails_closed_when_a_source_cannot_be_read(tmp_path): + result = _run("26.11.1.dev0", "--published-file", str(tmp_path / "missing.txt")) + assert result.returncode == 2 + assert result.stdout.startswith("ERROR") + assert ( + _run( + "26.11.1.dev0", "--git-remote", str(tmp_path / "no-such-remote") + ).returncode + == 2 + ) + + +def test_git_remote_source_strips_prefix_and_leaves_the_version_itself_out(tmp_path): + remote = tmp_path / "remote" + git_bin = shutil.which("git") + # No skip: a runner without git has to fail here, not report a green run that tested nothing + assert git_bin is not None, "git is required for this test" + subprocess.run( + [git_bin, "init", "-q", str(remote)], check=True + ) # nosec B603 - fixed argv, no shell + git = [ + git_bin, + "-C", + str(remote), + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + ] + subprocess.run( + [*git, "commit", "-q", "--allow-empty", "-m", "x"], check=True + ) # nosec B603 - fixed argv, no shell + for tag in ("26.11.1.dev0", "26.11.1.dev1", "native-v26.11.1.dev0"): + subprocess.run( + [*git, "tag", tag], check=True + ) # nosec B603 - fixed argv, no shell + # the tag being released is already pushed: it must not count as published + assert _run("26.11.1.dev1", "--git-remote", str(remote)).returncode == 0 + assert _run("26.11.1.dev2", "--git-remote", str(remote)).returncode == 0 + assert _run("26.11.1.dev3", "--git-remote", str(remote)).returncode == 1 + native = ["--git-remote", str(remote), "--tag-prefix", "native-v"] + assert _run("26.11.1.dev1", *native).returncode == 0 + assert _run("26.11.1.dev2", *native).returncode == 1 + + +def test_release_workflow_checks_the_tag_before_anything_is_built(): + # The check has to sit in the first job: every other job needs it, so a tag that breaks the + # counting rule builds and publishes nothing. It reads git tags and PyPI for the one + # distribution this workflow publishes (DIST_NAME), and a read failure (exit 2) fails the job. + workflow = RELEASE_WORKFLOW.read_text(encoding="utf-8") + first_job = workflow.split("\n test:\n", 1)[0] + assert "name: Validate tag" in first_job + assert re.search( + r'python3 -I scripts/check_version\.py "\$TAG" --git-remote origin --pypi "\$DIST_NAME"', + first_job, + ) + assert re.search(r"^ DIST_NAME: arcadedb$", workflow, re.MULTILINE)