diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index d693a2e2d..a4ad833aa 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -121,6 +121,11 @@ jobs: verbose: true skip-existing: true + # NOTE: Publishing this release is what triggers the Zenodo archive. Zenodo's + # GitHub integration webhooks the 'release: published' event and mints a new + # version under the concept DOI in CITATION.cff -- no job, token, or API call + # of ours is involved. The integration only sees releases published *after* it + # was enabled, so it never backfills. publish-github-release: name: Publish GitHub release needs: publish-pypi @@ -133,52 +138,3 @@ jobs: uses: softprops/action-gh-release@v3 with: generate_release_notes: true - - # publish-zenodo: - # name: Publish Zenodo release - # needs: publish-github-release - # runs-on: ubuntu-latest - # if: github.event_name == 'push' - # permissions: - # contents: read - # env: - # ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }} - # steps: - # - uses: actions/checkout@v7 - # with: - # fetch-depth: 0 - # - # - uses: actions/setup-python@v6 - # with: - # python-version: "3.12" - # - # - name: Install release tooling - # run: | - # python -m pip install --upgrade pip - # python -m pip install PyYAML - # shell: bash - # - # - name: Download artifacts - # uses: actions/download-artifact@v8 - # with: - # name: dist-${{ github.sha }}-${{ github.run_id }}-${{ github.run_number }} - # path: dist - # - # - name: Generate release citation metadata - # run: | - # python tools/release/sync_citation.py \ - # --tag "${GITHUB_REF_NAME}" \ - # --output "${RUNNER_TEMP}/CITATION.cff" - # shell: bash - # - # - name: Check tree stayed clean - # run: | - # git diff --quiet || (git status --short && git diff && exit 1) - # shell: bash - # - # - name: Publish to Zenodo - # run: | - # python tools/release/publish_zenodo.py \ - # --dist-dir dist \ - # --citation-file "${RUNNER_TEMP}/CITATION.cff" - # shell: bash diff --git a/CITATION.cff b/CITATION.cff index 0622fecf2..0bed322e5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,8 +8,12 @@ authors: - family-names: "Becker" given-names: "Matthew R." orcid: "https://orcid.org/0000-0001-7774-2246" -date-released: "2026-03-30" -version: "2.1.5" +# NOTE: Deliberately no 'version' or 'date-released'. The package version comes +# from the git tag via setuptools_scm, and Zenodo's GitHub integration takes the +# version and publication date from the release itself. Pinning them here means +# hand-editing this file on every release, which is how it drifted to 2.1.5 while +# the project shipped 2.4.0. The DOI below is the *concept* DOI: it is stable +# across releases and always resolves to the latest version. doi: "10.5281/zenodo.15733564" repository-code: "https://github.com/Ultraplot/UltraPlot" license: "MIT" diff --git a/tools/release/publish_zenodo.py b/tools/release/publish_zenodo.py deleted file mode 100644 index 5b55af38e..000000000 --- a/tools/release/publish_zenodo.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -import sys -from pathlib import Path -from urllib import error, parse, request - -try: - import tomllib -except ModuleNotFoundError: # pragma: no cover - import tomli as tomllib - -try: - import yaml -except ImportError as exc: # pragma: no cover - exercised in release workflow - raise SystemExit( - "PyYAML is required to publish Zenodo releases. Install it before " - "running tools/release/publish_zenodo.py." - ) from exc - - -DEFAULT_API_URL = "https://zenodo.org/api" -DOI_PREFIX = "10.5281/zenodo." - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Publish the current UltraPlot release artifacts to Zenodo." - ) - parser.add_argument( - "--dist-dir", - type=Path, - default=Path("dist"), - help="Directory containing the built release artifacts.", - ) - parser.add_argument( - "--citation", - type=Path, - default=Path("CITATION.cff"), - help="Path to the repository CITATION.cff file.", - ) - parser.add_argument( - "--citation-file", - type=Path, - default=Path("CITATION.cff"), - help="Path to the citation metadata file to upload.", - ) - parser.add_argument( - "--pyproject", - type=Path, - default=Path("pyproject.toml"), - help="Path to the repository pyproject.toml file.", - ) - parser.add_argument( - "--api-url", - default=os.environ.get("ZENODO_API_URL", DEFAULT_API_URL), - help="Zenodo API base URL.", - ) - parser.add_argument( - "--access-token", - default=os.environ.get("ZENODO_ACCESS_TOKEN"), - help="Zenodo personal access token.", - ) - return parser.parse_args() - - -def load_citation(path: Path) -> dict: - with path.open("r", encoding="utf-8") as handle: - data = yaml.safe_load(handle) - if not isinstance(data, dict): - raise ValueError(f"{path} did not parse to a mapping") - return data - - -def load_pyproject(path: Path) -> dict: - with path.open("rb") as handle: - return tomllib.load(handle) - - -def author_to_creator(author: dict) -> dict: - family = author["family-names"].strip() - given = author["given-names"].strip() - creator = {"name": f"{family}, {given}"} - orcid = author.get("orcid") - if orcid: - creator["orcid"] = normalize_orcid(orcid) - return creator - - -def normalize_orcid(orcid: str) -> str: - return orcid.removeprefix("https://orcid.org/").rstrip("/") - - -def build_related_identifiers(citation: dict) -> list[dict]: - related = [] - repository = citation.get("repository-code", "").rstrip("/") - version = citation["version"] - if repository: - related.append( - { - "relation": "isSupplementTo", - "identifier": f"{repository}/tree/v{version}", - "scheme": "url", - "resource_type": "software", - } - ) - for reference in citation.get("references", []): - url = reference.get("url") - if not url: - continue - related.append( - { - "relation": "isDerivedFrom", - "identifier": url, - "scheme": "url", - } - ) - return related - - -def build_metadata(citation: dict, pyproject: dict) -> dict: - project = pyproject["project"] - creators = [author_to_creator(author) for author in citation["authors"]] - description = project["description"].strip() - repository = citation.get("repository-code") - if repository: - description = f"{description}\n\nSource code: {repository}" - metadata = { - "title": citation["title"], - "upload_type": "software", - "description": description, - "creators": creators, - "access_right": "open", - "license": citation.get("license"), - "keywords": citation.get("keywords", []), - "version": citation["version"], - "publication_date": citation["date-released"], - } - related = build_related_identifiers(citation) - if related: - metadata["related_identifiers"] = related - return metadata - - -def doi_record_id(doi: str) -> str: - value = doi.removeprefix("https://doi.org/").strip() - if not value.startswith(DOI_PREFIX): - raise ValueError( - f"Unsupported Zenodo DOI {doi!r}. Expected prefix {DOI_PREFIX!r}." - ) - return value.removeprefix(DOI_PREFIX) - - -def api_request( - method: str, - url: str, - *, - token: str | None = None, - json_data: dict | None = None, - data: bytes | None = None, - content_type: str | None = None, - expect_json: bool = True, -): - headers = {"Accept": "application/json"} - if token: - headers["Authorization"] = f"Bearer {token}" - body = data - if json_data is not None: - body = json.dumps(json_data).encode("utf-8") - headers["Content-Type"] = "application/json" - elif content_type: - headers["Content-Type"] = content_type - req = request.Request(url, data=body, headers=headers, method=method) - try: - with request.urlopen(req) as response: - payload = response.read() - except error.HTTPError as exc: - details = exc.read().decode("utf-8", errors="replace") - raise RuntimeError(f"{method} {url} failed with {exc.code}: {details}") from exc - if not expect_json: - return None - if not payload: - return None - return json.loads(payload) - - -def resolve_concept_recid(api_url: str, doi: str) -> str: - recid = doi_record_id(doi) - record = api_request("GET", f"{api_url}/records/{recid}") - return str(record.get("conceptrecid") or record.get("id") or recid) - - -def latest_record_id(api_url: str, conceptrecid: str) -> int: - query = parse.urlencode( - { - "q": f"conceptrecid:{conceptrecid}", - "all_versions": 1, - "sort": "mostrecent", - "size": 1, - } - ) - payload = api_request("GET", f"{api_url}/records?{query}") - hits = payload.get("hits", {}).get("hits", []) - if not hits: - raise RuntimeError( - f"Could not find any Zenodo records for conceptrecid {conceptrecid}." - ) - return int(hits[0]["id"]) - - -def create_new_version(api_url: str, token: str, record_id: int) -> dict: - response = api_request( - "POST", - f"{api_url}/deposit/depositions/{record_id}/actions/newversion", - token=token, - ) - latest_draft = response.get("links", {}).get("latest_draft") - if not latest_draft: - raise RuntimeError( - "Zenodo did not return links.latest_draft after requesting a new version." - ) - return api_request("GET", latest_draft, token=token) - - -def clear_draft_files(draft: dict, token: str) -> None: - files_url = draft.get("links", {}).get("files") - deposition_id = draft["id"] - if not files_url: - return - files = api_request("GET", files_url, token=token) or [] - for file_info in files: - file_id = file_info["id"] - api_request( - "DELETE", - f"{files_url}/{file_id}", - token=token, - expect_json=False, - ) - print(f"Deleted inherited Zenodo file {file_id} from draft {deposition_id}.") - - -def upload_dist_files(draft: dict, token: str, dist_dir: Path) -> None: - bucket_url = draft.get("links", {}).get("bucket") - if not bucket_url: - raise RuntimeError("Zenodo draft is missing the upload bucket URL.") - for path in sorted(dist_dir.iterdir()): - if not path.is_file(): - continue - with path.open("rb") as handle: - api_request( - "PUT", - f"{bucket_url}/{parse.quote(path.name)}", - token=token, - data=handle.read(), - # Zenodo bucket uploads reject extension-specific types like - # application/gzip for source tarballs and require raw bytes. - content_type="application/octet-stream", - ) - print(f"Uploaded {path.name} to Zenodo draft {draft['id']}.") - - -def update_metadata(draft: dict, token: str, metadata: dict) -> dict: - return api_request( - "PUT", - draft["links"]["self"], - token=token, - json_data={"metadata": metadata}, - ) - - -def publish_draft(draft: dict, token: str) -> dict: - return api_request("POST", draft["links"]["publish"], token=token) - - -def validate_inputs(dist_dir: Path, access_token: str | None) -> None: - if not access_token: - raise SystemExit( - "Missing Zenodo access token. Set ZENODO_ACCESS_TOKEN or pass " - "--access-token." - ) - if not dist_dir.is_dir(): - raise SystemExit(f"Distribution directory {dist_dir} does not exist.") - files = [path for path in dist_dir.iterdir() if path.is_file()] - if not files: - raise SystemExit(f"Distribution directory {dist_dir} does not contain files.") - - -def find_existing_draft(api_url: str, token: str, record_id: int) -> dict | None: - record = api_request("GET", f"{api_url}/records/{record_id}", token=token) - conceptrecid = str(record.get("conceptrecid") or record.get("id")) - - page = 1 - while True: - payload = api_request( - "GET", - f"{api_url}/deposit/depositions?page={page}&size=100", - token=token, - ) - - if not payload: - break - - for dep in payload: - if str(dep.get("conceptrecid")) != conceptrecid: - continue - if dep.get("submitted"): - continue - - print(f"Found existing draft deposition {dep['id']}") - return dep - - if len(payload) < 100: - break - - page += 1 - - return None - - -def get_or_create_draft(api_url: str, token: str, record_id: int) -> dict: - try: - return create_new_version(api_url, token, record_id) - except RuntimeError as exc: - message = str(exc) - if "files.enabled" not in message: - raise - - draft = find_existing_draft(api_url, token, record_id) - if draft is None: - raise - print(f"Reusing existing Zenodo draft {draft['id']}.") - return draft - - -def main() -> int: - args = parse_args() - validate_inputs(args.dist_dir, args.access_token) - - citation = load_citation(args.citation_file) - pyproject = load_pyproject(args.pyproject) - metadata = build_metadata(citation, pyproject) - - conceptrecid = resolve_concept_recid(args.api_url, citation["doi"]) - record_id = latest_record_id(args.api_url, conceptrecid) - - draft = get_or_create_draft(args.api_url, args.access_token, record_id) - clear_draft_files(draft, args.access_token) - upload_dist_files(draft, args.access_token, args.dist_dir) - draft = update_metadata(draft, args.access_token, metadata) - published = publish_draft(draft, args.access_token) - - doi = published.get("doi") or published.get("metadata", {}).get("doi") - print( - f"Published Zenodo release record {published['id']} for " - f"version {metadata['version']} ({doi})." - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/release/sync_citation.py b/tools/release/sync_citation.py deleted file mode 100644 index 3104ebe31..000000000 --- a/tools/release/sync_citation.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import re -import subprocess -from pathlib import Path - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Sync CITATION.cff release metadata from a git tag." - ) - parser.add_argument( - "--tag", - required=True, - help="Release tag to sync from, for example v2.1.5 or refs/tags/v2.1.5.", - ) - parser.add_argument( - "--citation", - type=Path, - default=Path("CITATION.cff"), - help="Path to the source CITATION.cff file.", - ) - parser.add_argument( - "--output", - type=Path, - default=None, - help=( - "Optional output path for the synced citation file. " - "Defaults to overwriting --citation." - ), - ) - parser.add_argument( - "--date", - help="Explicit release date in YYYY-MM-DD format. Defaults to the git tag date.", - ) - parser.add_argument( - "--check", - action="store_true", - help="Validate the file contents instead of writing them.", - ) - return parser.parse_args() - - -def normalize_tag(tag: str) -> str: - return tag.strip().removeprefix("refs/tags/") - - -def tag_version(tag: str) -> str: - tag = normalize_tag(tag) - if not tag.startswith("v"): - raise ValueError(f"Release tag must start with 'v', got {tag!r}.") - return tag.removeprefix("v") - - -def resolve_release_date(tag: str, repo_root: Path) -> str: - result = subprocess.run( - [ - "git", - "for-each-ref", - f"refs/tags/{normalize_tag(tag)}", - "--format=%(creatordate:short)", - ], - check=True, - cwd=repo_root, - capture_output=True, - text=True, - ) - value = result.stdout.strip() - if not value: - raise ValueError(f"Could not resolve a release date for tag {tag!r}.") - return value - - -def replace_scalar(text: str, key: str, value: str) -> str: - pattern = rf'^(?P{re.escape(key)}:\s*)"[^"]*"\s*$' - updated, count = re.subn( - pattern, - rf'\g"{value}"', - text, - count=1, - flags=re.MULTILINE, - ) - if count != 1: - raise ValueError(f"Missing quoted scalar {key!r} in CITATION metadata.") - return updated - - -def build_synced_citation( - citation_path: Path, - *, - tag: str, - release_date: str | None = None, - repo_root: Path | None = None, -) -> tuple[str, str, bool]: - repo_root = repo_root or citation_path.resolve().parent - version = tag_version(tag) - release_date = release_date or resolve_release_date(tag, repo_root) - - original = citation_path.read_text(encoding="utf-8") - updated = replace_scalar(original, "version", version) - updated = replace_scalar(updated, "date-released", release_date) - changed = updated != original - return original, updated, changed - - -def sync_citation( - citation_path: Path, - *, - tag: str, - output_path: Path | None = None, - release_date: str | None = None, - repo_root: Path | None = None, - check: bool = False, -) -> bool: - original, updated, changed = build_synced_citation( - citation_path, - tag=tag, - release_date=release_date, - repo_root=repo_root, - ) - - if check: - if changed: - raise SystemExit( - f"{citation_path} is out of date for {normalize_tag(tag)}. " - "Run tools/release/sync_citation.py before releasing." - ) - return False - - destination = output_path or citation_path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(updated, encoding="utf-8") - return changed - - -def main() -> int: - args = parse_args() - destination = args.output or args.citation - - changed = sync_citation( - args.citation, - tag=args.tag, - output_path=args.output, - release_date=args.date, - repo_root=Path.cwd(), - check=args.check, - ) - - if args.check: - print(f"Validated {args.citation} for {normalize_tag(args.tag)}.") - else: - source = args.citation.resolve() - dest = destination.resolve() - if source == dest: - print(f"Updated {args.citation} for {normalize_tag(args.tag)}.") - else: - print( - f"Wrote synced citation metadata from {args.citation} " - f"to {destination} for {normalize_tag(args.tag)}." - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/ultraplot/tests/test_release_metadata.py b/ultraplot/tests/test_release_metadata.py index f5af944c2..66842ab5e 100644 --- a/ultraplot/tests/test_release_metadata.py +++ b/ultraplot/tests/test_release_metadata.py @@ -1,143 +1,89 @@ from __future__ import annotations -import importlib.util import re from pathlib import Path -import pytest import yaml ROOT = Path(__file__).resolve().parents[2] CITATION_CFF = ROOT / "CITATION.cff" README = ROOT / "README.rst" PUBLISH_WORKFLOW = ROOT / ".github" / "workflows" / "publish-pypi.yml" -PYPROJECT = ROOT / "pyproject.toml" -SYNC_CITATION_SCRIPT = ROOT / "tools" / "release" / "sync_citation.py" -ZENODO_SCRIPT = ROOT / "tools" / "release" / "publish_zenodo.py" +# Concept DOI: stable across releases, always resolves to the newest version. +CONCEPT_DOI = "10.5281/zenodo.15733564" -def _citation_scalar(key): - """ - Extract a quoted top-level scalar from the repository CFF metadata. - """ - text = CITATION_CFF.read_text(encoding="utf-8") - match = re.search(rf'^{re.escape(key)}:\s*"([^"]+)"\s*$', text, re.MULTILINE) - assert match is not None, f"Missing {key!r} in {CITATION_CFF}" - return match.group(1) +def _citation(): + return yaml.safe_load(CITATION_CFF.read_text(encoding="utf-8")) -def _load_publish_zenodo(): - """ - Import the Zenodo release helper directly from the repo checkout. - """ - spec = importlib.util.spec_from_file_location("publish_zenodo", ZENODO_SCRIPT) - if spec is None or spec.loader is None: - raise ImportError(f"Could not load publish_zenodo from {ZENODO_SCRIPT}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - -def _load_sync_citation(): +def test_citation_pins_the_concept_doi(): """ - Import the citation sync helper directly from the repo checkout. + Citing the concept DOI keeps every release under one record. A versioned DOI + here would pin readers to whichever release happened to be current. """ - spec = importlib.util.spec_from_file_location("sync_citation", SYNC_CITATION_SCRIPT) - if spec is None or spec.loader is None: - raise ImportError(f"Could not load sync_citation from {SYNC_CITATION_SCRIPT}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + assert _citation()["doi"] == CONCEPT_DOI -def test_sync_citation_updates_release_metadata(tmp_path): - """ - Release automation should be able to sync CITATION.cff from a tag. - """ - sync_citation = _load_sync_citation() - citation = tmp_path / "CITATION.cff" - citation.write_text(CITATION_CFF.read_text(encoding="utf-8"), encoding="utf-8") - - changed = sync_citation.sync_citation( - citation, - tag="v9.9.9", - release_date="2030-01-02", - repo_root=ROOT, - ) - - text = citation.read_text(encoding="utf-8") - assert changed is True - assert 'version: "9.9.9"' in text - assert 'date-released: "2030-01-02"' in text - - -def test_zenodo_release_metadata_is_built_from_repository_sources(): - """ - Zenodo metadata should be derived from the maintained repository metadata. - """ - publish_zenodo = _load_publish_zenodo() - citation = yaml.safe_load(CITATION_CFF.read_text(encoding="utf-8")) - pyproject = publish_zenodo.load_pyproject(PYPROJECT) - metadata = publish_zenodo.build_metadata(citation, pyproject) - assert metadata["title"] == citation["title"] - assert metadata["upload_type"] == "software" - assert metadata["version"] == _citation_scalar("version") - assert metadata["publication_date"] == _citation_scalar("date-released") - assert metadata["creators"][0]["name"] == "van Elteren, Casper" - assert metadata["creators"][0]["orcid"] == "0000-0001-9862-8936" - - -def test_zenodo_uploads_use_octet_stream(tmp_path, monkeypatch): +def test_citation_does_not_pin_a_version(): """ - Zenodo bucket uploads should use a generic binary content type. + The git tag is the single source of truth for the version: setuptools_scm + derives the package version from it and Zenodo takes the archive version from + the release. Restating it here means editing this file on every release, which + is exactly how it silently drifted to 2.1.5 while the project shipped 2.4.0. """ - publish_zenodo = _load_publish_zenodo() - calls = [] - - def fake_api_request(method, url, **kwargs): - calls.append((method, url, kwargs)) - return None - - monkeypatch.setattr(publish_zenodo, "api_request", fake_api_request) - (tmp_path / "ultraplot-2.1.5.tar.gz").write_bytes(b"sdist") - (tmp_path / "ultraplot-2.1.5-py3-none-any.whl").write_bytes(b"wheel") - - publish_zenodo.upload_dist_files( - {"id": 18492463, "links": {"bucket": "https://zenodo.example/files/bucket"}}, - "token", - tmp_path, - ) - - assert len(calls) == 2 - assert all(method == "PUT" for method, _, _ in calls) - assert all( - kwargs["content_type"] == "application/octet-stream" for _, _, kwargs in calls - ) + citation = _citation() + assert "version" not in citation + assert "date-released" not in citation def test_zenodo_json_is_not_committed(): """ - Zenodo metadata should no longer be duplicated in a separate committed file. + Zenodo reads .zenodo.json in preference to CITATION.cff, so committing one + would reintroduce a second, competing copy of the release metadata. """ assert not (ROOT / ".zenodo.json").exists() def test_readme_citation_section_uses_repository_metadata(): - """ - The README should point readers at the maintained citation metadata. - """ + """The README should point readers at the maintained citation metadata.""" text = README.read_text(encoding="utf-8") assert "CITATION.cff" in text assert "@software{" not in text -def test_publish_workflow_creates_github_release(): +def test_publish_workflow_creates_github_release_on_tags(): """ - Release tags should sync citation metadata and create a GitHub release. + Publishing the GitHub release is what triggers the Zenodo archive, via + Zenodo's own webhook. If this step goes away, Zenodo silently stops updating. """ text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") assert 'tags: ["v*"]' in text - assert "tools/release/sync_citation.py" in text - assert "--tag" in text - assert "--output" in text assert "softprops/action-gh-release@" in text + + +def test_publish_workflow_has_no_zenodo_api_job(): + """ + Zenodo is synced by its GitHub integration, not by us. A second, API-based + path would race the webhook and mint duplicate versions under the concept DOI. + + NOTE: Match on uncommented lines only -- the previous job sat commented out in + this workflow for months, and a test asserting on the raw text passed against + dead YAML the whole time. + """ + live = [ + line + for line in PUBLISH_WORKFLOW.read_text(encoding="utf-8").splitlines() + if not re.match(r"^\s*#", line) + ] + text = "\n".join(live).lower() + assert "zenodo_access_token" not in text + assert "publish_zenodo" not in text + assert "sync_citation" not in text + + +def test_zenodo_release_tooling_is_gone(): + """The API-based release scripts are removed; nothing should resurrect them.""" + assert not (ROOT / "tools" / "release" / "publish_zenodo.py").exists() + assert not (ROOT / "tools" / "release" / "sync_citation.py").exists()