diff --git a/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml b/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml index 5fbd56593..c4e10102d 100644 --- a/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml +++ b/.github/ISSUE_TEMPLATE/3_robostack_bot.yaml @@ -19,6 +19,7 @@ body: | `update-vinca` | maintainers | Move the vinca pin in `pixi.toml` to the newest commit of its branch (or the ref given under Packages), lock and check. The PR lists the vinca commits. | | `dependency-report` | maintainers | Check the dependencies against the pins and update the distribution's dependency-report issue (closed when nothing conflicts). Runs weekly. | | `update-build-timings` | maintainers | Record the build times of the last week's build-branch runs in `shared/build_timings/`, which the build workflows use to pack each stage's recipes into jobs by time. Runs weekly. | + | `update-pixi` | maintainers | Upgrade the tooling's dependencies in `pixi.toml` / `pixi.lock` to their newest versions (`pixi upgrade`; not Python or the capped ones) and check. The PR lists the changes. Runs monthly. | On a pull request, `@robostack-bot rebuild-dependents` (maintainers) commits the rebuilds its ABI check suggests to the pull request's branch. - type: dropdown @@ -26,7 +27,7 @@ body: attributes: label: Command options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, update-vinca, - dependency-report, update-build-timings] + dependency-report, update-build-timings, update-pixi] validations: required: true - type: input diff --git a/.github/workflows/bot.yaml b/.github/workflows/bot.yaml index b350077e9..56a492369 100644 --- a/.github/workflows/bot.yaml +++ b/.github/workflows/bot.yaml @@ -10,13 +10,14 @@ # distribution (": dependency report", label dependency-report), closed when # nothing conflicts; and a PR with the build times the week's build-branch runs # recorded (shared/build_timings/, for packing the build jobs by time). +# - monthly (the 1st): a PR upgrading the tooling's pixi dependencies (pixi.toml / pixi.lock). # - on demand (owners, members and collaborators): # Actions > robostack-bot > Run workflow, an issue from the "robostack-bot command" # template, or a comment: @robostack-bot [] # commands: update-rosdistro-snapshot [...|all] | find-stale-packages [...|all] | # update-conda-forge-pinning [...|all] | add-package ... [...] | # rebuild ... [...|all] [--with-dependents] | update-vinca [] | -# dependency-report [...|all] | update-build-timings | +# dependency-report [...|all] | update-build-timings | update-pixi | # rebuild-dependents (a comment on a pull request: commits the rebuilds its ABI # check suggests to its branch; not possible for pull requests from forks) # - package requests (the "Package request" issue form or `@robostack-bot add-package`, @@ -31,12 +32,13 @@ on: schedule: - cron: "17 3 * * *" - cron: "43 4 * * 1" # weekly dependency report (the plan job matches this string) + - cron: "29 5 1 * *" # monthly pixi upgrade (the plan job matches this string) workflow_dispatch: inputs: command: type: choice options: [add-package, update-rosdistro-snapshot, find-stale-packages, update-conda-forge-pinning, rebuild, - update-vinca, dependency-report, update-build-timings] + update-vinca, dependency-report, update-build-timings, update-pixi] distro: description: Distribution (for the per-distribution commands; empty = all) required: false @@ -90,6 +92,10 @@ jobs: run: | case "$EVENT" in schedule) + if [ "$SCHEDULE" = "29 5 1 * *" ]; then + echo 'jobs=[{"command":"update-pixi","distro":"","args":"","preview":false}]' | tee -a "$GITHUB_OUTPUT" + exit 0 + fi if [ "$SCHEDULE" = "43 4 * * 1" ]; then # weekly: the dependency reports, and the build times of the week's builds jobs=$(ROBOSTACK_BOT_BODY="@robostack-bot dependency-report all" ROBOSTACK_BOT_ASSOCIATION=OWNER pixi run -q rs parse-command | @@ -166,6 +172,12 @@ jobs: update-vinca) pixi run rs update-vinca $ARGS --summary "$RUNNER_TEMP/summary.md" ;; # the build-branch runs' build-times artifacts (actions: read) update-build-timings) pixi run rs update-build-timings --summary "$RUNNER_TEMP/summary.md" ;; + # pixi upgrade (pixi.toml, pixi.lock) first, outside pixi run: it reinstalls the + # environment the command would run in + update-pixi) + # shellcheck disable=SC2046 # the --exclude arguments, one word each + pixi upgrade --json $(pixi run -q rs update-pixi --excludes) > "$RUNNER_TEMP/pixi-upgrade.json" && + pixi run rs update-pixi --changes "$RUNNER_TEMP/pixi-upgrade.json" --summary "$RUNNER_TEMP/summary.md" ;; # creates / updates the distribution's dependency-report issue dependency-report) pixi run rs "$DISTRO" dependency-report --summary "$RUNNER_TEMP/summary.md" ;; esac diff --git a/README.md b/README.md index 7e738a5b4..c3825a728 100644 --- a/README.md +++ b/README.md @@ -150,8 +150,9 @@ waiting for the macOS runners; remove them for a full build before merging. | `update-vinca []` | maintainers | Moves the vinca pin in `pixi.toml` to the newest commit of the pinned repository's branch (`robostack-integration` for Tobias-Fischer/vinca, else its default branch) or of ``, runs `pixi lock`, re-renders the pinning and runs `rs check`. The PR links the comparison and lists the vinca commits. Labelled `build:none` (see below). | | `dependency-report [... \| all]` | maintainers | Runs the dependency check (`check-deps`, linux-64) and keeps one issue per distribution, *``: dependency report* (label `dependency-report`), up to date with the conflicts, notes and conda-forge migration status; the issue is closed when nothing conflicts. Runs weekly (Mondays). | | `update-build-timings` | maintainers | Merges the build times the last week's build-branch runs recorded (`rs build-ci`, kept as each job's artifact) into `shared/build_timings/.json` and opens a PR. The build workflows pack each stage's recipes into jobs by these times. Runs weekly (Mondays). Labelled `build:none` (see below). | +| `update-pixi` | maintainers | Runs `pixi upgrade`: the newest versions of the tooling's dependencies in `pixi.toml` and `pixi.lock` (rattler-build, rattler-index, gh, pytest, ...), except Python (moved by hand) and requirements that are only an upper bound (`cmake = "<4.0"`: capped on purpose); the vinca pin is `update-vinca`'s. Runs `rs check`; the PR lists the changes as a table (pixi-diff-to-markdown). Runs monthly (on the 1st). Labelled `build:none` (see below). | -The PRs of `update-vinca` and `update-build-timings` don't change packages directly, but touch files outside `distros/`, which would build every distribution: they are labelled `build:none`, so only the checks run (and nothing is built while a build is under way). To build one, remove the label, or replace it with `build:` labels. +The PRs of `update-vinca`, `update-build-timings` and `update-pixi` don't change packages directly, but touch files outside `distros/`, which would build every distribution: they are labelled `build:none`, so only the checks run (and nothing is built while a build is under way). To build one, remove the label, or replace it with `build:` labels. There are three ways to trigger a command: - a *Package request* or *robostack-bot command* issue; diff --git a/tools/maintenance.py b/tools/maintenance.py index ed48dcb33..8c0f81d49 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -10,6 +10,7 @@ pixi run rs rebuild PKG... [DISTRO...] [--with-dependents] # per-package build-number bumps pixi run rs rebuild-dependents PR # commit the ABI check's suggested rebuilds to the PR pixi run rs update-vinca [REF] # move the vinca pin to the newest commit of its branch + pixi run rs update-pixi # `pixi upgrade` the tooling's dependencies pixi run rs dependency-report # dependency check, kept in a dependency-report issue pixi run rs parse-command --body TEXT --association ROLE # for @robostack-bot comments @@ -42,6 +43,7 @@ "rebuild": "new builds of ROS packages, optionally with everything depending on them (opens a PR)", "rebuild-dependents": "on a pull request: add the rebuilds its ABI check suggests (pushes to its branch)", "update-vinca": "move the vinca pin to the newest commit of its branch (opens a PR)", + "update-pixi": "upgrade the tooling's dependencies in pixi.toml / pixi.lock to their newest versions (opens a PR)", "dependency-report": "check the dependencies against the pins, in the distribution's dependency-report issue", "update-build-timings": "record the build times of the last week's build-branch runs, for batching by time (opens a PR)", } @@ -1058,6 +1060,65 @@ def update_vinca(ref: str | None = None) -> Result: return Result(f"Update vinca to {new[:10]}", "\n".join(lines), changed=True, labels=[rs.NO_BUILD_LABEL]) +def held_back(manifest: dict) -> list[str]: + """The dependencies update-pixi leaves alone: Python (a new minor version comes + before builds of everything the tooling needs, so it is moved by hand), and every + requirement that is only an upper bound, like `cmake = "<4.0"` (a deliberate cap).""" + tables = [manifest.get("dependencies", {})] + tables += [target.get("dependencies", {}) for target in manifest.get("target", {}).values()] + held = {"python"} + for table in tables: + for name, spec in table.items(): + version = spec if isinstance(spec, str) else spec.get("version", "") + if version.strip().startswith("<"): + held.add(name) + return sorted(held) + + +def pixi_upgrade_args() -> list[str]: + """`pixi upgrade` of everything except held_back().""" + import tomllib + + held = held_back(tomllib.loads((rs.ROOT / "pixi.toml").read_text())) + return ["pixi", "upgrade", "--json", *(a for name in held for a in ("--exclude", name))] + + +def update_pixi(changes: Path | None = None) -> Result: + """`pixi upgrade`: the newest versions of the tooling's dependencies in pixi.toml and + pixi.lock (except held_back()), with pixi-diff-to-markdown's table of the changes, + then the repository checks. vinca's pin is update-vinca's. + + changes: the output of an earlier `pixi upgrade --json` (`rs update-pixi --excludes` + gives its arguments). The bot upgrades before `pixi run`, so that the environment + isn't reinstalled under the running command; without it, this runs `pixi upgrade`.""" + import tomllib + + held = held_back(tomllib.loads((rs.ROOT / "pixi.toml").read_text())) + if changes is not None: + upgrade = subprocess.CompletedProcess([], 0, stdout=changes.read_text(), stderr="") + else: + print("pixi upgrade", flush=True) + upgrade = subprocess.run(pixi_upgrade_args(), cwd=rs.ROOT, capture_output=True, text=True) + if upgrade.returncode: + return Result("update-pixi: pixi upgrade failed", f"```\n{tail(upgrade.stdout + upgrade.stderr, 40)}\n```", ok=False) + kept = ", ".join(f"`{name}`" for name in held) + if subprocess.run(["git", "diff", "--quiet", "--", "pixi.toml", "pixi.lock"], cwd=rs.ROOT).returncode == 0: + return Result("pixi dependencies are up to date", f"`pixi upgrade` found nothing newer (held back: {kept}).") + table = subprocess.run(["pixi", "exec", "pixi-diff-to-markdown"], input=upgrade.stdout, + cwd=rs.ROOT, capture_output=True, text=True) + changes = table.stdout.strip() if table.returncode == 0 and table.stdout.strip() else ( + f"```json\n{tail(upgrade.stdout, 80)}\n```") + lines = [f"`pixi upgrade`: the newest versions of the tooling's dependencies in `pixi.toml` and `pixi.lock`. " + f"Held back: {kept} (Python is moved by hand, the others are capped on purpose); " + "the vinca pin is `update-vinca`'s.", "", changes, ""] + print("pixi run rs check", flush=True) + checked = subprocess.run(["pixi", "run", "rs", "check"], cwd=rs.ROOT, capture_output=True, text=True) + lines.append("`pixi run rs check`: " + ("✅ ok" if checked.returncode == 0 else + f"❌ problems:\n\n```\n{tail(checked.stdout + checked.stderr, 40)}\n```")) + lines += ["", NO_BUILD_NOTE.format(what="the tooling's dependencies; a new rattler-build is worth a build")] + return Result("Upgrade pixi dependencies", "\n".join(lines), changed=True, labels=[rs.NO_BUILD_LABEL]) + + REPORT_LABEL = "dependency-report" REPORT_MARKER = "" MAX_ISSUE_BODY = 60000 # GitHub's limit is 65536 characters @@ -1382,6 +1443,9 @@ def main(command: str, argv: list[str], distro: str | None = None) -> int: parser.add_argument("--with-dependents", action="store_true", help="also every package depending on them") if command == "rebuild-dependents": parser.add_argument("pull_request") + if command == "update-pixi": + parser.add_argument("--changes", type=Path, help="output of an earlier `pixi upgrade --json`") + parser.add_argument("--excludes", action="store_true", help="print the --exclude arguments for pixi upgrade") if command == "update-vinca": parser.add_argument("ref", nargs="?", help="vinca branch, tag or commit (default: the pinned branch)") args = parser.parse_args(argv) @@ -1401,6 +1465,11 @@ def main(command: str, argv: list[str], distro: str | None = None) -> int: return report(dependency_report(distro), args.summary) if command == "update-build-timings": return report(update_build_timings(), args.summary) + if command == "update-pixi": + if args.excludes: + print(" ".join(pixi_upgrade_args()[3:])) + return 0 + return report(update_pixi(args.changes), args.summary) if command == "add-package": known = set(rs.distros()) distros = [n for n in args.names if n in known] diff --git a/tools/robostack.py b/tools/robostack.py index 8b723dfa9..fe76b41e4 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -692,7 +692,7 @@ def changed_distros(base: str) -> list[str]: REPO_COMMANDS = ("check", "update-pinning", "new-distro", "parse-command", "add-package", "rosdistro-syncs", - "rebuild", "rebuild-dependents", "update-vinca", "update-build-timings") + "rebuild", "rebuild-dependents", "update-vinca", "update-build-timings", "update-pixi") def main() -> int: diff --git a/tools/tests/test_bot_commands.py b/tools/tests/test_bot_commands.py index 25002538f..ae9c55b0d 100644 --- a/tools/tests/test_bot_commands.py +++ b/tools/tests/test_bot_commands.py @@ -1,3 +1,4 @@ +import subprocess """robostack-bot: command parsing and the pure helpers of rebuild, rebuild-dependents, update-vinca and dependency-report (no network).""" @@ -317,3 +318,42 @@ def test_uploads_retry_five_times_waiting_longer(monkeypatch): monkeypatch.setattr(rs.time, "sleep", waits.append) assert rs.retry(["upload"], rs.ROOT, "anaconda.org") == 0 assert waits == [30, 60, 120, 240] + + +def test_update_pixi_parsing(): + assert jobs("@robostack-bot update-pixi") == [{"command": "update-pixi", "distro": "", "args": "", "preview": False}] + assert maintenance.parse_command("@robostack-bot update-pixi", "NONE") == [] + + +def test_update_pixi_holds_back_python_and_caps(): + manifest = {"dependencies": {"python": ">=3.14,<3.15", "cmake": "<4.0", "gh": ">=2.1,<3"}, + "target": {"win-64": {"dependencies": {"m2-patch": "*", "foo": {"version": " <2"}}}}} + assert maintenance.held_back(manifest) == ["cmake", "foo", "python"] + import tomllib + assert maintenance.held_back(tomllib.loads((rs.ROOT / "pixi.toml").read_text())) == ["cmake", "python", "setuptools"] + + +def test_update_pixi_without_changes_opens_no_pr(monkeypatch): + calls = [] + + def run(cmd, **kwargs): + calls.append(cmd) + return subprocess.CompletedProcess(cmd, 0, stdout="{}", stderr="") + + monkeypatch.setattr(maintenance.subprocess, "run", run) + result = maintenance.update_pixi() + assert result.ok and not result.changed + assert calls[0] == ["pixi", "upgrade", "--json", "--exclude", "cmake", "--exclude", "python", "--exclude", "setuptools"] + + +def test_update_pixi_takes_an_earlier_upgrade(monkeypatch, tmp_path): + calls = [] + + def run(cmd, **kwargs): + calls.append(cmd) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + monkeypatch.setattr(maintenance.subprocess, "run", run) + (tmp_path / "upgrade.json").write_text("{}") + assert not maintenance.update_pixi(tmp_path / "upgrade.json").changed + assert all(cmd[:2] != ["pixi", "upgrade"] for cmd in calls) # the bot upgraded before `pixi run`