From f4702cb0a4a764755582734b4622e1f8006edbfb Mon Sep 17 00:00:00 2001 From: Paul Meyer Date: Wed, 11 Nov 2026 14:38:09 +0100 Subject: [PATCH 1/2] bin: use env in host shebangs Signed-off-by: Paul Meyer --- bin/tine | 2 +- tools/ci.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/tine b/bin/tine index a0eba943..d8bc5cdf 100755 --- a/bin/tine +++ b/bin/tine @@ -1,4 +1,4 @@ -#!/usr/bin/python3 -SIB +#!/usr/bin/env -S python3 -SIB # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 diff --git a/tools/ci.sh b/tools/ci.sh index 7549ade7..7f3d1586 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 From 5e4f211e6805a0fc21213d8043137f0df6de6952 Mon Sep 17 00:00:00 2001 From: Paul Meyer Date: Wed, 11 Nov 2026 14:39:10 +0100 Subject: [PATCH 2/2] python: add use_host_python option would be nice to have a hermetic way, but as long as we are downloading dynamically linked stuff that must run on the host we need a way to work around it. Signed-off-by: Paul Meyer --- BUCK | 4 +- README.md | 23 +++++++- bin/tine.py | 51 +++++++++++++++- bin/tine_test.py | 67 +++++++++++++++++++++ python/BUCK | 2 +- python/defs.bzl | 51 ++++++++++++++++ tests/BUCK | 20 +++++++ tests/host_python.py | 115 +++++++++++++++++++++++++++++++++++++ tests/host_python_probe.py | 19 ++++++ tools/ci.sh | 2 + 10 files changed, 347 insertions(+), 7 deletions(-) create mode 100644 tests/host_python.py create mode 100644 tests/host_python_probe.py diff --git a/BUCK b/BUCK index 59312513..71c9a9bb 100644 --- a/BUCK +++ b/BUCK @@ -3,14 +3,12 @@ """Shared tine Python support, and the toolchain Buck2 looks up as `toolchains//`.""" -load("@prelude//toolchains:python.bzl", "python_bootstrap_toolchain") load("//python:defs.bzl", "python") # Declared here rather than in a `toolchains` cell of its own: Buck2 forbids nesting a cell inside an # external cell, so declaring it here is what lets a consuming project pull tine in as one. -python_bootstrap_toolchain( +python.bootstrap_toolchain( name = "python_bootstrap", - interpreter = read_config("tine", "python_interpreter", "tine//tools:python3"), visibility = ["PUBLIC"], ) diff --git a/README.md b/README.md index 610b7134..7d9ecc8a 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,9 @@ The host needs only: [sysctl](https://www.freedesktop.org/software/systemd/man/latest/sysctl.d.html): `kernel.apparmor_restrict_unprivileged_userns=0`. * A host root that supports `pivot_root()`; some kernels cannot pivot an initramfs root - * glibc ≥ 2.17, which the pinned Python that starts every sandbox links against - * `/usr/bin/python3` ≥ 3.14, or ≥ 3.12 together with `zstd` + * glibc ≥ 2.17, which the pinned Python that starts every sandbox links against, unless the project + [uses the host's Python](#hosts-without-a-glibc-loader) + * `python3` ≥ 3.14 on `PATH`, or ≥ 3.12 together with `zstd` * `git` * `/bin/sh`, which runs the fetch of a Rust project's git dependencies * `/dev/kvm` to run VMs for testing; without it, you can still build everything @@ -33,6 +34,24 @@ The first [`tine`](bin/tine) invocation that runs Buck fetches and verifies the `${XDG_CACHE_HOME:-~/.cache}/tine/buck2` (one directory per pin, safe to delete); everything else is fetched and cached by Buck itself. +### Hosts without a glibc loader + +The pinned Python that runs tine's drivers on the host needs a glibc loader at the standard path, such as +`/lib64/ld-linux-x86-64.so.2`. On a host without one, such as NixOS or a musl-based distribution, set this +in the project's configuration, for example in `.buckconfig.local`: + +```ini +[tine] +use_host_python = true +``` + +`tine` then runs the drivers with the `python3` on `PATH`, which must be 3.14 or newer, and fails before +the build if it is not. It runs that interpreter isolated from `PYTHONPATH`, `sitecustomize` and the user's +site directory. A driver that runs in a box still uses the pinned Python, on the box's own loader. Every +action names the host interpreter's path and build, so upgrading it reruns the actions. That also keys +them apart from the pinned interpreter's, so a shared cache filled by builds with it does not serve them. +The setting replaces `tine.python_interpreter`, and the two cannot be combined. + ### HOME directory Buck2 refuses to start without `$HOME` (some CI environments don't set it), as it runs its daemon from diff --git a/bin/tine.py b/bin/tine.py index aad713d0..68af5200 100644 --- a/bin/tine.py +++ b/bin/tine.py @@ -63,6 +63,23 @@ RE_CLIENT = "buck2_re_client" # Whether the configured cache takes uploads, which the execution platform reads back. CACHE_UPLOADS = "cache-uploads" +# A project that sets USE_HOST_PYTHON gets the host's python3 resolved into HOST_PYTHON and +# HOST_PYTHON_IDENTITY, which python/defs.bzl reads back. +USE_HOST_PYTHON = "use_host_python" +HOST_PYTHON = "host-python" +HOST_PYTHON_IDENTITY = "host-python-identity" +# The drivers decompress RPM payloads with compression.zstd, which arrived in 3.14, so importing it is +# the version check. The digest of sys.version tells apart two builds of one version. +_HOST_PYTHON_PROBE = """\ +import hashlib, os, sys +import compression.zstd +print(os.path.realpath(sys.executable)) +build = hashlib.sha256(sys.version.encode()).hexdigest()[:12] +print("{}.{}.{}".format(*sys.version_info[:3]), build, sep="-") +""" +# python/defs.bzl runs the host's python3 with these flags, and so does the probe, to see what the +# build sees. +HOST_PYTHON_FLAGS = ("-E", "-s", "-S") PLATFORMS = ("Linux-aarch64", "Linux-riscv64", "Linux-x86_64") @@ -70,7 +87,10 @@ DEFAULT_PLATFORM = f"{CELL}//platform:default" BLOCK_BEGIN = "# @generated by `tine`; rewritten on every command." -BLOCK_NOTE = "# Buck reads the version, dev targets, mount ignores and cache settings; tine reads its TOML." +BLOCK_NOTE = ( + "# Buck reads the version, dev targets, mount ignores, cache settings and host python; " + "tine reads its TOML." +) BLOCK_END = "# @end generated by `tine`; what follows is yours." SHELLS = ("bash", "fish", "zsh") @@ -587,6 +607,34 @@ def dev_targets(root: Path, config: dict[str, dict[str, str]], mounts: list[str] return list(dict.fromkeys([entry for entry in configured if entry] + sorted(mounts))) +def host_python(root: Path) -> dict[str, str]: + """Resolve the host's python3 for the bootstrap toolchain, if the project sets `use_host_python`. + + A bare `python3` on a command line would let Buck reuse outputs across a host upgrade. Its + resolved path and build identity go on every command line instead, so an upgrade reruns the + actions. + """ + if read_project_buckconfig(root).get(SECTION, {}).get(USE_HOST_PYTHON) != "true": + return {} + import shutil + import subprocess + + setting = f"[{SECTION}] {USE_HOST_PYTHON}" + python = shutil.which("python3") + if python is None: + fail(f"{setting} is set, but there is no python3 on PATH") + proc = subprocess.run( + [python, *HOST_PYTHON_FLAGS, "-c", _HOST_PYTHON_PROBE], capture_output=True, text=True + ) + if proc.returncode != 0: + fail( + f"{setting} is set, but {python} cannot run the build, which needs python 3.14 or newer " + f"with compression.zstd:\n{proc.stderr.strip()}" + ) + path, identity = proc.stdout.splitlines() + return {HOST_PYTHON: path, HOST_PYTHON_IDENTITY: identity} + + def render_local_config_block( root: Path, ignores: list[str], @@ -600,6 +648,7 @@ def render_local_config_block( lines.append(f"# no version components: {derived}") else: own |= {f"version-{key}": value for key, value in derived.items()} + own |= host_python(root) if dev: own[DEV] = ", ".join(dev) if cache is not None: diff --git a/bin/tine_test.py b/bin/tine_test.py index 9aae095d..46882be7 100644 --- a/bin/tine_test.py +++ b/bin/tine_test.py @@ -392,6 +392,73 @@ def test_the_address_reads_back_out_of_the_block(self) -> None: self.assertNotIn(tine.RE_CLIENT, tine.read_project_buckconfig(self.repo)) +class TestHostPython(unittest.TestCase): + """`use_host_python` resolves the python3 on PATH into the generated block.""" + + @override + def setUp(self) -> None: + self.root = scratch(self) + self.path = scratch(self, "tine-test-path.") + patched = unittest.mock.patch.dict(os.environ, {"PATH": str(self.path)}) + patched.start() + self.addCleanup(patched.stop) + + def use_host_python(self) -> None: + (self.root / tine.LOCAL).write_text(f"[{tine.SECTION}]\n{tine.USE_HOST_PYTHON} = true\n") + + def python3(self, script: str | None = None) -> None: + """Put this test's interpreter on PATH as python3, or a shell script standing in for one.""" + command = self.path / "python3" + if script is None: + command.symlink_to(sys.executable) + else: + command.write_text(f"#!/bin/sh\n{script}\n") + command.chmod(0o755) + + def test_nothing_without_the_setting(self) -> None: + self.python3() + self.assertEqual(tine.host_python(self.root), {}) + + def test_resolves_the_interpreter_and_its_build(self) -> None: + self.use_host_python() + self.python3() + resolved = tine.host_python(self.root) + self.assertEqual(resolved[tine.HOST_PYTHON], os.path.realpath(sys.executable)) + version = re.escape("{}.{}.{}".format(*sys.version_info[:3])) + self.assertRegex(resolved[tine.HOST_PYTHON_IDENTITY], rf"^{version}-[0-9a-f]{{12}}$") + + def test_lands_in_the_generated_block(self) -> None: + self.use_host_python() + self.python3() + tine.refresh_local_buckconfig(self.root, []) + generated = read_generated_buckconfig(self.root / tine.LOCAL)[tine.SECTION] + self.assertEqual(generated[tine.HOST_PYTHON], os.path.realpath(sys.executable)) + self.assertIn(tine.HOST_PYTHON_IDENTITY, generated) + # The setting itself stays the developer's. + project = tine.read_project_buckconfig(self.root)[tine.SECTION] + self.assertEqual(project, {tine.USE_HOST_PYTHON: "true"}) + + def test_no_python3_on_path(self) -> None: + self.use_host_python() + with self.assertRaisesRegex(SystemExit, "no python3 on PATH"): + tine.host_python(self.root) + + def test_an_interpreter_without_compression_zstd_is_rejected(self) -> None: + self.use_host_python() + self.python3("echo \"ModuleNotFoundError: No module named 'compression'\" >&2; exit 1") + rejected = "needs python 3.14 or newer(.|\n)*No module named 'compression'" + with self.assertRaisesRegex(SystemExit, rejected): + tine.host_python(self.root) + + def test_runs_the_probe_isolated_from_the_host(self) -> None: + # The build runs the interpreter with these flags, so the probe sees what the build will. + self.use_host_python() + self.python3() + with unittest.mock.patch.dict(os.environ, {"PYTHONPATH": str(scratch(self))}): + (Path(os.environ["PYTHONPATH"]) / "hashlib.py").write_text("raise ImportError('shadowed')\n") + self.assertIn(tine.HOST_PYTHON, tine.host_python(self.root)) + + class TestMergeGeneratedConfigBlock(unittest.TestCase): """The block is this command's; the rest of the file stays the developer's.""" diff --git a/python/BUCK b/python/BUCK index c5e8269e..cf2f6837 100644 --- a/python/BUCK +++ b/python/BUCK @@ -16,7 +16,7 @@ python.test( deps = ["//archive:cpio"], env = { "PYTHON_TEST_INPUT": "$(location :test-input)", - "PYTHON_TEST_INTERPRETER": "$(location_exec //tools:python3)", + "PYTHON_TEST_INTERPRETER": python.bootstrap_interpreter(), "PYTHON_TEST_VALUE": "spaces 'quotes' \"double quotes\" $literal;", }, ) diff --git a/python/defs.bzl b/python/defs.bzl index 7a788383..790b1fd6 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -9,6 +9,7 @@ load("@prelude//decls:toolchains_common.bzl", "toolchains_common") load("@prelude//python_bootstrap:python_bootstrap.bzl", "PythonBootstrapSources") load("@prelude//python_bootstrap:toolchain.bzl", "PythonBootstrapToolchainInfo") load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info") +load("@prelude//toolchains:python.bzl", "python_bootstrap_toolchain") load("@prelude//utils:selects.bzl", "selects") load("//box:defs.bzl", "box") @@ -204,8 +205,58 @@ def _test( _python_test(name = name, box = box, srcs = srcs, deps = deps, **kwargs) ty_check(name = name + "-ty", srcs = srcs, deps = deps, boxes = selects.apply(box, lambda value: [] if value == None else [value])) +def _host_python() -> list[str] | None: + """The command of the host's python3 that `tine.use_host_python` selects, or None for the pinned one. + + bin/tine resolves the interpreter into the generated config, with an identity of its build. + """ + if read_root_config("tine", "use_host_python") != "true": + return None + path = read_root_config("tine", "host-python") + identity = read_root_config("tine", "host-python-identity") + if path == None or identity == None: + fail("tine.use_host_python is set, but bin/tine has not resolved the host's python3; run Buck through `tine buck`") + + # -E, -s and -S keep the host's PYTHON* variables, user site directory and sitecustomize out of + # the build. Python only records an -X option it does not know, so this one puts the + # interpreter's build on every command line and an upgrade reruns the actions. + return [path, "-B", "-E", "-s", "-S", "-X", "tine-host-python=" + identity] + +def _host_python_toolchain_impl(ctx: AnalysisContext) -> list[Provider]: + return [DefaultInfo(), PythonBootstrapToolchainInfo(interpreter = cmd_args(ctx.attrs.command))] + +_host_python_toolchain = rule( + impl = _host_python_toolchain_impl, + attrs = {"command": attrs.list(attrs.string())}, + is_toolchain_rule = True, +) + +def _bootstrap_toolchain(name: str, **kwargs) -> None: + """Declare the bootstrap toolchain on the pinned interpreter, or on the host's `python3` if asked to. + + The pinned interpreter needs a glibc loader at the standard path. A host without one sets + `tine.use_host_python = true` in the project's config. + """ + host = _host_python() + if host == None: + python_bootstrap_toolchain(name = name, interpreter = read_config("tine", "python_interpreter", "tine//tools:python3"), **kwargs) + elif read_config("tine", "python_interpreter") != None: + fail("tine.use_host_python and tine.python_interpreter both choose the bootstrap interpreter; set only one") + else: + _host_python_toolchain(name = name, command = host, **kwargs) + +def _bootstrap_interpreter() -> str: + """The path of the interpreter the bootstrap toolchain runs, for an attribute that expands macros. + + ty only reads the pinned interpreter's tree and never runs it, so its checks keep that one. + """ + host = _host_python() + return "$(location_exec tine//tools:python3)" if host == None else host[0] + python = struct( binary = _binary, + bootstrap_interpreter = _bootstrap_interpreter, + bootstrap_toolchain = _bootstrap_toolchain, flat_tree = flat_tree, library = _library, test = _test, diff --git a/tests/BUCK b/tests/BUCK index 483c30ac..620499d9 100644 --- a/tests/BUCK +++ b/tests/BUCK @@ -40,6 +40,26 @@ python.binary( deps = ["//:util"], ) +# The bootstrap toolchain on the host's python3 (`tine.use_host_python`), in an isolation dir of its +# own. It runs nested `tine buck` commands, so tools/ci.sh runs it as a group instead of as a test suite. +python.binary( + name = "host-python-driver", + main = "host_python.py", + deps = ["//:util"], +) + +command_alias( + name = "host-python", + args = ["--fallback", "$(location_exec //tools:python3)"], + exe = ":host-python-driver", +) + +# What the driver above runs under the toolchain it switched. +python.binary( + name = "host-python-probe", + main = "host_python_probe.py", +) + # Assert an artifact mentions what it should. Shared because "these things must appear in that # file" is the same test whether the file is an SBOM or a module manifest. export_file( diff --git a/tests/host_python.py b/tests/host_python.py new file mode 100644 index 00000000..7b53d4f4 --- /dev/null +++ b/tests/host_python.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Fail when `tine.use_host_python` does not put the bootstrap toolchain on the host's python3. + + buck run tine//tests:host-python + +The test enables the setting for the project while it runs, and builds in its own isolation dir to +leave the developer's build state alone. A host whose python3 is older than the build needs, such as +a CI runner on Ubuntu 24.04, stands in the pinned interpreter for it. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +from contextlib import ExitStack +from pathlib import Path + +from util import buck_output, fail + +ISOLATION = "host-python" +SETTING = Path(".buckconfig.d/tine-host-python") +PROBE = "tine//tests:host-python-probe" +SUITE = "tine//python:test" + + +def realpath(python: str) -> str | None: + """Return the interpreter's resolved path, or None if it cannot run the build.""" + proc = subprocess.run( + [python, "-c", "import os, sys, compression.zstd; print(os.path.realpath(sys.executable))"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + return proc.stdout.strip() if proc.returncode == 0 else None + + +class Nested: + """Run nested `tine buck` commands in the test's isolation dir.""" + + def __init__(self, root: Path, path: str) -> None: + self.root = root + # The outer `tine buck run` exports BUCK2_BINARY, BUCK2_ARG0 and TINE_MOUNTS. Without them, the + # nested bin/tine resolves Buck and sets up the mounts itself, as it does for a developer. + self.environment = { + name: value + for name, value in os.environ.items() + if name not in ("BUCK2_BINARY", "BUCK2_ARG0", "TINE_MOUNTS") + } | {"PATH": path} + + def tine(self, *arguments: str) -> str: + """Run `tine buck` in the isolation dir and return its stdout. Buck's progress goes to stderr.""" + command = [str(self.root / "bin" / "tine"), "buck", "--isolation-dir", ISOLATION, *arguments] + print("+", " ".join(command[1:]), file=sys.stderr, flush=True) + proc = subprocess.run( + command, cwd=self.root, env=self.environment, stdout=subprocess.PIPE, text=True + ) + if proc.returncode != 0: + print(proc.stdout, end="", file=sys.stderr, flush=True) + fail(f"tine buck {' '.join(arguments)} exited with {proc.returncode}") + return proc.stdout + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--fallback", required=True, help="the pinned interpreter") + args = parser.parse_args() + + root = Path(buck_output("root", "--kind", "project")) + path = os.environ.get("PATH", "") + host = shutil.which("python3", path=path) + expected = realpath(host) if host else None + if expected is None: + fallback = Path(args.fallback).absolute() + print(f"=== no python3 on PATH runs the build; using {fallback}", file=sys.stderr) + path = os.pathsep.join([str(fallback.parent), path]) + expected = realpath(str(fallback)) + if expected is None: + fail(f"{fallback} cannot run the build either") + + setting = root / SETTING + if setting.exists(): + fail(f"{setting} exists already; this test writes and removes it") + with ExitStack() as stack: + run = Nested(root, path) + # `clean` deletes the isolation dir's buck-out and stops its daemon. + stack.callback(run.tine, "clean") + stack.callback(setting.unlink) + setting.parent.mkdir(exist_ok=True) + setting.write_text("[tine]\nuse_host_python = true\n") + + print("=== the toolchain starts the host's python3, isolated from the host", file=sys.stderr) + # bin/tine prints its own notes between the probe's output, which is one JSON line. + lines = [line for line in run.tine("run", PROBE).splitlines() if line.startswith("{")] + if not lines: + fail(f"{PROBE} printed nothing") + probe = json.loads(lines[-1]) + if probe["executable"] != expected: + fail(f"{PROBE} ran on {probe['executable']}, not on {expected}") + flags = ("ignore_environment", "no_site", "no_user_site") + if unset := [flag for flag in flags if not probe[flag]]: + fail(f"{PROBE} ran without {', '.join(unset)}") + if "tine-host-python" not in probe["xoptions"]: + fail(f"{PROBE} ran without the interpreter's identity on its command line") + + print("=== a suite runs on the interpreter it is told it runs on", file=sys.stderr) + run.tine("test", SUITE) + print(f"=== the bootstrap toolchain ran on {expected}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tests/host_python_probe.py b/tests/host_python_probe.py new file mode 100644 index 00000000..4bb28b28 --- /dev/null +++ b/tests/host_python_probe.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Print how the bootstrap toolchain started this interpreter, for tine//tests:host-python.""" + +import json +import os +import sys + +json.dump( + { + "executable": os.path.realpath(sys.executable), + "ignore_environment": sys.flags.ignore_environment, + "no_site": sys.flags.no_site, + "no_user_site": sys.flags.no_user_site, + "xoptions": sys._xoptions, + }, + sys.stdout, +) diff --git a/tools/ci.sh b/tools/ci.sh index 7f3d1586..f7147af6 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -240,4 +240,6 @@ if [ -z "${TINE_TEST_DISTRO:-}" ]; then group rebuild-reproducibility -- "${buck[@]}" run tine//tests:rebuild-reproducibility # The shared cache with a real Buck on both ends; in its own isolation dir. group remote-cache -- "${buck[@]}" run tine//tests:cache-roundtrip + # The bootstrap toolchain on the host's python3; in its own isolation dir. + group host-python -- "${buck[@]}" run tine//tests:host-python fi