From a8e39847a18bfb1fcf90bdc3e462f7336d92369f Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 7 Nov 2026 16:59:14 +0100 Subject: [PATCH 1/7] image: Write complete UAPI.16 sequences The manifest uses the wrong timestamp field and lacks the trailer that marks a complete sequence. Let's use mTimeNSec and require the trailer in readers and writers. Let's also move the format out of the image driver into a uapi16 package of its own. Other packages can then read and write manifests without depending on image/. Signed-off-by: Daan De Meyer --- docs/design/architecture.md | 2 + examples/image-secureboot/BUCK | 16 +- examples/image/expected.json | 6 +- examples/image/manifest_check.py | 3 +- image/BUCK | 18 +- image/ddi.py | 6 +- image/manifest_main.py | 33 ++++ ruff.toml | 1 + uapi16/BUCK | 20 ++ uapi16/pyproject.toml | 4 + image/manifest.py => uapi16/uapi16.py | 178 ++++++++++-------- .../manifest_test.py => uapi16/uapi16_test.py | 106 ++++++++--- 12 files changed, 263 insertions(+), 130 deletions(-) create mode 100755 image/manifest_main.py create mode 100644 uapi16/BUCK create mode 100644 uapi16/pyproject.toml rename image/manifest.py => uapi16/uapi16.py (69%) mode change 100755 => 100644 rename image/manifest_test.py => uapi16/uapi16_test.py (78%) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index fb6680e6..8518853e 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -53,6 +53,8 @@ tine//box/ box bootstrap and sandbox command construction tine//rootfs/ bind/overlay mounting and stored-delta translation tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and drivers, and VM runners +tine//uapi16/ UAPI.16 manifests and the snapshot stores built on them +tine//archive/ tar and cpio readers and writers tine//git/ pinned Git repositories with local checkout overrides tine//cargo/ vendored crate trees and offline Rust source builds tine//go/ go.sum-verified module fetches and offline Go source builds diff --git a/examples/image-secureboot/BUCK b/examples/image-secureboot/BUCK index 17266ad2..a36ea053 100644 --- a/examples/image-secureboot/BUCK +++ b/examples/image-secureboot/BUCK @@ -314,13 +314,15 @@ box.sh_test( name = "disk-manifest-test", labels = ["image"], box = "@tine//tools:dev.box", - test = "@tine//tests:contains.sh", - args = [ - "$(location :image[disk-manifest])", - '"name":"usr/bin/bash"', - '"name":"boot/tine-tmpfiles","type":"dir","mode":448', - '"name":"boot/EFI/Linux', - ], + args = ["$(location :image[disk-manifest])"], + script = """ +# Authored tmpfiles must set the boot directory mode in the disk's filesystem listing. +jq --seq -es ' + any(.[]; .name == "usr/bin/bash") and + any(.[]; .name == "boot/tine-tmpfiles" and .type == "dir" and .mode == 448) and + any(.[]; .name | strings | startswith("boot/EFI/Linux")) +' "$1" +""", ) # The kernel install hook is bypassed, it didn't generate its own initrd. diff --git a/examples/image/expected.json b/examples/image/expected.json index 92e5a631..1830c6e1 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -54,13 +54,13 @@ "sha256": "1dbd49e1af726cb9023fd1da842d5441736563c5dea426901a98e2b5e22c14cb" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "8c434b3a0a28f6c6eb6e1609bb766258dbf51c5e0b7e2c79b7161df918561713" + "sha256": "f2130ab8eb7af424c6be60b592f90015d757a24354af9fe103577b5a1d576619" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "a856a70ea0b09588e28798cf88250f550ae2475f705ed6d84cea88c076b79946" + "sha256": "16862b16926547262c44e1cc39de94e1136d1aa0b1ffa74fd4a1b25281db284b" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "0451b4a974d9376d975cf92902fc3a287e1a94679947cbddaebc540d838ba861" + "sha256": "456389b2e66d0a9dd1a60af74d1496cafffd89cd0141d0d166b1cd8a6135d949" }, "boot-demo.fedora[partitions][usr-verity]": { "sha256": "8baac782ec9d31c05161bbf86cec6afab8b55d41faad269d932fd67c8680338f" diff --git a/examples/image/manifest_check.py b/examples/image/manifest_check.py index 4cd53714..f96a097b 100644 --- a/examples/image/manifest_check.py +++ b/examples/image/manifest_check.py @@ -27,7 +27,8 @@ def decode(data: bytes) -> list[dict[str, Any]]: objects = decode(Path(sys.argv[1]).read_bytes()) -root, files = objects[0], objects[1:] +assert objects[-1] == {"mediaType": "application/vnd.uapi.16.trailer"}, "missing trailer" +root, files = objects[0], objects[1:-1] assert root == {"mediaType": "application/vnd.uapi.16.manifest"}, f"bad root object: {root}" assert files, "the manifest lists no files at all" diff --git a/image/BUCK b/image/BUCK index 9e6a2c58..1124633c 100644 --- a/image/BUCK +++ b/image/BUCK @@ -57,10 +57,10 @@ python.binary( main = "ddi.py", deps = [ ":finalize", - ":manifest-lib", "//:specs", "//:util", "//rootfs:rootfs", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) @@ -124,8 +124,6 @@ python.test( "finalize_test.py", "layer.py", "layer_test.py", - "manifest.py", - "manifest_test.py", "preload.py", "preload_test.py", "substitute.py", @@ -145,6 +143,7 @@ python.test( "//kmod:kmod", "//package:installer", "//rootfs:rootfs", + "//uapi16:uapi16", ], ) @@ -160,25 +159,17 @@ python.binary( python.binary( name = "manifest", - main = "manifest.py", + main = "manifest_main.py", deps = [ ":finalize", "//:specs", "//:util", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) # A driver that packages a tree lists it from inside its own action, where the tree is final. -python.library( - name = "manifest-lib", - srcs = ["manifest.py"], - deps = [ - ":finalize", - "//:specs", - "//:util", - ], -) python.library( name = "finalize", @@ -220,6 +211,7 @@ python.binary( "//:specs", "//:util", "//kmod:kmod", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) diff --git a/image/ddi.py b/image/ddi.py index 244ff3ba..df6685c5 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -21,8 +21,8 @@ import util import finalize -import manifest import rootfs +import uapi16 _SEED_NAMESPACE = uuid.UUID("5af2de99-4f9f-4e0b-a04b-bde36b068c4f") @@ -511,7 +511,7 @@ def main(argv: list[str] | None = None) -> None: listings = [] for partition in written: if partition.manifest is not None: - manifest.write(tree, partition.manifest, epoch, roots=partition.definition.sources) + uapi16.write(tree, partition.manifest, epoch, roots=partition.definition.sources) listings.append(partition.manifest) # An imported partition was listed by the invocation that filled it, and this one holds @@ -519,7 +519,7 @@ def main(argv: list[str] | None = None) -> None: # partition this invocation mounts over a directory another one holds is the one on top. if spec["manifest"]: imported = [it.manifest for it in partitions if it.manifest is not None] - manifest.merge(imported + listings, Path(spec["manifest"])) + uapi16.merge(imported + listings, Path(spec["manifest"])) scratch = Path(scratch_dir) repart_definitions = scratch / "repart.d" diff --git a/image/manifest_main.py b/image/manifest_main.py new file mode 100755 index 00000000..d3b69f84 --- /dev/null +++ b/image/manifest_main.py @@ -0,0 +1,33 @@ +#!/usr/bin/python3 + +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Export the finalized image's UAPI.16 listing.""" + +import os +import sys +from pathlib import Path + +import specs + +import finalize +import uapi16 + + +class Spec(finalize.ImageSpec): + out: str + + +def main(argv: list[str] | None = None) -> None: + spec = specs.parse(Spec, "manifest", argv) + + out = Path(spec["out"]) + epoch = int(os.environ["SOURCE_DATE_EPOCH"]) + with finalize.image(spec, program="manifest") as tree: + written = uapi16.write(tree, out, epoch) + print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/ruff.toml b/ruff.toml index 5ddc2873..5677281c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -21,6 +21,7 @@ src = [ "package_system/rpm", "remote_cache", "rootfs", + "uapi16", ] lint.select = [ "E", # pycodestyle diff --git a/uapi16/BUCK b/uapi16/BUCK new file mode 100644 index 00000000..98170b32 --- /dev/null +++ b/uapi16/BUCK @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""UAPI.16 manifests and the snapshot stores built on them.""" + +load("//python:defs.bzl", "python") + +python.library( + name = "uapi16", + srcs = ["uapi16.py"], + deps = ["//:util"], + visibility = ["tine//..."], +) + +python.test( + name = "test", + box = "//tools:dev.box", + srcs = ["uapi16_test.py"], + deps = [":uapi16", "//:util"], +) diff --git a/uapi16/pyproject.toml b/uapi16/pyproject.toml new file mode 100644 index 00000000..a0e58bb2 --- /dev/null +++ b/uapi16/pyproject.toml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +# Zed needs a Python manifest here to route these files to their own ty instance. diff --git a/image/manifest.py b/uapi16/uapi16.py old mode 100755 new mode 100644 similarity index 69% rename from image/manifest.py rename to uapi16/uapi16.py index 5dfd8189..f9d7d51e --- a/image/manifest.py +++ b/uapi16/uapi16.py @@ -1,45 +1,23 @@ -#!/usr/bin/python3 - # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""List everything a logical image ships, as a UAPI.16 File Manifest. - -The format is an RFC7464 JSON-SEQ sequence: a root object naming the media type, then one object -per inode, each directory immediately followed by its own contents. Two consumers want this: a -comparison of the paths two images ship (a path in both a system extension and the /usr it merges -onto silently shadows an OS file), and per-image size accounting. Both need every inode type an OS -tree holds, so a type that cannot be listed is an error rather than an omission. - -A driver writing a listing of what it is about to package imports this rather than running it: only -the driver holds the finished tree, and only it knows which paths of that tree its artifact carries. -`roots` is how it says so, and names stay relative to the image, so a listing reads as where each -path lands on a machine rather than where it sits inside one partition. -""" +"""Read, write and combine UAPI.16 filesystem listings.""" import base64 import hashlib import json import os import stat -import sys from collections import Counter from collections.abc import Iterator, Sequence from dataclasses import dataclass from pathlib import Path -from typing import cast +from typing import TypedDict, cast -import specs from util import fail -import finalize - - -class Spec(finalize.ImageSpec): - out: str - - MEDIA_TYPE = "application/vnd.uapi.16.manifest" +TRAILER_TYPE = "application/vnd.uapi.16.trailer" # RFC7464 frames every JSON text with a leading record separator and a trailing line feed. _RS = b"\x1e" @@ -47,7 +25,7 @@ class Spec(finalize.ImageSpec): # A directory holding a manifest cannot also list it, so the format reserves the name at the top # level; deeper in the tree it is an ordinary file name. -_RESERVED = "Uapi16Manifest" +MANIFEST = "Uapi16Manifest" _INODE_TYPES = ( (stat.S_ISREG, "reg"), @@ -62,6 +40,35 @@ class Spec(finalize.ImageSpec): _NANOSECONDS = 10**9 +class Content(TypedDict, total=False): + file: str + literal: str + + +class Xattr(TypedDict, total=False): + text: str + base64: str + + +class Object(TypedDict, total=False): + """One record of a UAPI.16 manifest.""" + + mediaType: str + name: str + type: str + mode: int + uid: int + gid: int + mTimeNSec: int + size: int + major: int + minor: int + sha256: str + contents: list[Content] + inodeToken: int + extendedAttributes: dict[str, dict[str, Xattr]] + + @dataclass(frozen=True) class Entry: """One inode below the image root, under the name the manifest gives it.""" @@ -83,19 +90,17 @@ def inode_type(mode: int) -> str: def check_name(name: str) -> str: - """Reject a path the format cannot encode, rather than listing the image incompletely. - - Nothing normalizes here: scandir yields neither "." nor ".." nor an empty component, so what - is left is the format's own restrictions on the bytes and on the reserved file name. - """ + """Reject a path the format cannot encode, rather than listing the image incompletely.""" try: name.encode("utf-8") except UnicodeEncodeError: fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be") if any(ord(character) < 32 or ord(character) == 127 for character in name): fail(f"manifest: {name!r} contains a control character") + if any(part in ("", ".", "..") for part in name.split("/")): + fail(f"manifest: {name!r} must be a normalized relative path") top = name.split("/", 1)[0] - if top == _RESERVED or top.startswith(_RESERVED + "."): + if top == MANIFEST or top.startswith(MANIFEST + "."): fail(f"manifest: {name!r} is the reserved manifest file name") return name @@ -172,10 +177,10 @@ def _digest(path: Path) -> str: return hashlib.file_digest(stream, "sha256").hexdigest() -def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[str, object]: - """The record for one inode, with its fields in the order the specification lists them.""" +def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object: + """Describe one inode using UAPI.16 metadata and content fields.""" kind = inode_type(entry.st.st_mode) - obj: dict[str, object] = {"name": entry.name, "type": kind} + obj: Object = {"name": entry.name, "type": kind} target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b"" if kind == "reg": @@ -189,9 +194,9 @@ def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[s obj["mode"] = stat.S_IMODE(entry.st.st_mode) obj["uid"] = entry.st.st_uid obj["gid"] = entry.st.st_gid - # Clamped exactly as every other reproducible output here clamps: what a package stamped - # survives, what this build stamped collapses onto the epoch. - obj["mTime"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) + obj["mTimeNSec"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) + if entry.st.st_mtime_ns < 0: + fail(f"manifest: {entry.name!r} has a timestamp before the UNIX epoch") if token is not None: obj["inodeToken"] = token if kind == "lnk": @@ -205,8 +210,12 @@ def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[s return obj -def _record(obj: dict[str, object]) -> bytes: - return _RS + json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + _LF +def _record(obj: Object) -> bytes: + return ( + _RS + + json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + _LF + ) def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: @@ -219,6 +228,7 @@ def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes tokens = hardlink_tokens(found) for entry in found: yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode))) + yield _record({"mediaType": TRAILER_TYPE}) def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: @@ -231,21 +241,55 @@ def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: return written -def _objects(source: Path) -> Iterator[dict[str, object]]: - """The named objects of a manifest, the root object it opens with aside.""" +def read(source: Path) -> list[Object]: + """Read a complete manifest, rejecting truncated sequences and trailing data.""" data = source.read_bytes() - head, separator, rest = data.partition(_RS) - if not separator or head: + if not data.startswith(_RS): fail(f"manifest: {source} does not open with a record") - for index, record in enumerate(rest.split(_RS)): + objects: list[Object] = [] + names: set[str] = set() + trailer = False + for record in data[1:].split(_RS): + if trailer: + fail(f"manifest: {source} has data after its trailer") if not record.endswith(_LF): fail(f"manifest: {source} holds a record that no line feed ends") - obj = cast(dict[str, object], json.loads(record)) - if index == 0: - if obj != {"mediaType": MEDIA_TYPE}: - fail(f"manifest: {source} does not open with a {MEDIA_TYPE} object") + try: + obj = cast(Object, json.loads(record)) + except (ValueError, UnicodeDecodeError) as error: + fail(f"manifest: {source} has invalid JSON: {error}") + if not isinstance(obj, dict): + fail(f"manifest: {source} holds a non-object record") + if not objects: + if obj.get("mediaType") != MEDIA_TYPE or obj.get("name") is not None: + fail(f"manifest: {source} does not open with a {MEDIA_TYPE} root object") + elif obj.get("mediaType") == TRAILER_TYPE: + text = record.decode("utf-8") + _, end = json.JSONDecoder().raw_decode(text, len(text) - len(text.lstrip())) + if obj != {"mediaType": TRAILER_TYPE} or text[end:].lstrip(" \t\r") != "\n": + fail(f"manifest: {source} has an invalid trailer or data after its trailer") + trailer = True + continue + elif obj.get("mediaType") not in (None, MEDIA_TYPE): continue - yield obj + else: + name = obj.get("name") + if not isinstance(name, str) or name in names: + fail(f"manifest: {source} has a missing or duplicate name: {name!r}") + names.add(check_name(name)) + objects.append(obj) + if not trailer: + fail(f"manifest: {source} is incomplete: missing trailer") + return objects + + +def write_objects(objects: Sequence[Object], out: Path) -> int: + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("wb") as stream: + for obj in objects: + stream.write(_record(obj)) + stream.write(_record({"mediaType": TRAILER_TYPE})) + return len(objects) + 1 def order(name: str) -> list[bytes]: @@ -264,11 +308,11 @@ def merge(sources: Sequence[Path], out: Path) -> int: between them. A name more than one of them lists is taken from the last, which is the partition mounted over a directory the others merely hold. """ - objects: dict[str, dict[str, object]] = {} + objects: dict[str, Object] = {} offset = 0 for source in sources: highest = 0 - for obj in _objects(source): + for obj in read(source)[1:]: token = obj.get("inodeToken") if isinstance(token, int): # A token numbers an inode within one manifest, so each part keeps its own grouping @@ -278,25 +322,11 @@ def merge(sources: Sequence[Path], out: Path) -> int: objects[str(obj["name"])] = obj offset += highest - written = 1 - out.parent.mkdir(parents=True, exist_ok=True) - with out.open("wb") as stream: - stream.write(_record({"mediaType": MEDIA_TYPE})) - for name in sorted(objects, key=order): - stream.write(_record(objects[name])) - written += 1 - return written - - -def main(argv: list[str] | None = None) -> None: - spec = specs.parse(Spec, "manifest", argv) - - out = Path(spec["out"]) - epoch = int(os.environ["SOURCE_DATE_EPOCH"]) - with finalize.image(spec, program="manifest") as tree: - written = write(tree, out, epoch) - print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) - - -if __name__ == "__main__": - main() + tokens: dict[int, int] = {} + counts = Counter(obj.get("inodeToken") for obj in objects.values() if obj.get("inodeToken")) + ordered = [objects[name] for name in sorted(objects, key=order)] + for obj in ordered: + token = obj.pop("inodeToken", None) + if isinstance(token, int) and counts[token] > 1: + obj["inodeToken"] = tokens.setdefault(token, len(tokens) + 1) + return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out) diff --git a/image/manifest_test.py b/uapi16/uapi16_test.py similarity index 78% rename from image/manifest_test.py rename to uapi16/uapi16_test.py index 5375e9ae..00a7d594 100644 --- a/image/manifest_test.py +++ b/uapi16/uapi16_test.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Tests for the UAPI.16 file manifest. +"""Tests for the UAPI.16 file uapi16. buck test tine//image:test """ @@ -17,7 +17,7 @@ from pathlib import Path from typing import override -import manifest +import uapi16 # The epoch every image action runs under (box/runtime.bzl); any value works, but using the real # one keeps the clamping assertions honest about what a build actually produces. @@ -33,16 +33,17 @@ def parse(data: bytes) -> list[dict[str, object]]: for record in rest.split(b"\x1e"): assert record.endswith(b"\n"), "a record is not terminated by a line feed" objects.append(json.loads(record.decode("utf-8"))) - return objects + assert objects[-1] == {"mediaType": uapi16.TRAILER_TYPE} + return objects[:-1] class TreeTest(unittest.TestCase): @override def setUp(self) -> None: - self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest."))) + self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest."))) def manifest(self, tree: Path | None = None, roots: Sequence[str] = ()) -> list[dict[str, object]]: - return parse(b"".join(manifest.records(tree or self.tree, EPOCH, roots))) + return parse(b"".join(uapi16.records(tree or self.tree, EPOCH, roots))) def named(self, tree: Path | None = None) -> dict[str, dict[str, object]]: return {str(obj["name"]): obj for obj in self.manifest(tree)[1:]} @@ -54,18 +55,20 @@ def test_the_first_object_is_the_root_and_names_the_media_type(self) -> None: root = self.manifest()[0] self.assertEqual(root, {"mediaType": "application/vnd.uapi.16.manifest"}) - def test_an_empty_tree_is_the_root_object_alone(self) -> None: + def test_an_empty_tree_has_no_named_objects(self) -> None: + # An empty filesystem still has its root and trailer, with no named file objects. self.assertEqual(len(self.manifest()), 1) def test_each_record_is_framed_by_a_separator_and_a_line_feed(self) -> None: + # A tree with one file needs three JSON-SEQ records: the root, file and trailer. (self.tree / "a").write_text("a") - data = b"".join(manifest.records(self.tree, EPOCH)) - self.assertEqual(data.count(b"\x1e"), 2) + data = b"".join(uapi16.records(self.tree, EPOCH)) + self.assertEqual(data.count(b"\x1e"), 3) self.assertTrue(data.endswith(b"\n")) def test_the_json_carries_no_gratuitous_whitespace(self) -> None: (self.tree / "a").write_text("a") - self.assertNotIn(b" ", b"".join(manifest.records(self.tree, EPOCH))) + self.assertNotIn(b" ", b"".join(uapi16.records(self.tree, EPOCH))) class TestOrder(TreeTest): @@ -92,14 +95,14 @@ def test_the_bytes_do_not_depend_on_the_order_the_tree_was_created_in(self) -> N names = ["gamma", "alpha", "beta", "delta"] outputs = [] for order in (names, list(reversed(names))): - with tempfile.TemporaryDirectory(prefix="manifest.") as scratch: + with tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.") as scratch: tree = Path(scratch) for name in order: (tree / name).mkdir() (tree / name / "file").write_text(name) os.utime(tree / name / "file", (EPOCH - 1, EPOCH - 1)) os.utime(tree / name, (EPOCH - 1, EPOCH - 1)) - outputs.append(b"".join(manifest.records(tree, EPOCH))) + outputs.append(b"".join(uapi16.records(tree, EPOCH))) self.assertEqual(outputs[0], outputs[1]) @@ -193,7 +196,9 @@ def test_a_file_reached_by_one_name_alone_carries_no_token(self) -> None: def test_a_link_count_above_one_is_not_enough_when_the_other_name_is_outside_the_tree( self, ) -> None: - outside = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.outside."))) + outside = Path( + self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.outside.")) + ) (outside / "one").write_text("shared") os.link(outside / "one", self.tree / "inside") self.assertEqual((self.tree / "inside").stat().st_nlink, 2) @@ -229,24 +234,24 @@ def test_a_unix_socket_is_listed_as_one(self) -> None: def test_the_inode_type_names_follow_the_format(self) -> None: self.assertEqual( - [manifest.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], + [uapi16.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], ["blk", "sock", "lnk"], ) def test_an_inode_type_with_no_representation_fails_rather_than_being_guessed(self) -> None: with self.assertRaises(SystemExit): - manifest.inode_type(0) + uapi16.inode_type(0) class TestDeviceNodes(unittest.TestCase): """Device nodes, from a fabricated inode: this sandbox refuses mknod.""" - def device(self, mode: int, rdev: int) -> dict[str, object]: + def device(self, mode: int, rdev: int) -> uapi16.Object: st = os.stat_result( (mode, 0, 0, 1, 0, 0, 0, 0, EPOCH - 1, 0), {"st_rdev": rdev, "st_mtime_ns": (EPOCH - 1) * 10**9}, ) - return manifest.file_object(manifest.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) + return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -> None: self.assertEqual( @@ -259,7 +264,7 @@ def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) - "mode": 0o666, "uid": 0, "gid": 0, - "mTime": (EPOCH - 1) * 10**9, + "mTimeNSec": (EPOCH - 1) * 10**9, }, ) @@ -273,12 +278,12 @@ class TestTimestamps(TreeTest): def test_an_mtime_past_the_epoch_is_clamped_onto_it(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", (EPOCH + 10, EPOCH + 10)) - self.assertEqual(self.named()["file"]["mTime"], EPOCH * 10**9) + self.assertEqual(self.named()["file"]["mTimeNSec"], EPOCH * 10**9) def test_an_mtime_before_the_epoch_survives_in_nanoseconds(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", ns=(0, (EPOCH - 5) * 10**9 + 123)) - self.assertEqual(self.named()["file"]["mTime"], (EPOCH - 5) * 10**9 + 123) + self.assertEqual(self.named()["file"]["mTimeNSec"], (EPOCH - 5) * 10**9 + 123) class TestUnencodableNames(TreeTest): @@ -304,28 +309,32 @@ def test_a_reserved_suffix_at_the_top_level_is_refused_too(self) -> None: class TestWrite(TreeTest): - def test_the_written_file_is_the_sequence_and_the_count_includes_the_root(self) -> None: + def test_the_count_includes_the_root_and_trailer(self) -> None: + # Writing one file must report three records and finish with the mandatory trailer. (self.tree / "file").write_text("x") - out = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.out."))) / "m" - self.assertEqual(manifest.write(self.tree, out, EPOCH), 2) + out = ( + Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.out."))) + / "m" + ) + self.assertEqual(uapi16.write(self.tree, out, EPOCH), 3) self.assertEqual([obj.get("name") for obj in parse(out.read_bytes())], [None, "file"]) class TestScope(unittest.TestCase): def test_a_leading_and_trailing_slash_is_not_part_of_the_name(self) -> None: - self.assertEqual(manifest.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) + self.assertEqual(uapi16.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) def test_the_same_root_named_twice_is_listed_once(self) -> None: - self.assertEqual(manifest.scope(["/usr", "usr/"]), ["usr"]) + self.assertEqual(uapi16.scope(["/usr", "usr/"]), ["usr"]) def test_a_root_below_another_is_dropped_rather_than_listed_twice(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) + self.assertEqual(uapi16.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) def test_a_name_that_merely_shares_a_prefix_is_kept(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) + self.assertEqual(uapi16.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) def test_the_whole_tree_covers_every_other_root(self) -> None: - self.assertEqual(manifest.scope(["/", "/usr"]), [""]) + self.assertEqual(uapi16.scope(["/", "/usr"]), [""]) class TestScopedTree(TreeTest): @@ -361,7 +370,7 @@ def test_no_roots_is_the_whole_tree(self) -> None: class TestMerge(TreeTest): def part(self, name: str, roots: Sequence[str]) -> Path: out = self.tree / name - manifest.write(self.tree / "parts", out, EPOCH, roots) + uapi16.write(self.tree / "parts", out, EPOCH, roots) return out @override @@ -374,7 +383,7 @@ def setUp(self) -> None: def merged(self, *sources: Path) -> list[dict[str, object]]: out = self.tree / "merged" - manifest.merge(sources, out) + uapi16.merge(sources, out) return parse(out.read_bytes()) def test_the_whole_is_every_part_under_one_root_object(self) -> None: @@ -418,5 +427,44 @@ def test_something_that_is_not_a_manifest_is_refused(self) -> None: self.merged(self.tree / "junk") +class TestReader(TreeTest): + def test_requires_trailer_and_rejects_trailing_data(self) -> None: + # A partial write and data after a trailer must fail instead of becoming a snapshot. + out = self.tree / "manifest" + root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n' + trailer = b'\x1e{"mediaType":"application/vnd.uapi.16.trailer"}\n' + for data in (root, root + trailer + trailer, root + trailer + b" ", root + trailer + b"\n"): + out.write_bytes(data) + with self.subTest(data=data), self.assertRaises(SystemExit): + uapi16.read(out) + out.write_bytes(root + trailer) + self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}]) + + def test_accepts_whitespace_before_the_trailers_line_feed(self) -> None: + # JSON whitespace may precede the trailer's final LF, but any bytes after that LF + # are trailing data and must still fail. + out = self.tree / "manifest" + root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n' + trailer = b'\x1e { "mediaType": "application/vnd.uapi.16.trailer" } \t\r\n' + out.write_bytes(root + trailer) + self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}]) + out.write_bytes(root + trailer + b" ") + with self.assertRaises(SystemExit): + uapi16.read(out) + + def test_accepts_root_metadata(self) -> None: + # A storage manifest includes root metadata, which must survive parsing unchanged. + out = self.tree / "manifest" + objects: list[uapi16.Object] = [{"mediaType": uapi16.MEDIA_TYPE, "mode": 0o750}] + uapi16.write_objects(objects, out) + self.assertEqual(uapi16.read(out), objects) + + def test_rejects_non_relative_names(self) -> None: + # UAPI.16 forbids path traversal and unnormalized names even in an external uapi16. + for name in ("", "/a", "a/", "a//b", "a/../b", "./a", "a/.", "a/..", "a\n"): + with self.subTest(name=name), self.assertRaises(SystemExit): + uapi16.check_name(name) + + if __name__ == "__main__": unittest.main() From 7d0317e599471d904a9c49845e5dab781004fd0c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 7 Nov 2026 17:02:14 +0100 Subject: [PATCH 2/7] uapi16: Store snapshots with immutable content blobs Buck directory artifacts lose inode metadata and hardlink identity. Let's keep complete filesystem manifests beside new content blobs and restore private roots before running commands, with declared hardlinks in the upper so index=off can't split them on copy-up. Signed-off-by: Daan De Meyer --- image/BUCK | 1 + uapi16/BUCK | 11 ++- uapi16/storage.py | 193 +++++++++++++++++++++++++++++++++++++++++ uapi16/storage_test.py | 189 ++++++++++++++++++++++++++++++++++++++++ uapi16/uapi16.py | 22 +++++ uapi16/uapi16_test.py | 4 +- 6 files changed, 417 insertions(+), 3 deletions(-) create mode 100644 uapi16/storage.py create mode 100644 uapi16/storage_test.py diff --git a/image/BUCK b/image/BUCK index 1124633c..2ef62314 100644 --- a/image/BUCK +++ b/image/BUCK @@ -143,6 +143,7 @@ python.test( "//kmod:kmod", "//package:installer", "//rootfs:rootfs", + "//uapi16:storage", "//uapi16:uapi16", ], ) diff --git a/uapi16/BUCK b/uapi16/BUCK index 98170b32..936460d2 100644 --- a/uapi16/BUCK +++ b/uapi16/BUCK @@ -12,9 +12,16 @@ python.library( visibility = ["tine//..."], ) +python.library( + name = "storage", + srcs = ["storage.py"], + deps = [":uapi16", "//:util", "//box:isolation"], + visibility = ["tine//..."], +) + python.test( name = "test", box = "//tools:dev.box", - srcs = ["uapi16_test.py"], - deps = [":uapi16", "//:util"], + srcs = ["storage_test.py", "uapi16_test.py"], + deps = [":storage", ":uapi16", "//:util", "//box:isolation"], ) diff --git a/uapi16/storage.py b/uapi16/storage.py new file mode 100644 index 00000000..7433eb8f --- /dev/null +++ b/uapi16/storage.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Store complete filesystem snapshots independently of artifact filesystem metadata.""" + +import base64 +import os +import stat +import tempfile +from collections.abc import Generator, Sequence +from contextlib import ExitStack, contextmanager +from pathlib import Path + +from util import clone_file, fail + +import uapi16 +from isolation import Overlay + + +class Snapshot: + def __init__(self, stores: Sequence[str | Path]) -> None: + self.stores = [Path(store) for store in stores] + self.objects: list[uapi16.Object] = ( + uapi16.read(self.stores[-1] / uapi16.MANIFEST) + if self.stores + else [{"mediaType": uapi16.MEDIA_TYPE, "type": "dir", "mode": 0o755}] + ) + self.blobs: dict[str, Path] = {} + for store in self.stores: + for blob in (store / "blobs").iterdir(): + self.blobs.setdefault(blob.name, blob) + self._validate() + + def _validate(self) -> None: + directories = {""} + groups: dict[int, uapi16.Object] = {} + for obj in self.objects[1:]: + name = obj["name"] + if name.rpartition("/")[0] not in directories: + fail(f"storage: {name!r} does not follow its parent directory") + if obj["type"] == "dir": + directories.add(name) + if token := obj.get("inodeToken"): + if obj["type"] == "dir": + fail(f"storage: directory {name!r} has an inodeToken") + inode = obj.copy() + del inode["name"] + if groups.setdefault(token, inode) != inode: + fail(f"storage: hardlink group {token} has conflicting metadata") + if obj["type"] == "reg": + digest = obj["sha256"] + if obj["contents"] != [{"file": f"blobs/{digest}"}]: + fail(f"storage: unsupported content reference for {name!r}") + self.content(obj) + + def content(self, obj: uapi16.Object) -> Path: + digest = obj["sha256"] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(c not in "0123456789abcdef" for c in digest) + ): + fail(f"storage: invalid SHA-256 digest {digest!r}") + if digest not in self.blobs: + fail(f"storage: missing blob {digest} for {obj['name']!r}") + path = self.blobs[digest] + if not stat.S_ISREG(path.lstat().st_mode) or path.stat().st_size != obj["size"]: + fail(f"storage: invalid blob {digest} for {obj['name']!r}") + return path + + +def capture(tree: Path, out: Path, parents: Sequence[str | Path] = (), *, epoch: int) -> None: + """Write the snapshot of `tree` to `out`, with modification times clamped to `epoch`.""" + parent = Snapshot(parents) + blobs = out / "blobs" + blobs.mkdir(parents=True, exist_ok=True) + found = list(uapi16.entries(tree)) + tokens = uapi16.hardlink_tokens(found) + root = uapi16.file_object(uapi16.Entry("", tree, tree.lstat()), epoch=epoch) + root.pop("name") + root["mediaType"] = uapi16.MEDIA_TYPE + objects = [root] + inodes: dict[tuple[int, int], uapi16.Object] = {} + for entry in found: + if entry.inode in inodes: + obj = inodes[entry.inode].copy() + obj["name"] = entry.name + else: + obj = uapi16.file_object(entry, epoch=epoch, token=tokens.get(entry.inode)) + if obj["type"] == "reg": + digest = obj["sha256"] + obj["contents"] = [{"file": f"blobs/{digest}"}] + destination = blobs / digest + if digest not in parent.blobs and not destination.exists(): + clone_file(entry.path, destination) + destination.chmod(0o444) + inodes[entry.inode] = obj + objects.append(obj) + for obj in objects: + # Snapshot ownership is root even when a package manager installs another owner. + obj["uid"] = obj["gid"] = 0 + uapi16.write_objects(objects, out / uapi16.MANIFEST) + + +def metadata(path: Path, obj: uapi16.Object) -> None: + if "uid" in obj or "gid" in obj: + os.chown(path, obj.get("uid", 0), obj.get("gid", 0), follow_symlinks=False) + if obj["type"] != "lnk": + path.chmod(obj["mode"]) + for name, value in obj.get("extendedAttributes", {}).get("user", {}).items(): + data = value["text"].encode() if "text" in value else base64.b64decode(value["base64"]) + os.setxattr(path, "user." + name, data, follow_symlinks=False) + if "mTimeNSec" in obj: + os.utime(path, ns=(obj["mTimeNSec"], obj["mTimeNSec"]), follow_symlinks=False) + + +def restore(snapshot: Snapshot, tree: Path, *, hardlinks: bool | None = None) -> None: + """Restore private inodes, optionally selecting linked or independent files.""" + tree.mkdir(parents=True, exist_ok=True) + groups: dict[int, Path] = {} + directories = [(tree, snapshot.objects[0])] + for obj in snapshot.objects[1:]: + path = tree / obj["name"] + kind = obj["type"] + token = obj.get("inodeToken") + if kind == "dir": + path.mkdir() + directories.append((path, obj)) + continue + if hardlinks is not None and bool(token) != hardlinks: + continue + if token and token in groups: + os.link(groups[token], path, follow_symlinks=False) + continue + if kind == "reg": + # Metadata restoration and writes must never change a cached blob's inode. + clone_file(snapshot.content(obj), path) + elif kind == "lnk": + path.symlink_to(os.fsdecode(base64.b64decode(obj["contents"][0]["literal"]))) + elif kind == "fifo": + os.mkfifo(path) + elif kind in ("chr", "blk"): + mode = stat.S_IFCHR if kind == "chr" else stat.S_IFBLK + os.mknod(path, mode | obj["mode"], os.makedev(obj["major"], obj["minor"])) + elif kind == "sock": + os.mknod(path, stat.S_IFSOCK | obj["mode"]) + else: + fail(f"storage: unsupported inode type {kind!r}") + metadata(path, obj) + if token: + groups[token] = path + # Creating children changes directory mtimes, so restore directories last. + for path, obj in reversed(directories): + metadata(path, obj) + + +def _identities(objects: Sequence[uapi16.Object]) -> dict[str, tuple[uapi16.Object, frozenset[str]]]: + """Map each name to its object without the inodeToken and to the names of its hardlink group.""" + groups: dict[int, set[str]] = {} + for obj in objects: + if token := obj.get("inodeToken"): + groups.setdefault(token, set()).add(obj["name"]) + identities: dict[str, tuple[uapi16.Object, frozenset[str]]] = {} + for obj in objects: + rest = obj.copy() + token = rest.pop("inodeToken", None) + identities[obj["name"]] = (rest, frozenset(groups[token]) if token else frozenset()) + return identities + + +@contextmanager +def root(stores: Sequence[str | Path], *, base: int = 0) -> Generator[Path]: + snapshot = Snapshot(stores) + if base: + inherited = _identities(Snapshot(stores[:base]).objects[1:]) + changed = { + name for name, obj in _identities(snapshot.objects[1:]).items() if inherited.get(name) != obj + } + needed = {str(parent) for name in changed for parent in Path(name).parents} + snapshot.objects = [snapshot.objects[0]] + [ + obj for obj in snapshot.objects[1:] if obj["name"] in changed or obj["name"] in needed + ] + with ExitStack() as stack: + scratch = Path( + stack.enter_context(tempfile.TemporaryDirectory(prefix="snapshot-root.", dir="/var/tmp")) + ) + lower, upper, work, merged = (scratch / name for name in ("lower", "upper", "work", "root")) + restore(snapshot, lower, hardlinks=False) + # index=off splits lower hardlinks on copy-up. Upper hardlinks keep their inode on writes. + restore(snapshot, upper, hardlinks=True) + work.mkdir() + stack.enter_context(Overlay((lower,), upper, work, merged)).mount() + yield merged diff --git a/uapi16/storage_test.py b/uapi16/storage_test.py new file mode 100644 index 00000000..f5747006 --- /dev/null +++ b/uapi16/storage_test.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Snapshot storage and reconstruction regressions.""" + +import os +import shutil +import socket +import stat +import tempfile +import unittest +from pathlib import Path +from typing import override + +import storage +import uapi16 + +# Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. +CAPTURE_EPOCH = 2**31 + + +class TestStorage(unittest.TestCase): + @override + def setUp(self) -> None: + self.scratch = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp"))) + self.tree = self.scratch / "source" + self.tree.mkdir() + self.store = self.scratch / "store" + + def files(self) -> None: + for name, mode, time in (("a", 0o640, 111), ("b", 0o755, 1800000000000000222)): + path = self.tree / name + path.write_text("identical") + path.chmod(mode) + os.utime(path, ns=(time, time)) + os.link(self.tree / "a", self.tree / "linked") + + def test_roundtrip_after_artifact_metadata_is_lost(self) -> None: + # Copy the store without timestamps or hardlinks. The manifest must restore inode identity, + # original modes and times, literal marker names, symlinks, empty directories and user xattrs. + self.files() + for name in ("back\\slash", ".wh.literal", ".wh..wh..opq", ".esc.literal"): + (self.tree / name).write_text(name) + (self.tree / "empty").mkdir() + (self.tree / "link").symlink_to("/a") + os.utime(self.tree / "link", ns=(333, 333), follow_symlinks=False) + os.utime(self.tree / "empty", ns=(444, 444)) + os.setxattr(self.tree / "a", "user.example", b"\x00\xff") + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + copied = self.scratch / "copied" + shutil.copytree(self.store, copied, copy_function=shutil.copyfile) + restored = self.scratch / "restored" + storage.restore(storage.Snapshot([copied]), restored) + self.assertEqual((restored / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((restored / "a").stat().st_mode), 0o640) + self.assertEqual((restored / "b").stat().st_mtime_ns, 1800000000000000222) + self.assertEqual(stat.S_IMODE((restored / "b").stat().st_mode), 0o755) + self.assertTrue((restored / "a").samefile(restored / "linked")) + self.assertFalse((restored / "a").samefile(restored / "b")) + self.assertEqual(os.getxattr(restored / "a", "user.example"), b"\x00\xff") + self.assertEqual(os.readlink(restored / "link"), "/a") + self.assertEqual((restored / "link").lstat().st_mtime_ns, 333) + self.assertEqual((restored / "empty").stat().st_mtime_ns, 444) + self.assertEqual({p.name for p in restored.iterdir()}, {p.name for p in self.tree.iterdir()}) + + def test_capture_clamps_times_to_the_epoch(self) -> None: + # A package manager writes the time of the install into every file and directory that it + # changes. Capture must store the epoch for a time after it and keep an earlier time, so two + # installs at different times write the same manifest. Capture a tree with one file before and + # one file after the epoch, and check the times in the manifest. + self.files() + storage.capture(self.tree, self.store, epoch=1700000000) + objects = {obj.get("name"): obj for obj in storage.Snapshot([self.store]).objects} + self.assertEqual(objects["a"]["mTimeNSec"], 111) + self.assertEqual(objects["b"]["mTimeNSec"], 1700000000 * 10**9) + + def test_overlay_writes_and_metadata_leave_blobs_unchanged(self) -> None: + # Declared links start in the upper because index=off splits lower links on writes. + # Equal-content independent files must copy up separately without changing the blob. + self.files() + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + blob = next((self.store / "blobs").iterdir()) + before = blob.read_bytes(), blob.stat() + with storage.root([self.store]) as root: + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertFalse((root / "a").samefile(root / "b")) + (root / "a").write_text("linked write") + (root / "linked").chmod(0o600) + os.utime(root / "a", ns=(555, 555)) + self.assertEqual((root / "linked").read_text(), "linked write") + self.assertEqual(stat.S_IMODE((root / "a").stat().st_mode), 0o600) + self.assertEqual((root / "linked").stat().st_mtime_ns, 555) + self.assertEqual((root / "b").read_text(), "identical") + (root / "b").write_text("independent write") + self.assertEqual((root / "a").read_text(), "linked write") + self.assertEqual(blob.read_bytes(), before[0]) + self.assertEqual(blob.stat().st_mtime_ns, before[1].st_mtime_ns) + self.assertEqual(blob.stat().st_mode, before[1].st_mode) + self.assertEqual(blob.stat().st_uid, before[1].st_uid) + + def test_reuse_and_metadata_only_changes(self) -> None: + # Changing only metadata must alter the complete manifest while reusing the parent's blob. + self.files() + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root([self.store]) as root: + (root / "b").chmod(0o600) + os.utime(root / "b", ns=(999, 999)) + storage.capture(root, child, [self.store], epoch=CAPTURE_EPOCH) + self.assertEqual(list((child / "blobs").iterdir()), []) + self.assertNotEqual( + (child / uapi16.MANIFEST).read_bytes(), (self.store / uapi16.MANIFEST).read_bytes() + ) + with storage.root([self.store, child]) as root: + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o600) + self.assertEqual((root / "b").stat().st_mtime_ns, 999) + + def test_deletions_opacity_and_replacements(self) -> None: + # Delete inherited paths, replace files and directories, and rename a fresh directory over + # an inherited directory. Capture the merged view so no native overlay markers persist. + (self.tree / "dir").mkdir() + (self.tree / "dir/old").write_text("old") + (self.tree / "deleted").write_text("deleted") + (self.tree / "file").write_text("file") + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root([self.store]) as root: + (root / "deleted").unlink() + shutil.rmtree(root / "dir") + (root / "fresh").mkdir() + (root / "fresh/new").write_text("new") + (root / "fresh").rename(root / "dir") + (root / "file").unlink() + (root / "file").mkdir() + (root / "file/child").write_text("child") + storage.capture(root, child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child]) as root: + self.assertFalse((root / "deleted").exists()) + self.assertEqual([p.name for p in (root / "dir").iterdir()], ["new"]) + self.assertEqual((root / "file/child").read_text(), "child") + shutil.rmtree(root / "file") + (root / "file").write_text("replacement") + grandchild = self.scratch / "grandchild" + storage.capture(root, grandchild, [self.store, child], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child, grandchild]) as root: + self.assertEqual((root / "file").read_text(), "replacement") + self.assertEqual({p.name for p in child.iterdir()}, {"blobs", uapi16.MANIFEST}) + self.assertNotIn(b"overlay.opaque", (child / uapi16.MANIFEST).read_bytes()) + + def test_child_stores_only_new_contents(self) -> None: + # A child reuses every inherited content digest and writes one blob for two new equal files. + self.files() + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root([self.store]) as root: + (root / "new").write_text("new contents") + (root / "new-independent").write_text("new contents") + storage.capture(root, child, [self.store], epoch=CAPTURE_EPOCH) + self.assertEqual(len(list((child / "blobs").iterdir())), 1) + with storage.root([self.store, child]) as root: + self.assertEqual((root / "a").read_text(), "identical") + self.assertFalse((root / "new").samefile(root / "new-independent")) + + def test_device_nodes(self) -> None: + # Preserve device numbers when the namespace permits mknod; unprivileged hosts may deny it. + try: + os.mknod(self.tree / "device", stat.S_IFCHR | 0o600, os.makedev(1, 3)) + except PermissionError: + self.skipTest("the sandbox does not permit device nodes") + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + with storage.root([self.store]) as root: + attributes = (root / "device").lstat() + self.assertTrue(stat.S_ISCHR(attributes.st_mode)) + self.assertEqual(attributes.st_rdev, os.makedev(1, 3)) + + def test_special_files(self) -> None: + # FIFOs and sockets need manifest inode types because Buck only stores regular blobs. + os.mkfifo(self.tree / "fifo", 0o600) + with socket.socket(socket.AF_UNIX) as sock: + sock.bind(str(self.tree / "sock")) + storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + with storage.root([self.store]) as root: + self.assertTrue(stat.S_ISFIFO((root / "fifo").lstat().st_mode)) + self.assertTrue(stat.S_ISSOCK((root / "sock").lstat().st_mode)) + self.assertEqual(stat.S_IMODE((root / "fifo").stat().st_mode), 0o600) + + +if __name__ == "__main__": + unittest.main() diff --git a/uapi16/uapi16.py b/uapi16/uapi16.py index f9d7d51e..eb8690fb 100644 --- a/uapi16/uapi16.py +++ b/uapi16/uapi16.py @@ -197,6 +197,28 @@ def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object obj["mTimeNSec"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) if entry.st.st_mtime_ns < 0: fail(f"manifest: {entry.name!r} has a timestamp before the UNIX epoch") + attributes: dict[str, Xattr] = {} + for name in sorted(os.listxattr(entry.path, follow_symlinks=False)): + if not name.startswith("user.") or name.startswith("user.overlay."): + continue + short = name.removeprefix("user.") + if not short or any(ord(c) < 32 or ord(c) == 127 for c in short): + fail(f"manifest: unsupported extended attribute name {name!r}") + try: + short.encode("utf-8") + except UnicodeEncodeError: + fail(f"manifest: unsupported extended attribute name {name!r}") + value = os.getxattr(entry.path, name, follow_symlinks=False) + try: + text = value.decode("utf-8") + except UnicodeDecodeError: + text = None + if text is None or "\0" in text: + attributes[short] = {"base64": base64.b64encode(value).decode("ascii")} + else: + attributes[short] = {"text": text} + if attributes: + obj["extendedAttributes"] = {"user": attributes} if token is not None: obj["inodeToken"] = token if kind == "lnk": diff --git a/uapi16/uapi16_test.py b/uapi16/uapi16_test.py index 00a7d594..3dabfab9 100644 --- a/uapi16/uapi16_test.py +++ b/uapi16/uapi16_test.py @@ -16,6 +16,7 @@ from collections.abc import Sequence from pathlib import Path from typing import override +from unittest import mock import uapi16 @@ -251,7 +252,8 @@ def device(self, mode: int, rdev: int) -> uapi16.Object: (mode, 0, 0, 1, 0, 0, 0, 0, EPOCH - 1, 0), {"st_rdev": rdev, "st_mtime_ns": (EPOCH - 1) * 10**9}, ) - return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) + with mock.patch.object(uapi16.os, "listxattr", return_value=[]): + return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -> None: self.assertEqual( From dc4de622fe1401d044ec938e78162938be77781b Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 7 Nov 2026 17:06:18 +0100 Subject: [PATCH 3/7] image: Capture metadata before publishing image outputs Image layers lose package and operation metadata when Buck stores their directory artifacts. Let's capture snapshots in the producing action and restore them for image commands, solvers and terminals while keeping box layers on their direct mounting path. Signed-off-by: Daan De Meyer --- README.md | 2 +- docs/design/architecture.md | 8 ++-- docs/user/images.md | 8 ++-- examples/image/BUCK | 59 ++------------------------- image/BUCK | 5 ++- image/ddi.py | 16 ++++---- image/finalize.py | 39 ++++++++++++------ image/image.bzl | 8 ++-- image/layer.py | 72 ++++++++++----------------------- image/layer_test.py | 75 ++++++++++++++++++++++++++++++++--- package/BUCK | 2 +- package/transaction.py | 17 ++++++++ package_system/deb/BUCK | 2 - package_system/deb/plan.py | 3 +- package_system/pacman/BUCK | 3 +- package_system/pacman/plan.py | 3 +- package_system/rpm/BUCK | 4 +- package_system/rpm/plan.py | 3 +- rootfs/rootfs.py | 16 ++++++-- uapi16/uapi16.py | 4 +- 20 files changed, 181 insertions(+), 168 deletions(-) diff --git a/README.md b/README.md index ec0e9a12..64e2fc93 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: MPL-2.0 tine composes operating-system images out of packages and pinned third-party repositories with [Buck2](https://buck2.build/). Every build action runs in an unprivileged sandbox without network access. -Filesystem images are stacks of overlay deltas. tine's primary output targets are [unified kernel +Each image layer describes a complete filesystem snapshot. tine's primary output targets are [unified kernel images](https://uapi-group.org/specifications/specs/unified_kernel_image/) and dm-verity protected GPT disks in various formats, and also additional artifacts like [SBOM](https://en.wikipedia.org/wiki/Software_supply_chain). diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 8518853e..9cc57b50 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -1520,10 +1520,10 @@ A package installs configuration that describes state rather than carrying it: m index, `hwdb.d` with no compiled database, a locale list with no archive. On a running system scriptlets and boot-time units produce it; an image being assembled is neither, so tine builds it explicitly. Each generator is an operation of its own rather than a step of one pass, so it is positioned where the image wants it, -configured on its own terms, and adopted one at a time; what it writes persists into the delta that captures -it, so it is built once and cached with the layer rather than repeated by every terminal output. An `image` -gets the content its operations ask for and nothing else, which is the same reason terminal rules leave -package state alone. +configured on its own terms, and adopted one at a time; what it writes persists into the snapshot that +captures it, so it is built once and cached with the layer rather than repeated by every terminal output. An +`image` gets the content its operations ask for and nothing else, which is the same reason terminal rules +leave package state alone. Each generator is a no-op for an image that carries none of what it acts on, so none of them needs per-distribution knowledge: `image.ops.depmod()` is silent without kernels, `image.ops.hwdb()` without diff --git a/docs/user/images.md b/docs/user/images.md index 574318c7..0218e51a 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -182,7 +182,7 @@ be trusted because procfs magic links can access paths outside the image. `image.layer` has two construction modes. An initial image takes `package_manager`, which defaults to the package manager of the distribution that the build chooses. A derived image takes `parent` and inherits the package manager of that image. A call with operations applies one ordered operation sequence in one action -and persists exactly one delta: +and persists exactly one snapshot: - `packages` and `package_sets` are rule attributes rather than operations: a layer installs them as one request before any of its operations run, so every operation sees what the layer adds. `package_sets` @@ -203,7 +203,7 @@ and persists exactly one delta: project like any other artifact. It takes the same `env` and `chroot` arguments, so the script either sees the image at `/buildroot` or has it as its own root; in the chrooted case nothing of the interpreter reaches the - delta, because the project bind carrying it lives under `/run`. + snapshot, because the project bind carrying it lives under `/run`. - `image.ops.copy` introduces a declared Buck artifact at an absolute image path; `image.ops.mkdir` and `image.ops.symlink` mutate the same root. `image.ops.remove` treats its absolute path as a glob pattern, including recursive `**`, and removes every matching file, symlink, or directory tree. A pattern that @@ -377,7 +377,7 @@ database whenever the image has a package manager. The same artifacts are expose ``` The `version` attribute sets the SBOM source version and defaults to `"0"`. Merely building the image still -produces only its latest delta. Selecting `[pkgdb]` runs the image's package system's capture driver; +produces only its latest snapshot. Selecting `[pkgdb]` runs the image's package system's capture driver; selecting either nested SBOM format runs one shared scan of the completed stack. `[manifest]` is a [UAPI.16 File Manifest](https://uapi-group.org/specifications/): an RFC7464 JSON-SEQ @@ -1212,7 +1212,7 @@ tine buck build //examples/image:demo-ext.fedora tine buck build //examples/image:demo-portable.fedora ``` -These validate package installation and commands sharing one delta, incremental layering, archive packing, +These validate package installation and commands sharing one layer, incremental layering, archive packing, versioned UKI creation, semantic boot-artifact extraction, ESP-layer assembly, and disk composition. `demo-ext` is a system extension and `demo-confext` a configuration extension on top of `boot-demo`. `demo-portable` is a portable service image, and `demo-portable-ext` an extension for it. Running diff --git a/examples/image/BUCK b/examples/image/BUCK index 71d8a8c0..c01a3bad 100644 --- a/examples/image/BUCK +++ b/examples/image/BUCK @@ -90,14 +90,14 @@ image.archive( format = "tar", ) -# Base-only image exercising a single-component delta stack. +# An image with a single layer. image.archive( name = "base", packages = ["bash"], format = "tar", ) -# Chained images: the child adds one file and whiteouts another. +# The child image adds one file and removes a file of its parent. image.layer( name = "chained-base", packages = ["bash", "coreutils"], @@ -122,7 +122,7 @@ image.archive( format = "tar", ) -# Package installation and mutations sharing one physical delta. +# One layer installs packages and runs commands. image.layer( name = "layered-install-base", packages = ["bash"], @@ -146,35 +146,6 @@ image.archive( format = "tar", ) -# Round-trip an escaped name and an escaped whiteout through stored deltas. -image.layer( - name = "escaped-base", - packages = ["bash", "coreutils"], - ops = [ - image.ops.mkdir("/etc/tine"), - image.ops.run( - [ - "/usr/bin/bash", - "-c", - "echo kept > '/etc/tine/esc\\x2dkept'; echo gone > '/etc/tine/esc\\x2dgone'", - ], - chroot = True, - ), - ], -) - -image.layer( - name = "escaped-child.layer", - parent = ":escaped-base", - ops = [image.ops.remove("/etc/tine/esc\\x2dgone")], -) - -image.archive( - name = "escaped", - parent = ":escaped-child.layer", - format = "tar", -) - # Bootable GPT image; `buck run :boot-demo-vm` starts it with the box's vmspawn stack. # Supply-chain and format subtargets such as `:boot-demo[sbom]` and `:boot-demo[qcow2]` build on demand. # The version stamps os-release, partition labels, and the UKI and disk file names. A product names @@ -532,30 +503,6 @@ box.sh_test( args = ["$(location :tree)"] + distribution.select({d["release"]: [name if name == "arch" else "none"] for name, d in DISTRIBUTIONS.items()}), ) -# Replacing a populated directory exercises the opaque-dir marker. -image.layer( - name = "opaque-base", - packages = ["bash", "coreutils"], - ops = [ - image.ops.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo old > /etc/tine/sub/old"], chroot = True), - ], -) - -image.layer( - name = "opaque-child.layer", - parent = ":opaque-base", - ops = [ - image.ops.remove("/etc/tine/sub"), - image.ops.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo new > /etc/tine/sub/fresh"], chroot = True), - ], -) - -image.archive( - name = "opaque", - parent = ":opaque-child.layer", - format = "tar", -) - # The UKI's module selection still carries what the demo image boots through. The smoke proves the # same thing by booting; this pins it against the real kernel without a VM, so a change to the # default list that drops one fails here with the module named. Only modules Fedora builds as diff --git a/image/BUCK b/image/BUCK index 2ef62314..3c400cd8 100644 --- a/image/BUCK +++ b/image/BUCK @@ -35,6 +35,7 @@ python.binary( "//box:isolation", "//package:installer", "//rootfs:rootfs", + "//uapi16:storage", ], visibility = ["PUBLIC"], ) @@ -48,6 +49,7 @@ python.binary( "//:util", "//archive:cpio", "//archive:tar", + "//uapi16:storage", ], visibility = ["PUBLIC"], ) @@ -60,6 +62,7 @@ python.binary( "//:specs", "//:util", "//rootfs:rootfs", + "//uapi16:storage", "//uapi16:uapi16", ], visibility = ["PUBLIC"], @@ -175,7 +178,7 @@ python.binary( python.library( name = "finalize", srcs = ["finalize.py"], - deps = ["//:util", "//rootfs:rootfs"], + deps = ["//:util", "//rootfs:rootfs", "//uapi16:storage"], visibility = ["PUBLIC"], ) diff --git a/image/ddi.py b/image/ddi.py index df6685c5..26a3a338 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -21,7 +21,7 @@ import util import finalize -import rootfs +import storage import uapi16 _SEED_NAMESPACE = uuid.UUID("5af2de99-4f9f-4e0b-a04b-bde36b068c4f") @@ -291,24 +291,20 @@ def pin_release(parent: dict[str, str], release: dict[str, str]) -> dict[str, st def _extension_layers(spec: Spec, extension: ExtensionSpec) -> tuple[list[Release], list[str | Path]]: - """Return the extension-release files and the layers the extension packages. - - With a parent, the fields are pinned to the parent's os-release and the layers are the delta - above it. Without a parent, the function returns the releases and layers of the spec unchanged. - """ + """Pin release fields to the parent and retain every store needed to resolve contents.""" releases = extension["releases"] parent = extension["parent"] lowers: list[str | Path] = list(spec["lower"]) if not parent: return releases, lowers - with rootfs.rootfs("/buildroot", lowers=lowers) as tree: + with storage.root(lowers[:parent]) as tree: fields = _os_release(tree) if "ID" not in fields: util.fail("ddi: the parent os-release lacks ID") pinned = [ Release(path=release["path"], fields=pin_release(fields, release["fields"])) for release in releases ] - return pinned, lowers[parent:] + return pinned, lowers def empty_directories(tree: Path, directories: list[str]) -> None: @@ -477,7 +473,9 @@ def main(argv: list[str] | None = None) -> None: if extension: releases, lowers = _extension_layers(spec, extension) with ( - finalize.image(spec, program="ddi", lowers=lowers, binds=binds) as tree, + finalize.image( + spec, program="ddi", lowers=lowers, binds=binds, base=extension["parent"] if extension else 0 + ) as tree, tempfile.TemporaryDirectory(prefix="repart.") as scratch_dir, ): if extension: diff --git a/image/finalize.py b/image/finalize.py index a27e6015..ff49fe24 100644 --- a/image/finalize.py +++ b/image/finalize.py @@ -15,6 +15,7 @@ import gzip import os import shutil +import stat import subprocess import sys import tempfile @@ -25,6 +26,7 @@ from util import fail import rootfs +import storage class ImageSpec(TypedDict): @@ -34,21 +36,31 @@ class ImageSpec(TypedDict): tmpfiles: list[str] -# The directories whose whole point is a mode, so a layer cannot carry them: sticky and -# world-writable, which is exactly what Buck cannot store and the tar filter strips. +def _normalize_mode(path: Path) -> None: + """Normalize file permissions for path. + + See docs/user/images.md. + """ + mode = path.lstat().st_mode + desired = 0o755 if stat.S_ISDIR(mode) or mode & 0o111 else 0o644 + if (stat.S_ISDIR(mode) or stat.S_ISREG(mode)) and stat.S_IMODE(mode) != desired: + path.chmod(desired) + + +def normalize_modes(tree: Path) -> None: + _normalize_mode(tree) + for path in tree.rglob("*"): + _normalize_mode(path) + + _WORLD_WRITABLE = ("tmp", "var/tmp") def world_writable(tree: Path) -> None: - """Restore the mode on the temporary directories, which nothing upstream of here can keep. - - Buck stores no mode beyond the executable bit, so a package's `0o1777` is gone by the time a - layer is a build artifact however faithfully it was extracted. Here is the last point before - the image is written, and the only one where the mode survives into it. - """ + """Apply the temporary-directory exception to the image permission policy.""" for name in _WORLD_WRITABLE: path = tree / name - if path.is_dir() and not path.is_symlink(): + if path.is_dir() and not path.is_symlink() and stat.S_IMODE(path.stat().st_mode) != 0o1777: path.chmod(0o1777) @@ -59,11 +71,14 @@ def image( program: str, lowers: Sequence[str | Path] | None = None, binds: Sequence[tuple[str | Path, str | Path]] | None = None, + base: int = 0, ) -> Generator[Path]: """Mount the image stack the spec names and yield the finalized tree.""" - with rootfs.rootfs( - "/buildroot", lowers=spec["lower"] if lowers is None else lowers, binds=binds - ) as tree: + with ( + storage.root(spec["lower"] if lowers is None else lowers, base=base) as restored, + rootfs.rootfs("/buildroot", bind=restored, binds=binds) as tree, + ): + normalize_modes(tree) # Before the snippets, so an image that states a mode of its own still wins. world_writable(tree) apply_tmpfiles(tree, spec["tmpfiles"], program=program) diff --git a/image/image.bzl b/image/image.bzl index d625d4e5..cfe57182 100644 --- a/image/image.bzl +++ b/image/image.bzl @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Logical filesystem images built as ordered overlay deltas.""" +"""Logical filesystem images stored as complete snapshots and immutable blobs.""" load("//:specs.bzl", "specs") load("//box:defs.bzl", "box") @@ -120,7 +120,7 @@ ImageSbomInfo = provider( ) ImageInfo = provider( - doc = "A logical filesystem image represented by an ordered delta stack and lazy metadata.", + doc = "A logical filesystem image represented by snapshot stores and lazy metadata.", fields = { "box": provider_field(Dependency), # Accumulated install specs; they seed the local-packages closure of every derived layer @@ -645,14 +645,12 @@ def declare_image( ) package_manager_info = package_manager[PackageManagerInfo] - delta = declare_out(ctx, identifier, "delta", dir = True) - work = declare_out(ctx, identifier, "overlay.work", dir = True) if layers else None + delta = declare_out(ctx, identifier, "snapshot", dir = True) spec = { "install": None, "lower": layers, "operations": [_encode_operation(operation, tools.python) for operation in operations], "out": delta.as_output(), - "work": work.as_output() if work != None else None, } if package_manager_info != None: system = package_manager_info.package_system[PackageSystemInfo] diff --git a/image/layer.py b/image/layer.py index f9408036..e0360dbd 100755 --- a/image/layer.py +++ b/image/layer.py @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Apply ordered operations against one mounted root and capture its overlay delta.""" +"""Apply ordered operations against one mounted root and capture its complete snapshot.""" import glob # noqa: F401 # Preload for Path.glob before entering the image chroot. import os @@ -25,6 +25,7 @@ import finalize import installer import rootfs +import storage from isolation import AT_FDCWD, Bind, Close, open_directory @@ -41,7 +42,6 @@ class LayerInstall(TypedDict): class Spec(TypedDict): lower: list[str] out: str - work: str | None install: LayerInstall | None operations: list[object] @@ -149,25 +149,6 @@ def _remove_glob(tree: Path, value: str) -> None: util.remove_path(path) -def _normalize_mode(path: Path) -> None: - """Normalize file permissions for path. - - See docs/user/images.md. - """ - mode = path.lstat().st_mode - if stat.S_ISDIR(mode): - path.chmod(0o755) - elif stat.S_ISREG(mode): - path.chmod(0o755 if mode & 0o111 else 0o644) - # ignore symlinks (meaningless mode), whiteout nodes, and non-files - - -def _normalize_modes(tree: Path) -> None: - _normalize_mode(tree) - for path in tree.rglob("*"): - _normalize_mode(path) - - # The ELF fields the stack check reads; see elf(5). Per class: the header after e_ident up to # e_phnum, and the offset of p_flags in a program header. Per data encoding: the byte order. _ELF_CLASSES = {1: ("HHIIIIIHHH", 24), 2: ("HHIQQQIHHH", 4)} @@ -311,37 +292,24 @@ def main(argv: list[str] | None = None) -> None: install = spec["install"] lower = spec["lower"] - if lower: - if spec["work"] is None: - util.fail("image lower stack needs a work overlay directory") - mounted = rootfs.rootfs( - "/buildroot", - lowers=lower, - upperdir=out, - workdir=spec["work"], - apivfs=True, - ) - else: - if spec["work"] is not None: - util.fail("image work overlay directory requires a lower stack") - mounted = rootfs.rootfs( - "/buildroot", - bind=out, - capture_bind=True, - apivfs=True, - ) - - with mounted as target, tempfile.TemporaryDirectory(prefix="layer.") as scratch: - # Packages land before the operations so every one of them sees what this layer installs. - if install is not None: - _install(install, target, Path(scratch)) - for operation in operations: - _apply(operation, target) - _normalize_modes(out) - stacks = _executable_stacks(out) - if stacks: - util.fail("image: executable stacks are not allowed, see docs/user/images.md:\n" + "\n".join(stacks)) - print(f"image: applied {len(operations)} ops over {len(lower)} lower(s) -> {out}", file=sys.stderr) + with ( + storage.root(lower) as tree, + tempfile.TemporaryDirectory(prefix="layer.", dir="/var/tmp") as scratch, + ): + with rootfs.rootfs("/buildroot", bind=tree, apivfs=True, strict_unmount=True) as target: + if install is not None: + _install(install, target, Path(scratch)) + for operation in operations: + _apply(operation, target) + stacks = _executable_stacks(tree) + if stacks: + util.fail( + "image: executable stacks are not allowed, see docs/user/images.md:\n" + "\n".join(stacks) + ) + storage.capture(tree, out, lower, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) + print( + f"image: applied {len(operations)} ops over {len(lower)} parent store(s) -> {out}", file=sys.stderr + ) if __name__ == "__main__": diff --git a/image/layer_test.py b/image/layer_test.py index 8cae59e8..7e3ace60 100644 --- a/image/layer_test.py +++ b/image/layer_test.py @@ -8,6 +8,8 @@ import errno import os +import shutil +import stat import struct import tempfile import unittest @@ -18,8 +20,15 @@ import specs +import finalize import layer import rootfs +import storage +import uapi16 +from isolation import umount2 + +# Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. +CAPTURE_EPOCH = 2**31 class TestImageWrites(unittest.TestCase): @@ -33,6 +42,8 @@ def setUp(self) -> None: self.out = self.scratch / "out" self.out.mkdir() (self.project / "source").write_text("copied", encoding="utf-8") + # The sandbox sets SOURCE_DATE_EPOCH for every action, and layer.main() clamps times to it. + self.enterContext(mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": str(2**31)})) def _build(self, operations: list[object], lower: Path | None = None) -> None: spec = specs.write( @@ -40,7 +51,6 @@ def _build(self, operations: list[object], lower: Path | None = None) -> None: { "out": str(self.out), "lower": [str(lower)] if lower is not None else [], - "work": str(self.scratch / "work") if lower is not None else None, "install": None, "operations": operations, }, @@ -74,10 +84,13 @@ def test_a_copy_source_must_stay_relative_to_the_project(self) -> None: layer._apply(["copy", source, "/destination"], self.out) def test_a_derived_layer_exposes_a_readonly_project_only_to_copy_and_run(self) -> None: + # Copy and run must see a read-only project, while removals affect only the image snapshot. lower = self.scratch / "lower" lower.mkdir() (lower / "cache.pyc").write_text("image cache", encoding="utf-8") (self.project / "cache.pyc").write_text("cache", encoding="utf-8") + stored = self.scratch / "stored" + storage.capture(lower, stored, epoch=CAPTURE_EPOCH) def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: self.assertEqual(cwd, layer.PROJECT) @@ -94,14 +107,64 @@ def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: ["run", ["command"], {}, True], ["remove", "/**/*.pyc"], ], - lower, + stored, ) command.assert_called_once() - self.assertEqual((self.out / "copied").read_text(encoding="utf-8"), "copied") - self.assertTrue((self.out / ".wh.cache.pyc").is_file()) + with storage.root([stored, self.out]) as tree: + self.assertEqual((tree / "copied").read_text(encoding="utf-8"), "copied") + self.assertFalse((tree / "cache.pyc").exists()) self.assertTrue((self.project / "cache.pyc").exists()) + def test_commands_see_inherited_metadata_and_can_override_it(self) -> None: + # Capture before artifact copying, then check a command sees the original metadata and + # explicit hardlinks. Its metadata changes must become authoritative in the child snapshot. + source = self.scratch / "source-root" + source.mkdir() + (source / "file").write_text("contents") + (source / "file").chmod(0o640) + os.utime(source / "file", ns=(123, 123)) + os.link(source / "file", source / "linked") + parent = self.scratch / "parent" + storage.capture(source, parent, epoch=CAPTURE_EPOCH) + copied = self.scratch / "copied-parent" + shutil.copytree(parent, copied, copy_function=shutil.copyfile) + + def command(*args: object) -> None: + path = Path("/buildroot/file") + self.assertEqual(path.stat().st_mtime_ns, 123) + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640) + self.assertTrue(path.samefile("/buildroot/linked")) + path.chmod(0o600) + os.utime(path, ns=(456, 456)) + + with mock.patch.object(layer, "_run", side_effect=command): + self._build([["run", ["command"], {}, False]], copied) + with storage.root([copied, self.out]) as tree: + self.assertEqual(stat.S_IMODE((tree / "file").stat().st_mode), 0o600) + self.assertEqual((tree / "linked").stat().st_mtime_ns, 456) + self.assertEqual(list((self.out / "blobs").iterdir()), []) + + def test_a_live_writer_prevents_capture(self) -> None: + # A command retains an open image file after it returns. Strict unmount must fail before + # capture can publish contents that the retained descriptor could still change. + descriptor = -1 + + def command(*args: object) -> None: + nonlocal descriptor + descriptor = os.open("/buildroot/file", os.O_WRONLY | os.O_CREAT, 0o644) + + try: + with mock.patch.object(layer, "_run", side_effect=command), self.assertRaises(OSError) as error: + self._build([["run", ["command"], {}, False]]) + self.assertEqual(error.exception.errno, errno.EBUSY) + self.assertFalse((self.out / uapi16.MANIFEST).exists()) + finally: + if descriptor >= 0: + os.close(descriptor) + if Path("/buildroot").is_mount(): + umount2("/buildroot", 0) + class TestRemove(unittest.TestCase): def test_plain_path_is_also_a_glob(self) -> None: @@ -159,6 +222,8 @@ def test_pattern_must_be_an_absolute_image_path(self) -> None: class TestNormalization(unittest.TestCase): def test_modes(self) -> None: + # Terminal policy normalizes regular files and directories without following links or + # changing special-file modes. Authored tmpfiles rules run after this normalization. with tempfile.TemporaryDirectory(prefix="layer-test.", dir="/var/tmp") as scratch: tree = Path(scratch) tree.chmod(0o700) @@ -175,7 +240,7 @@ def test_modes(self) -> None: (tree / "link").symlink_to("sub/secret") os.mkfifo(tree / "fifo", 0o600) - layer._normalize_modes(tree) + finalize.normalize_modes(tree) self.assertEqual(tree.stat().st_mode & 0o7777, 0o755) self.assertEqual((tree / "sub").stat().st_mode & 0o7777, 0o755) diff --git a/package/BUCK b/package/BUCK index b936ac1e..4825c08d 100644 --- a/package/BUCK +++ b/package/BUCK @@ -50,7 +50,7 @@ python.library( python.library( name = "transaction", srcs = ["transaction.py"], - deps = [":snapshotter", "//:util"], + deps = [":snapshotter", "//:util", "//rootfs:rootfs", "//uapi16:storage"], visibility = ["PUBLIC"], ) diff --git a/package/transaction.py b/package/transaction.py index ee6632a1..26272f22 100644 --- a/package/transaction.py +++ b/package/transaction.py @@ -10,12 +10,17 @@ """ import json +from collections.abc import Generator, Sequence +from contextlib import contextmanager from pathlib import Path from typing import Literal, NamedTuple, NotRequired, TypedDict from util import fail, text_destination +import rootfs import snapshotter +import storage +import uapi16 from snapshotter import MetadataFile @@ -170,3 +175,15 @@ def write(path: Path, packages: list[TransactionPackage], repositories: list[Rep with text_destination(path) as output: json.dump(entries, output, indent=2) output.write("\n") + + +@contextmanager +def installed_root(lowers: Sequence[str]) -> Generator[Path]: + # A snapshot store contains `uapi16.MANIFEST` at its top level. A box layer is a plain directory + # tree, and no package installs a file of that name into the root. + if (Path(lowers[-1]) / uapi16.MANIFEST).exists(): + with storage.root(lowers) as tree: + yield tree + else: + with rootfs.rootfs("/installroot", lowers=lowers) as tree: + yield tree diff --git a/package_system/deb/BUCK b/package_system/deb/BUCK index 9b3ab39e..2bc7ece4 100644 --- a/package_system/deb/BUCK +++ b/package_system/deb/BUCK @@ -46,7 +46,6 @@ python.binary( "//:util", "//package:href", "//package:transaction", - "//rootfs:rootfs", ], ) @@ -134,6 +133,5 @@ python.test( "//package:installer", "//package:snapshotter", "//package:transaction", - "//rootfs:rootfs", ], ) diff --git a/package_system/deb/plan.py b/package_system/deb/plan.py index 3f692fee..aa76c968 100755 --- a/package_system/deb/plan.py +++ b/package_system/deb/plan.py @@ -25,7 +25,6 @@ import util from util import MAGIC, decompressor -import rootfs import transaction from href import relative_href @@ -195,7 +194,7 @@ def solve( scratch = Path(stack.enter_context(TemporaryDirectory(prefix="deb-plan.", dir="/var/tmp"))) root = scratch / "root" if lower: - root = stack.enter_context(rootfs.rootfs("/installroot", lowers=lower)) + root = stack.enter_context(transaction.installed_root(lower)) status = root / "var/lib/dpkg/status" status.parent.mkdir(parents=True, exist_ok=True) status.touch(exist_ok=True) diff --git a/package_system/pacman/BUCK b/package_system/pacman/BUCK index 2ffed2c5..dc649268 100644 --- a/package_system/pacman/BUCK +++ b/package_system/pacman/BUCK @@ -46,7 +46,6 @@ python.binary( "//:util", "//package:href", "//package:transaction", - "//rootfs:rootfs", ], ) @@ -100,7 +99,7 @@ python.test( name = "plan-test", box = "//tools:dev.box", srcs = ["alpm.py", "plan.py", "plan_test.py"], - deps = ["//:specs", "//:util", "//package:href", "//package:transaction", "//rootfs:rootfs"], + deps = ["//:specs", "//:util", "//package:href", "//package:transaction"], ) python.test( diff --git a/package_system/pacman/plan.py b/package_system/pacman/plan.py index a2f9d271..5f5bb094 100755 --- a/package_system/pacman/plan.py +++ b/package_system/pacman/plan.py @@ -27,7 +27,6 @@ from util import fail import alpm -import rootfs import transaction from href import relative_href @@ -111,7 +110,7 @@ def main(argv: list[str] | None = None) -> None: scratch = Path(stack.enter_context(TemporaryDirectory(prefix="plan."))) if spec["lower"]: # An ephemeral upper keeps the lower stack unchanged while alpm writes into the root. - root = stack.enter_context(rootfs.rootfs("/installroot", lowers=spec["lower"])) + root = stack.enter_context(transaction.installed_root(spec["lower"])) else: root = scratch / "root" root.mkdir() diff --git a/package_system/rpm/BUCK b/package_system/rpm/BUCK index 2b9c8ab2..f1df5f52 100644 --- a/package_system/rpm/BUCK +++ b/package_system/rpm/BUCK @@ -35,7 +35,7 @@ python.binary( name = "plan", main = "plan.py", boxes = [_BOX], - deps = [":macros", "//:specs", "//:util", "//package:transaction", "//rootfs:rootfs"], + deps = [":macros", "//:specs", "//:util", "//package:transaction"], ) python.library(name = "rpmbuild", srcs = ["rpmbuild.py"], deps = ["//:util", "//package:builder"]) @@ -96,7 +96,7 @@ python.test( name = "plan-test", box = _BOX, srcs = ["createrepo.py", "plan.py", "plan_test.py"], - deps = [":macros", "//:specs", "//:util", "//package:transaction", "//rootfs:rootfs"], + deps = [":macros", "//:specs", "//:util", "//package:transaction"], ) python.test( diff --git a/package_system/rpm/plan.py b/package_system/rpm/plan.py index 9045461d..bc303313 100755 --- a/package_system/rpm/plan.py +++ b/package_system/rpm/plan.py @@ -23,7 +23,6 @@ from util import fail import macros -import rootfs import transaction # A multilib package in a pinned-arch transaction indicates a bad solve. @@ -250,7 +249,7 @@ def main(argv: list[str] | None = None) -> None: installroot = None if solve_spec["lower"]: # An ephemeral upper keeps the lower stack unchanged. - installroot = stack.enter_context(rootfs.rootfs("/installroot", lowers=solve_spec["lower"])) + installroot = stack.enter_context(transaction.installed_root(solve_spec["lower"])) tx = plan( repos, install, diff --git a/rootfs/rootfs.py b/rootfs/rootfs.py index 45f8479a..89356cfc 100644 --- a/rootfs/rootfs.py +++ b/rootfs/rootfs.py @@ -349,6 +349,7 @@ def rootfs( apivfs: bool = False, binds: Sequence[tuple[str | Path, str | Path]] | None = None, chroot: bool = False, + strict_unmount: bool = False, ) -> Generator[Path]: """Mount a bind or overlay root, optionally chrooting and persisting its output. @@ -382,17 +383,24 @@ def rootfs( # Enter before registering the unmount so capture runs after it. stack.enter_context(capture_on_exit(upper, resolved)) # capture() reads and rewrites the upperdir after the unmount, so a captured upper needs a - # strict unmount. Nobody reads an ephemeral upper, so it is unmounted lazily like every - # other mount. + # strict unmount. mounted: Overlay | Bind = stack.enter_context( - Overlay(tuple(components), upper, work, target, strict_unmount=upperdir is not None) + Overlay( + tuple(components), + upper, + work, + target, + strict_unmount=upperdir is not None or strict_unmount, + ) ) elif bind is not None: if capture_bind: # Enter before registering the unmount so capture runs after it. stack.enter_context(capture_on_exit(bind)) # capture() reads the bind source after the unmount, like the upper of an overlay. - mounted = stack.enter_context(Bind(Path(bind), target, strict_unmount=capture_bind)) + mounted = stack.enter_context( + Bind(Path(bind), target, strict_unmount=capture_bind or strict_unmount) + ) else: raise ValueError("rootfs needs bind= or lowers=") mounted.mount() diff --git a/uapi16/uapi16.py b/uapi16/uapi16.py index eb8690fb..16fc9fbd 100644 --- a/uapi16/uapi16.py +++ b/uapi16/uapi16.py @@ -243,8 +243,8 @@ def _record(obj: Object) -> bytes: def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: """The whole sequence for `tree`: the root object, then one object per inode below it.""" # The root object carries the media type alone, which is what marks the sequence a manifest. - # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the - # format defaults a nameless first object to a directory anyway. + # A partition manifest lists a part of the image, and the image root is not part of it. The format + # defaults a nameless first object to a directory. yield _record({"mediaType": MEDIA_TYPE}) found = list(entries(tree, roots)) tokens = hardlink_tokens(found) From a783e046b3c7147137be045e19786e8156b71c6b Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 7 Nov 2026 17:15:08 +0100 Subject: [PATCH 4/7] image: Export snapshots without artifact metadata Terminal archives and directory artifacts can lose metadata or mistake shared contents for shared inodes. Let's write archives from snapshots, keep terminal listings consistent with their outputs, and restore directory exports only after Buck has materialized their manifests and blobs. Signed-off-by: Daan De Meyer --- README.md | 1 + archive/BUCK | 5 +- archive/cpio.py | 169 ++++++++++++-- archive/cpio_test.py | 66 ++++++ archive/tar.py | 52 ++++- docs/design/architecture.md | 7 +- docs/user/images.md | 22 +- examples/image/expected.json | 14 +- examples/image/manifest_check.py | 2 +- image/BUCK | 8 +- image/archive.bzl | 19 +- image/archive.py | 108 +++++---- image/archive_test.py | 378 +++++++++++++++++++++++++++++++ image/compose.bzl | 10 +- image/ddi.bzl | 4 +- image/extension.bzl | 2 +- image/finalize.py | 59 +++++ image/manifest_main.py | 4 +- image/uki.py | 2 +- uapi16/storage.py | 59 ++++- uapi16/uapi16.py | 31 ++- util.py | 2 +- 22 files changed, 910 insertions(+), 114 deletions(-) create mode 100644 image/archive_test.py diff --git a/README.md b/README.md index 64e2fc93..3b2d84ec 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The host needs only: * `git` * `/bin/sh`, which runs the fetch of a Rust project's git dependencies * `/dev/kvm` to run VMs for testing; without it, you can still build everything + * Read access to `/proc/sys/kernel/overflowuid` for directory exports tine uses the host's `python` and `git` to bootstrap itself and to read the project's git state, such as the version an image derives from it. A build also runs the host's `git` to fetch git checkouts, and the host's diff --git a/archive/BUCK b/archive/BUCK index 494734f3..643d606b 100644 --- a/archive/BUCK +++ b/archive/BUCK @@ -9,18 +9,19 @@ python.library( name = "cpio", srcs = ["cpio.py"], visibility = ["tine//..."], - deps = ["//:util"], + deps = ["//:util", "//uapi16:uapi16"], ) python.library( name = "tar", srcs = ["tar.py"], visibility = ["tine//..."], + deps = ["//uapi16:uapi16"], ) python.test( name = "test", box = "//tools:dev.box", srcs = ["cpio.py", "cpio_test.py", "tar.py", "tar_test.py"], - deps = ["//:util"], + deps = ["//:util", "//uapi16:uapi16"], ) diff --git a/archive/cpio.py b/archive/cpio.py index 1275a5bd..59ccb6ef 100644 --- a/archive/cpio.py +++ b/archive/cpio.py @@ -10,12 +10,15 @@ import mmap import os import stat -from collections.abc import Iterable, Iterator +from collections import Counter +from collections.abc import Callable, Iterable, Iterator, Sequence from pathlib import Path, PurePath, PurePosixPath from typing import NamedTuple, Self import util +import uapi16 + # What an archive is read from, spelled concretely because the reader slices it: an mmap and a # `bytes` both hand back `bytes`, where the `Buffer` protocol promises no indexing at all. type Source = mmap.mmap | bytes @@ -151,7 +154,7 @@ def _extract(source: Source, dest: Path, src_fd: int, start: int) -> int: elif fmt == 0o120000: # symlink (last-writer-wins across multi-archive extracts) target.parent.mkdir(parents=True, exist_ok=True) target.unlink(missing_ok=True) - target.symlink_to(e.target.decode()) + target.symlink_to(os.fsdecode(e.target)) elif fmt == 0o100000: # regular file _put_regular(target, e, src_fd) else: # device/fifo/socket @@ -163,10 +166,13 @@ def _extract(source: Source, dest: Path, src_fd: int, start: int) -> int: _hardlink(link, target) count += 1 for links in pending.values(): # a fully-empty hardlink set has no content carrier - for target, mode in links: - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(b"") - target.chmod(mode & 0o7777) + first, mode = links[0] + first.parent.mkdir(parents=True, exist_ok=True) + first.write_bytes(b"") + first.chmod(mode & 0o7777) + count += 1 + for target, _ in links[1:]: + _hardlink(target, first) count += 1 return count @@ -197,6 +203,7 @@ def __init__(self, path: Path, epoch: int, *, block_align: bool = True) -> None: self._epoch = epoch self._pos = 0 self._ino = 0 + self._links: dict[int, tuple[int, int]] = {} # Alignment pads the name field, and the kernel's early cpio reader (lib/earlycpio.c) refuses # an entry whose name field exceeds its 18-byte name buffer, so an archive it reads before # unpacking anything, microcode, is written without. @@ -221,8 +228,22 @@ def _pad(self, align: int) -> None: if pad := (-self._pos) % align: self._write(b"\x00" * pad) - def _header(self, name: str, mode: int, filesize: int, *, mtime: int, block_align: bool) -> None: + def _header( + self, + name: str, + mode: int, + filesize: int, + *, + mtime: int, + block_align: bool, + ino: int | None = None, + nlink: int = 1, + major: int = 0, + minor: int = 0, + ) -> None: self._ino += 1 + if name == TRAILER: + name = "./" + name raw = name.encode() namesize = len(raw) + 1 # name + its NUL terminator if block_align: @@ -231,21 +252,75 @@ def _header(self, name: str, mode: int, filesize: int, *, mtime: int, block_alig padded = _roundup(self._pos + _HEADER + namesize, _BLOCK) - (self._pos + _HEADER) if padded <= _PATH_MAX: namesize = padded - fields = (self._ino, mode, 0, 0, 1, min(mtime, self._epoch), filesize, 0, 0, 0, 0, namesize, 0) + fields = ( + self._ino if ino is None else ino, + mode, + 0, + 0, + nlink, + min(mtime, self._epoch), + filesize, + 0, + 0, + major, + minor, + namesize, + 0, + ) self._write(MAGIC + b"".join(b"%08x" % v for v in fields)) self._write(raw + b"\x00" * (namesize - len(raw))) self._pad(4) + def _link(self, link: tuple[int, int] | None) -> tuple[int | None, int, int]: + if link is None: + return None, 1, 1 + token, nlink = link + ino, remaining = self._links.get(token, (self._ino + 1, nlink)) + self._links[token] = (ino, remaining - 1) + return ino, nlink, remaining + def add_dir(self, name: str, *, mode: int, mtime: int) -> None: self._header(name, stat.S_IFDIR | (mode & 0o7777), 0, mtime=mtime, block_align=False) - def add_symlink(self, name: str, target: str, *, mtime: int) -> None: - data = target.encode() - self._header(name, stat.S_IFLNK | 0o777, len(data), mtime=mtime, block_align=False) + def add_symlink( + self, name: str, target: str, *, mtime: int, link: tuple[int, int] | None = None + ) -> None: + ino, nlink, _ = self._link(link) + data = os.fsencode(target) + self._header( + name, stat.S_IFLNK | 0o777, len(data), mtime=mtime, block_align=False, ino=ino, nlink=nlink + ) self._write(data) self._pad(4) - def add_file(self, name: str, src: Path, *, mode: int, mtime: int) -> None: + def add_node( + self, + name: str, + *, + mode: int, + mtime: int, + major: int = 0, + minor: int = 0, + link: tuple[int, int] | None = None, + ) -> None: + ino, nlink, _ = self._link(link) + self._header( + name, mode, 0, mtime=mtime, block_align=False, major=major, minor=minor, ino=ino, nlink=nlink + ) + + def add_file( + self, + name: str, + src: Path, + *, + mode: int, + mtime: int, + link: tuple[int, int] | None = None, + ) -> None: + ino, nlink, remaining = self._link(link) + if remaining > 1: + self._header(name, stat.S_IFREG | mode, 0, mtime=mtime, block_align=False, ino=ino, nlink=nlink) + return fd = os.open(src, os.O_RDONLY) try: size = os.fstat(fd).st_size @@ -255,6 +330,8 @@ def add_file(self, name: str, src: Path, *, mode: int, mtime: int) -> None: size, mtime=mtime, block_align=self._block_align and size >= _BLOCK, + ino=ino, + nlink=nlink, ) util.copy_range(self._fd, self._pos, fd, 0, size) self._pos += size @@ -274,31 +351,27 @@ def close(self) -> None: self._fd = -1 -def _add(w: Writer, tree: Path, path: Path) -> bool: +def _add(w: Writer, tree: Path, path: Path, link: tuple[int, int] | None = None) -> bool: """Add one path under `tree`; false for a type an initramfs has no use for.""" st = path.lstat() name = str(path.relative_to(tree)) mode = stat.S_IMODE(st.st_mode) - mtime = int(st.st_mtime) + mtime = st.st_mtime_ns // 10**9 if stat.S_ISDIR(st.st_mode): w.add_dir(name, mode=mode, mtime=mtime) elif stat.S_ISLNK(st.st_mode): w.add_symlink(name, os.readlink(path), mtime=mtime) elif stat.S_ISREG(st.st_mode): - w.add_file(name, path, mode=mode, mtime=mtime) + w.add_file(name, path, mode=mode, mtime=mtime, link=link) else: return False # devices/fifos/sockets: unneeded in an initramfs (systemd mounts /dev) return True -def pack_tree(tree: Path, out: Path, epoch: int, *, block_align: bool = True) -> int: - """Pack a whole tree into `out`.""" +def pack_tree(tree: Path, out: Path, epoch: int) -> int: + """Pack a whole tree into `out` without block alignment, which the kernel's early cpio reader needs.""" tree = Path(tree) - count = 0 - with Writer(out, epoch, block_align=block_align) as w: - for path in sorted(tree.rglob("*")): - count += _add(w, tree, path) - return count + return _pack_paths(tree, out, epoch, sorted(tree.rglob("*")), block_align=False) def pack_paths(tree: Path, out: Path, epoch: int, paths: Iterable[str | PurePath]) -> int: @@ -307,8 +380,54 @@ def pack_paths(tree: Path, out: Path, epoch: int, paths: Iterable[str | PurePath Sorting is what puts a directory before its contents, which the kernel's unpacker needs. """ tree = Path(tree) + return _pack_paths( + tree, out, epoch, [tree / rel for rel in sorted({PurePosixPath(path) for path in paths})] + ) + + +def _pack_paths( + tree: Path, out: Path, epoch: int, paths: Sequence[Path], *, block_align: bool = True +) -> int: + inodes = {path: path.lstat() for path in paths} + counts = Counter((st.st_dev, st.st_ino) for st in inodes.values() if stat.S_ISREG(st.st_mode)) + tokens: dict[tuple[int, int], int] = {} count = 0 - with Writer(out, epoch) as w: - for rel in sorted({PurePosixPath(path) for path in paths}): - count += _add(w, tree, tree / rel) + with Writer(out, epoch, block_align=block_align) as w: + for path, st in inodes.items(): + inode = st.st_dev, st.st_ino + link = None + if stat.S_ISREG(st.st_mode) and counts[inode] > 1: + link = (tokens.setdefault(inode, len(tokens) + 1), counts[inode]) + count += _add(w, tree, path, link) return count + + +def pack_manifest( + objects: Sequence[uapi16.Object], + content: Callable[[uapi16.Object], Path], + out: Path, + epoch: int, +) -> None: + counts = Counter(obj.get("inodeToken") for obj in objects if obj.get("inodeToken")) + kinds = {"fifo": stat.S_IFIFO, "chr": stat.S_IFCHR, "blk": stat.S_IFBLK, "sock": stat.S_IFSOCK} + with Writer(out, epoch) as w: + for obj in objects: + kind = obj["type"] + name, mode, mtime = obj.get("name", "."), obj.get("mode", 0o755), obj["mTimeNSec"] // 10**9 + token = obj.get("inodeToken") + link = (token, counts[token]) if token else None + if kind == "reg": + w.add_file(name, content(obj), mode=mode, mtime=mtime, link=link) + elif kind == "dir": + w.add_dir(name, mode=mode, mtime=mtime) + elif kind == "lnk": + w.add_symlink(name, uapi16.symlink_target(obj), mtime=mtime, link=link) + else: + w.add_node( + name, + mode=kinds[kind] | mode, + mtime=mtime, + major=obj.get("major", 0), + minor=obj.get("minor", 0), + link=link, + ) diff --git a/archive/cpio_test.py b/archive/cpio_test.py index 51f4a39e..92413d8d 100644 --- a/archive/cpio_test.py +++ b/archive/cpio_test.py @@ -28,6 +28,19 @@ def payload(data: bytes) -> bytes: class TestUnpack(unittest.TestCase): + def test_symlink_targets_need_not_be_utf8(self) -> None: + # A symlink target is an arbitrary byte string even when its filename is valid UTF-8. + # Packing and extracting the target must preserve its bytes without a Unicode error. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + archive = root / "image.cpio" + target = b"target-\xff" + with cpio.Writer(archive, 0) as writer: + writer.add_symlink("link", os.fsdecode(target), mtime=0) + with archive.open("rb") as stream: + cpio.unpack(stream.fileno(), root / "unpacked") + self.assertEqual(os.fsencode(os.readlink(root / "unpacked/link")), target) + def test_extracts_an_ordinary_hardlink_set(self) -> None: with tempfile.TemporaryDirectory() as scratch: root = Path(scratch) @@ -133,5 +146,58 @@ def test_an_unaligned_archive_keeps_the_stock_name_field(self) -> None: self.assertEqual(int(header[6 + 11 * 8 : 6 + 12 * 8], 16), namesize) +class TestPackPaths(unittest.TestCase): + def test_a_literal_trailer_name_does_not_end_the_archive(self) -> None: + # A regular file named TRAILER!!! must be escaped with a relative prefix so the reader + # extracts it and the later file instead of treating it as the archive terminator. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + tree = root / "tree" + tree.mkdir() + (tree / "TRAILER!!!").write_text("ordinary file") + (tree / "z-last").write_text("later file") + out = root / "image.cpio" + cpio.pack_tree(tree, out, 0) + self.assertEqual([obj.name for obj in cpio.read(out.read_bytes())], ["./TRAILER!!!", "z-last"]) + with out.open("rb") as stream: + cpio.unpack(stream.fileno(), root / "unpacked") + self.assertEqual((root / "unpacked/TRAILER!!!").read_text(), "ordinary file") + self.assertEqual((root / "unpacked/z-last").read_text(), "later file") + + def test_hardlinks_depend_on_selected_paths(self) -> None: + # A full selection preserves explicit links without merging equal files. Selecting one + # member of a group must emit a self-contained file with nlink=1 and its own payload. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + tree = root / "tree" + tree.mkdir() + (tree / "a").write_text("same") + os.link(tree / "a", tree / "b") + (tree / "c").write_text("same") + out = root / "image.cpio" + cpio.pack_tree(tree, out, 0) + objects = {obj.name: obj for obj in cpio.read(out.read_bytes())} + self.assertEqual(objects["a"].ino, objects["b"].ino) + self.assertEqual(objects["a"].nlink, 2) + self.assertNotEqual(objects["a"].ino, objects["c"].ino) + cpio.pack_paths(tree, out, 0, ["a"]) + (obj,) = cpio.read(out.read_bytes()) + self.assertEqual(obj.nlink, 1) + self.assertEqual(obj.size, 4) + + def test_timestamp_truncation_does_not_round_up(self) -> None: + # A nanosecond below a second boundary must truncate to the preceding second, even when + # the floating-point stat timestamp rounds up to the boundary. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + (root / "file").write_text("contents") + cutoff = 1000000000 + os.utime(root / "file", ns=(0, cutoff * 10**9 - 1)) + out = root / "image.cpio" + cpio.pack_paths(root, out, cutoff, ["file"]) + header = out.read_bytes()[: cpio._HEADER] + self.assertEqual(int(header[6 + 5 * 8 : 6 + 6 * 8], 16), cutoff - 1) + + if __name__ == "__main__": unittest.main() diff --git a/archive/tar.py b/archive/tar.py index 025d4a4f..c8d0e486 100644 --- a/archive/tar.py +++ b/archive/tar.py @@ -9,9 +9,11 @@ import os import tarfile -from collections.abc import Callable +from collections.abc import Callable, Sequence from pathlib import Path +import uapi16 + def _xattrs(path: Path) -> dict[str, str]: """Encode Linux xattrs using the convention understood by GNU tar and star.""" @@ -52,3 +54,51 @@ def pack_tree(tree: Path, out: Path, epoch: int) -> int: ) count += 1 return count + + +def pack_manifest( + objects: Sequence[uapi16.Object], + content: Callable[[uapi16.Object], Path], + out: Path, +) -> None: + kinds = { + "reg": tarfile.REGTYPE, + "dir": tarfile.DIRTYPE, + "lnk": tarfile.SYMTYPE, + "chr": tarfile.CHRTYPE, + "blk": tarfile.BLKTYPE, + "fifo": tarfile.FIFOTYPE, + } + links: dict[int, str] = {} + with tarfile.open(out, "w", format=tarfile.PAX_FORMAT) as archive: + for obj in objects: + kind = obj["type"] + if kind not in kinds: + raise ValueError(f"tar cannot represent inode type {kind!r} at {obj.get('name', '.')!r}") + info = tarfile.TarInfo("./" + obj["name"] if "name" in obj else ".") + info.type = kinds[kind] + info.mode = obj.get("mode", 0o777 if kind == "lnk" else 0o755) + headers: dict[str, str] = {} + ns = obj["mTimeNSec"] + info.mtime = ns // 10**9 + if ns % 10**9: + headers["mtime"] = f"{ns // 10**9}.{ns % 10**9:09}" + info.devmajor, info.devminor = obj.get("major", 0), obj.get("minor", 0) + for name, value in obj.get("extendedAttributes", {}).get("user", {}).items(): + data = uapi16.xattr_value(value) + headers["SCHILY.xattr.user." + name] = data.decode("utf-8", "surrogateescape") + info.pax_headers = headers + token = obj.get("inodeToken") + if token and token in links: + info.type, info.linkname = tarfile.LNKTYPE, links[token] + elif kind == "lnk": + info.linkname = uapi16.symlink_target(obj) + elif kind == "reg": + info.size = obj["size"] + if token: + links.setdefault(token, info.name) + if info.isreg(): + with content(obj).open("rb") as stream: + archive.addfile(info, stream) + else: + archive.addfile(info) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 9cc57b50..1b6cdaa1 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -1780,10 +1780,9 @@ belong to one package system are listed in its own section instead: - A root box verifies its packages with the `rpmkeys` of its own unverified stage1, so a tampered rpm package in the seed could defeat the check for that box; every other box and every image verifies with a predecessor. Arch packages are not verified at all. -- Archive ownership is intentionally normalized to uid/gid zero. Capabilities, xattrs, and SELinux labels do - not survive as Buck directory metadata; deferred tmpfiles can restore xattrs at terminal assembly, and tar - preserves them in PAX headers, but newc cpio cannot represent general xattrs. -- Directory image output cannot represent backslashes in names; archive outputs should be used instead. +- Archive ownership is intentionally normalized to uid/gid zero. Snapshots keep user xattrs but no + capabilities, other xattr namespaces or SELinux labels. Deferred tmpfiles can restore xattrs at terminal + assembly, tar preserves user xattrs in PAX headers, and newc cpio cannot represent xattrs. - The default `/usr`-only disk has a volatile root, so the `/etc` a build writes is not what such an image boots with; first-boot defaults come from credentials instead. Package and authored state outside `/usr` is not yet translated into factory defaults or another persistent partition. How much `/etc` such an diff --git a/docs/user/images.md b/docs/user/images.md index 0218e51a..f7063042 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -852,8 +852,26 @@ helper label. The same `image.InitrdInfo`, containing its logical `image.ImageIn `image.ImageArchiveInfo`, is the sole typed provider published by `[initrd]`. Merely carrying an artifact in a provider does not build it; optional actions run only when a consumer uses the artifact or a user selects its subtarget. -`[directory]` has the same representability limit as `image.directory`: it fails when the completed tree -contains a path, such as a systemd-escaped unit name, that Buck directory artifacts cannot store. +`[directory]` contains a UAPI.16 manifest and content blobs. Export it after Buck has materialized the +artifact, so the destination retains modes, times, user extended attributes, symlinks and hardlinks. + +Directory export requires read access to `/proc/sys/kernel/overflowuid`. Keep the stored manifest and blobs +immutable while exporting. The destination must not exist. Its parent must be owned by you or root. A parent +with group or other write permissions must have the sticky bit, as `/var/tmp` does. Use a parent owned by you +if Buck runs in a user namespace that does not map its owner's uid. The exporter rejects a parent owned by +the overflow uid, which is how a user namespace shows an owner that it does not map. The exporter creates a +private user namespace when needed; uid/gid zero then maps to your host uid/gid. Device nodes require +permission to create them on the destination filesystem. + +```sh +tine buck run 'tine//examples/image:boot-demo.fedora[directory]' -- /var/tmp/tine-image-export/root +``` + +The stored manifest remains authoritative if you copy the Buck artifact elsewhere; copying an exported +directory with a tool that drops metadata requires exporting it again from the stored snapshot. + +The `[manifest]` of an archive or disk describes its terminal contents, including authored tmpfiles and +package-database removal. `[pkgdb]` and `[sbom]` describe the installed package set. `[boot]` is what the disk boots, as files: the `bootable` rule run against the composition's own ESP, so a direct kernel boot or a publisher needs no second target pointing back at it. Its `[kernel]` and `[initrd]` diff --git a/examples/image/expected.json b/examples/image/expected.json index 1830c6e1..ca9aaa2d 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -39,25 +39,25 @@ }, "x86_64": { "boot-demo-preloaded.fedora": { - "sha256": "5639ec39438ad8de3facbc193b55163785b36956abec603061c95dce4229a8d2" + "sha256": "ab9d19efdbc3fe08b8a2271fa42a492a0cf1ba80a04ecb6dcd21c31f97e4ce9c" }, "boot-demo.fedora": { - "sha256": "810f4d8cd0ae75dd5d58c09784615409a4c0c5983f44b9a7e68b3e2b758aaab0" + "sha256": "0756e175d735d6b5f28d27b0f7422661c088fbfe7fad7def9b90b2c0750eaa2c" }, "boot-demo.fedora[boot][initrd]": { - "sha256": "92c4e74ec361a81e7dcb0484dfb97ba060f2c3199f790c0524c2811f21306254" + "sha256": "13d9081dec73347d54c27ba45af327616284cedb16781282a8075c8c24d86b87" }, "boot-demo.fedora[boot][kernel]": { "sha256": "d505498dab710cf2acb7c757d7d50592f147fd418d7464bca95415b190d43c87" }, "boot-demo.fedora[boot][uki]": { - "sha256": "1dbd49e1af726cb9023fd1da842d5441736563c5dea426901a98e2b5e22c14cb" + "sha256": "5f2da6847834d25631d1d06e85e2103b70c7a99631450673efd765c4fcde64e6" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "f2130ab8eb7af424c6be60b592f90015d757a24354af9fe103577b5a1d576619" + "sha256": "bcd43826e4c56c7888f01d2add6e47defaae0c233b36e7455d0fd8bc805b7820" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "16862b16926547262c44e1cc39de94e1136d1aa0b1ffa74fd4a1b25281db284b" + "sha256": "fa57c5b36cb2268d4ebffc2c4be284bc978f0f4cf680538ba75e34ab4e3491b2" }, "boot-demo.fedora[manifests][usr]": { "sha256": "456389b2e66d0a9dd1a60af74d1496cafffd89cd0141d0d166b1cd8a6135d949" @@ -72,7 +72,7 @@ "sha256": "f3dbb593040d1eed578b65d23532032df07162777707ffcf0a2a6f70d0e501f5" }, "demo-ext.fedora": { - "sha256": "fe8308bb979aae3004152ed7acc1190b343c46cc822a54c5c65e7ae2e89934ab" + "sha256": "70c293ffdf46ee6c97c4544cfaafbb96601cbcb83bf336e849f895451b7ef3d5" } } } diff --git a/examples/image/manifest_check.py b/examples/image/manifest_check.py index f96a097b..367a0dde 100644 --- a/examples/image/manifest_check.py +++ b/examples/image/manifest_check.py @@ -29,7 +29,7 @@ def decode(data: bytes) -> list[dict[str, Any]]: objects = decode(Path(sys.argv[1]).read_bytes()) assert objects[-1] == {"mediaType": "application/vnd.uapi.16.trailer"}, "missing trailer" root, files = objects[0], objects[1:-1] -assert root == {"mediaType": "application/vnd.uapi.16.manifest"}, f"bad root object: {root}" +assert root.get("mediaType") == "application/vnd.uapi.16.manifest" and "name" not in root, root assert files, "the manifest lists no files at all" names = [obj["name"] for obj in files] diff --git a/image/BUCK b/image/BUCK index 3c400cd8..9c1eb067 100644 --- a/image/BUCK +++ b/image/BUCK @@ -49,7 +49,9 @@ python.binary( "//:util", "//archive:cpio", "//archive:tar", + "//box:isolation", "//uapi16:storage", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) @@ -98,6 +100,7 @@ python.binary( "//:specs", "//:util", "//archive:cpio", + "//archive:tar", "//kmod:kmod", ], visibility = ["PUBLIC"], @@ -122,6 +125,8 @@ python.test( srcs = [ "ami_import.py", "ami_import_test.py", + "archive.py", + "archive_test.py", "ddi.py", "ddi_test.py", "finalize_test.py", @@ -142,6 +147,7 @@ python.test( "//:specs", "//:util", "//archive:cpio", + "//archive:tar", "//box:isolation", "//kmod:kmod", "//package:installer", @@ -178,7 +184,7 @@ python.binary( python.library( name = "finalize", srcs = ["finalize.py"], - deps = ["//:util", "//rootfs:rootfs", "//uapi16:storage"], + deps = ["//:util", "//rootfs:rootfs", "//uapi16:storage", "//uapi16:uapi16"], visibility = ["PUBLIC"], ) diff --git a/image/archive.bzl b/image/archive.bzl index 2433d063..6523fe87 100644 --- a/image/archive.bzl +++ b/image/archive.bzl @@ -22,13 +22,16 @@ ImageArchiveInfo = provider( fields = { "archive": provider_field(Artifact), "format": provider_field(str), + "manifest": provider_field(Artifact), }, ) ImageDirectoryInfo = provider( - doc = "A materialized logical image directory.", + doc = "A stored snapshot and its command for faithful directory export.", fields = { "directory": provider_field(Artifact), + "export": provider_field(RunInfo), + "manifest": provider_field(Artifact), }, ) @@ -49,6 +52,7 @@ def declare_image_archive( captures the database from the tree itself. """ out = declare_out(ctx, identifier, "image." + format + _COMPRESSION_EXT[compression]) + listing = declare_out(ctx, identifier, "image." + format + ".Uapi16Manifest") cmd = terminal_image_command( ctx, # One image can be archived in several formats, so the spec is named like the output. @@ -59,12 +63,13 @@ def declare_image_archive( spec = { "compression": compression, "format": format, + "manifest": listing.as_output(), "out": out.as_output(), "pkgdb_paths": pkgdb_paths(image) if strip_pkgdb else [], }, ) ctx.actions.run(cmd, category = "image_" + format, identifier = identifier or format) - return ImageArchiveInfo(archive = out, format = format) + return ImageArchiveInfo(archive = out, format = format, manifest = listing) ARCHIVE_ATTRS = { "compression": attrs.enum(COMPRESSIONS, default = "none"), @@ -89,16 +94,22 @@ def declare_image_directory( spec = { "compression": "none", "format": "directory", + "manifest": None, "out": out.as_output(), "pkgdb_paths": [], }, ) ctx.actions.run(cmd, category = "image_directory", identifier = identifier or "directory") - return ImageDirectoryInfo(directory = out) + export = RunInfo(args = cmd_args(tools.archive[RunInfo], "export", "--store", out)) + return ImageDirectoryInfo(directory = out, manifest = out.project("Uapi16Manifest"), export = export) + +def image_directory_providers(info: ImageDirectoryInfo) -> list[Provider]: + """Return the providers of a `[directory]` target or sub-target for `info`.""" + return [DefaultInfo(default_output = info.directory, sub_targets = {"manifest": [DefaultInfo(default_output = info.manifest)]}), info.export, info] def _image_directory_impl(ctx: AnalysisContext) -> list[Provider]: info = declare_image_directory(ctx, tools = ctx.attrs._tools[ImageToolsInfo], image = ctx.attrs.image[ImageInfo]) - return [DefaultInfo(default_output = info.directory), info] + return image_directory_providers(info) _image_directory = rule( impl = _image_directory_impl, diff --git a/image/archive.py b/image/archive.py index 925ed2ff..6b4573c0 100755 --- a/image/archive.py +++ b/image/archive.py @@ -3,16 +3,13 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Merge a logical image into a deterministic tar, cpio, or directory. - -Archive ownership is normalized to uid/gid 0. The newc cpio format has no general -extended-attribute representation. -""" +"""Export image snapshots as reproducible archives or directory stores.""" +import argparse import os -import subprocess import sys import tempfile +from contextlib import chdir from pathlib import Path import specs @@ -20,73 +17,98 @@ import cpio import finalize +import storage import tar +import uapi16 +from isolation import AT_FDCWD, Close, open_directory, unshare_sandbox class Spec(finalize.ImageSpec): out: str format: str compression: str - # Image paths holding the package database, stripped from the archive. pkgdb_paths: list[str] + manifest: str | None -def _clamp_mtimes(tree: Path, epoch: int) -> None: - """Clamp directory-format mtimes without an archive filter.""" - for path in [tree, *tree.rglob("*")]: - if path.lstat().st_mtime > epoch: - os.utime(path, (epoch, epoch), follow_symlinks=False) - - -def _pack(tree: Path, out: Path, fmt: str, epoch: int) -> None: +def _pack(snapshot: storage.Snapshot, out: Path, fmt: str, epoch: int) -> None: if fmt == "tar": - tar.pack_tree(tree, out, epoch) + tar.pack_manifest(snapshot.objects, snapshot.content, out) elif fmt == "cpio": - cpio.pack_tree(tree, out, epoch) + cpio.pack_manifest(snapshot.objects, snapshot.content, out, epoch) else: util.fail(f"unknown archive format {fmt!r}") -def _archive(tree: Path, out: Path, fmt: str, epoch: int, compression: str) -> None: +def _archive(snapshot: storage.Snapshot, out: Path, fmt: str, epoch: int, compression: str) -> None: if fmt == "directory": if compression != "none": util.fail("archive: the directory format cannot be compressed") - - # Reject names that would wedge Buck while storing the thawed tree. - for path in tree.rglob("*"): - if "\\" in path.name: - util.fail( - f"archive: {path.relative_to(tree)} contains a backslash, which buck cannot " - "store; the directory format cannot represent this image — use tar" - ) - out.mkdir(parents=True, exist_ok=True) - subprocess.run(["cp", "-a", "--reflink=auto", f"{tree}/.", str(out)], check=True) - _clamp_mtimes(out, epoch) + storage.save(snapshot, out) elif compression == "none": - _pack(tree, out, fmt, epoch) + _pack(snapshot, out, fmt, epoch) elif compression != "zstd": util.fail(f"archive: unknown compression {compression!r}") else: - # zstd needs the finished archive, so pack it beside the output rather than in TMPDIR: the - # shared filesystem keeps the packer's reflink cloning working, and Buck only ever sees the - # compressed result. - with tempfile.TemporaryDirectory(dir=out.parent) as scratch: + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: raw = Path(scratch) / out.name - _pack(tree, raw, fmt, epoch) + _pack(snapshot, raw, fmt, epoch) util.compress_zstd(raw, out) -def main(argv: list[str] | None = None) -> None: - spec = specs.parse(Spec, "archive", argv) +def export(argv: list[str]) -> None: + parser = argparse.ArgumentParser(prog="image-directory") + parser.add_argument("--store", required=True, type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args(argv) + # Restoration changes the working directory, so blob references need an absolute store. + snapshot = storage.Snapshot([args.store.absolute()]) + with Close(open_directory(AT_FDCWD, str(args.destination.parent))) as parent: + parent_attributes = os.fstat(parent) + # A user namespace shows an owner that it doesn't map as the overflow uid. The caller's own uid in + # the namespace can be the overflow uid as well. A parent owned by the overflow uid is never trusted. + overflow_uid = int(Path("/proc/sys/kernel/overflowuid").read_text()) + if ( + parent_attributes.st_uid not in (0, os.geteuid()) + or parent_attributes.st_uid == overflow_uid + or (parent_attributes.st_mode & 0o022 and not parent_attributes.st_mode & 0o1000) + ): + util.fail(f"image-directory: export needs a protected parent: {args.destination.parent}") + unshare_sandbox(become_root=True) + name = args.destination.name or "." + try: + os.mkdir(name, mode=0o700, dir_fd=parent) + except FileExistsError: + util.fail(f"image-directory: destination already exists: {args.destination}") + with ( + Close(os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)) as directory, + chdir("."), + ): + attributes = os.fstat(directory) + if attributes.st_uid != os.geteuid() or attributes.st_mode & 0o077 or os.listdir(directory): + util.fail(f"image-directory: destination changed before export: {args.destination}") + os.fchdir(directory) + storage.restore(snapshot, Path(".")) + +def main(argv: list[str] | None = None) -> None: + arguments = sys.argv[1:] if argv is None else argv + if arguments[:1] == ["export"]: + export(arguments[1:]) + return + spec = specs.parse(Spec, "archive", arguments) epoch = int(os.environ["SOURCE_DATE_EPOCH"]) out = Path(spec["out"]) - with finalize.image(spec, program="archive") as tree: - # The package database is a supply-chain artifact that the image's `[pkgdb]` subtarget - # captures separately, so an archive nothing resolves packages in can drop it. - for relative in spec["pkgdb_paths"]: - util.remove_path(tree / relative, with_parents=True) - _archive(tree, out, spec["format"], epoch, spec["compression"]) + with finalize.snapshot( + spec, program="archive", epoch=epoch, pkgdb_paths=spec["pkgdb_paths"] + ) as snapshot: + if spec["format"] == "cpio": + for obj in snapshot.objects: + obj["mTimeNSec"] = obj["mTimeNSec"] // 10**9 * 10**9 + obj.pop("extendedAttributes", None) + _archive(snapshot, out, spec["format"], epoch, spec["compression"]) + if spec["manifest"]: + uapi16.write_listing(snapshot.objects, Path(spec["manifest"])) print(f"archive: wrote {spec['format']} (epoch={epoch}) -> {out}", file=sys.stderr) diff --git a/image/archive_test.py b/image/archive_test.py new file mode 100644 index 00000000..4e595db2 --- /dev/null +++ b/image/archive_test.py @@ -0,0 +1,378 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Terminal metadata and archive identity regressions.""" + +import os +import shutil +import stat +import tarfile +import tempfile +import unittest +from contextlib import chdir +from decimal import Decimal +from pathlib import Path +from typing import override +from unittest import mock + +import specs + +import archive +import cpio +import finalize +import storage +import uapi16 + +EPOCH = 1000000000 + + +# Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. +CAPTURE_EPOCH = 2**31 + + +class TestArchive(unittest.TestCase): + @override + def setUp(self) -> None: + self.scratch = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp"))) + source = self.scratch / "source" + (source / "empty-dir").mkdir(parents=True) + for name, mode, ns in (("a", 0o640, 123), ("b", 0o755, 456), ("empty", 0o600, 789)): + path = source / name + path.write_text("contents" if name != "empty" else "") + path.chmod(mode) + os.utime(path, ns=(ns, ns)) + os.link(source / "a", source / "linked") + os.link(source / "empty", source / "empty-linked") + (source / "link").symlink_to("/a") + for name in ("back\\slash", ".wh.literal", ".esc.literal", "TRAILER!!!"): + (source / name).write_text(name) + os.mkfifo(source / "fifo", 0o600) + os.setxattr(source / "a", "user.example", b"value") + (source / "var/lib/database").mkdir(parents=True) + (source / "var/lib/database/contents").write_text("database") + self.store = self.scratch / "store" + storage.capture(source, self.store, epoch=CAPTURE_EPOCH) + self.spec = finalize.ImageSpec(lower=[str(self.store)], tmpfiles=[]) + + def test_archive_writers_read_blobs_without_reconstruction(self) -> None: + # A terminal without tmpfiles can package its snapshot directly. Tar must preserve + # declared links, independent equal files, original nanoseconds and user attributes. + out = self.scratch / "image.tar" + with mock.patch.object(storage, "root", side_effect=AssertionError("unexpected reconstruction")): + with finalize.snapshot(self.spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, out, "tar", EPOCH, "none") + with tarfile.open(out) as packed: + objects = {entry.name: entry for entry in packed.getmembers()} + self.assertTrue(objects["./linked"].islnk()) + self.assertEqual(objects["./linked"].linkname, "./a") + self.assertTrue(objects["./b"].isreg()) + self.assertEqual(objects["./a"].mode, 0o644) + self.assertEqual(objects["./a"].pax_headers["mtime"], "0.000000123") + self.assertEqual(objects["./a"].pax_headers["SCHILY.xattr.user.example"], "value") + self.assertEqual(objects["./link"].linkname, "/a") + self.assertTrue(objects["./fifo"].isfifo()) + self.assertIn("./back\\slash", objects) + + def test_cpio_preserves_nonempty_and_empty_hardlinks(self) -> None: + # Newc needs one payload and a shared inode per declared group, including empty files. + # Equal contents alone must not give unrelated regular files the same inode. + out = self.scratch / "image.cpio" + with finalize.snapshot(self.spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, out, "cpio", EPOCH, "none") + objects = {obj.name: obj for obj in cpio.read(out.read_bytes())} + self.assertGreater(objects["./TRAILER!!!"].size, 0) + self.assertEqual(objects["a"].ino, objects["linked"].ino) + self.assertEqual(objects["a"].nlink, 2) + self.assertEqual(objects["empty"].ino, objects["empty-linked"].ino) + self.assertNotEqual(objects["a"].ino, objects["b"].ino) + dest = self.scratch / "unpacked" + with out.open("rb") as stream: + cpio.unpack(stream.fileno(), dest) + self.assertTrue((dest / "a").samefile(dest / "linked")) + self.assertTrue((dest / "empty").samefile(dest / "empty-linked")) + self.assertFalse((dest / "a").samefile(dest / "b")) + + def test_tmpfiles_and_database_removal_match_exported_manifest(self) -> None: + # Apply terminal tmpfiles and database removal before writing both the archive and its + # listing. The manifest must describe the modes and timestamps that tar actually encodes. + out, listing = self.scratch / "image.tar", self.scratch / "listing" + spec = specs.write( + self.scratch / "spec.json", + dict(self.spec) + | { + "out": str(out), + "format": "tar", + "compression": "none", + "manifest": str(listing), + "pkgdb_paths": ["var/lib/database"], + "tmpfiles": ["z /a 0600 - - -", "f /created 0640 - - - authored"], + }, + ) + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": str(EPOCH)}): + archive.main(["--spec", str(spec)]) + objects = {obj.get("name", "."): obj for obj in uapi16.read(listing)} + with tarfile.open(out) as packed: + for entry in packed.getmembers(): + name = entry.name.removeprefix("./") + obj = objects[name] + if not entry.issym(): + self.assertEqual(entry.mode, obj["mode"]) + ns = int(Decimal(entry.pax_headers.get("mtime", str(entry.mtime))) * 10**9) + self.assertEqual(ns, obj["mTimeNSec"]) + stream = packed.extractfile("./created") + assert stream is not None + with stream: + self.assertEqual(stream.read(), b"authored") + self.assertEqual(objects["a"]["mode"], 0o600) + self.assertEqual(objects["linked"]["mode"], 0o600) + self.assertNotIn("var/lib/database", objects) + self.assertNotIn("var", objects) + + def test_directory_export_restores_after_materialization(self) -> None: + # The directory artifact contains only safe blob names and a uapi16. Copying that + # artifact before export must not lose modes, times, hardlinks or logical filenames. + out, copied = self.scratch / "directory", self.scratch / "copied" + with finalize.snapshot(self.spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, out, "directory", EPOCH, "none") + shutil.copytree(out, copied, copy_function=shutil.copyfile) + with chdir(self.scratch): + archive.export(["--store", "copied", "exported"]) + exported = self.scratch / "exported" + self.assertEqual((exported / "a").stat().st_mtime_ns, 123) + self.assertEqual(stat.S_IMODE((exported / "a").stat().st_mode), 0o644) + self.assertTrue((exported / "a").samefile(exported / "linked")) + self.assertFalse((exported / "a").samefile(exported / "b")) + self.assertTrue((exported / "back\\slash").exists()) + self.assertEqual({p.name for p in out.iterdir()}, {"blobs", uapi16.MANIFEST}) + + def test_export_refuses_a_destination_created_during_validation(self) -> None: + # Another process inserts a destination and a symlink while the store is validated. + # Export must fail without following that symlink or changing the unrelated file. + destination, victim = self.scratch / "exported", self.scratch / "victim" + victim.write_text("unrelated") + read = storage.Snapshot + + def raced(stores: list[Path]) -> storage.Snapshot: + snapshot = read(stores) + destination.mkdir() + (destination / "a").symlink_to(victim) + return snapshot + + with mock.patch.object(storage, "Snapshot", side_effect=raced), self.assertRaises(SystemExit): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual(victim.read_text(), "unrelated") + + def test_export_keeps_the_open_directory_after_rename(self) -> None: + # Another process renames the private destination after it is opened and inserts a symlink. + # Restoration must stay in the opened directory and leave the symlink's target unchanged. + destination, saved = self.scratch / "exported", self.scratch / "renamed" + victim = self.scratch / "victim" + victim.mkdir() + (victim / "a").write_text("unrelated") + restore = storage.restore + + def raced(snapshot: storage.Snapshot, tree: Path) -> None: + destination.rename(saved) + destination.symlink_to(victim, target_is_directory=True) + restore(snapshot, tree) + + with mock.patch.object(storage, "restore", side_effect=raced): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual((victim / "a").read_text(), "unrelated") + self.assertEqual((saved / "a").read_text(), "contents") + + def test_export_rejects_a_replacement_directory(self) -> None: + # Another process replaces the destination before open with a private directory containing + # a symlink. Export must reject the nonempty replacement without following the symlink. + destination, saved = self.scratch / "exported", self.scratch / "renamed" + replacement, victim = self.scratch / "replacement", self.scratch / "victim" + replacement.mkdir(mode=0o700) + victim.write_text("unrelated") + (replacement / "a").symlink_to(victim) + open_directory = os.open + + def raced(path: str | Path, flags: int, *, dir_fd: int | None = None) -> int: + if path == destination.name: + destination.rename(saved) + replacement.rename(destination) + return open_directory(path, flags, dir_fd=dir_fd) + + with mock.patch.object(archive.os, "open", side_effect=raced), self.assertRaises(SystemExit): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual(victim.read_text(), "unrelated") + + def test_export_keeps_the_parent_directory_after_rename(self) -> None: + # Another user replaces an ancestor after mkdir. The destination then resolves to an existing + # private empty directory. Opening through the original parent must retain the created root. + parent, saved = self.scratch / "parent", self.scratch / "renamed" + parent.mkdir() + destination = parent / "out" + victim = self.scratch / "victim" + (victim / "out").mkdir(parents=True, mode=0o700) + os.utime(victim / "out", ns=(111, 111)) + open_directory = os.open + + def raced(path: str | Path, flags: int, *, dir_fd: int | None = None) -> int: + if path == destination.name: + parent.rename(saved) + parent.symlink_to(victim, target_is_directory=True) + return open_directory(path, flags, dir_fd=dir_fd) + + with mock.patch.object(archive.os, "open", side_effect=raced): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual(list((victim / "out").iterdir()), []) + self.assertEqual((victim / "out").stat().st_mtime_ns, 111) + self.assertEqual((saved / "out/a").read_text(), "contents") + + def test_export_requires_a_protected_parent(self) -> None: + # A writable nonsticky parent permits another user to replace the new root with an existing + # private empty directory. Reject that parent before creating or changing either directory. + parent = self.scratch / "shared" + parent.mkdir() + parent.chmod(0o777) + private = parent / "private" + private.mkdir(mode=0o700) + destination = parent / "out" + with self.assertRaises(SystemExit): + archive.export(["--store", str(self.store), str(destination)]) + self.assertFalse(destination.exists()) + self.assertEqual(stat.S_IMODE(private.stat().st_mode), 0o700) + + # A trusted sticky parent prevents other users from renaming the exporter's private root. + parent.chmod(0o1777) + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual((destination / "a").read_text(), "contents") + + def test_export_creates_a_protected_parent_with_a_shared_umask(self) -> None: + # Direct execution can inherit a shared umask. A missing parent must be created with + # protected permissions so the exporter can restore the image beneath it. + destination = self.scratch / "missing" / "out" + previous = os.umask(0o002) + try: + archive.export(["--store", str(self.store), str(destination)]) + finally: + os.umask(previous) + self.assertEqual((destination / "a").read_text(), "contents") + + def test_export_rejects_an_unmapped_owner_matching_the_caller(self) -> None: + # A user namespace shows an unmapped parent owner as the overflow uid, which can equal the + # caller's uid. Report the overflow uid as both the caller's uid and the owner of the parent, and + # check that the export fails without creating the destination. + destination = self.scratch / "out" + parent = self.scratch.stat() + overflow = int(Path("/proc/sys/kernel/overflowuid").read_text()) + fstat = os.fstat + + def attributes(fd: int) -> os.stat_result: + result = fstat(fd) + if result.st_ino == parent.st_ino and result.st_dev == parent.st_dev: + values = list(result) + values[4] = overflow + return os.stat_result(values) + return result + + with ( + mock.patch.object(archive.os, "geteuid", return_value=overflow), + mock.patch.object(archive.os, "fstat", side_effect=attributes), + self.assertRaises(SystemExit), + ): + archive.export(["--store", str(self.store), str(destination)]) + self.assertFalse(destination.exists()) + + def test_database_removal_resolves_intermediate_symlinks(self) -> None: + # A database can be reached through a relative or absolute directory symlink. Direct + # terminal export and tmpfiles finalization must remove it inside the image in either case. + source = self.scratch / "source" + (source / "var/lib").rename(source / "relocated") + for index, target in enumerate(("../relocated", "/relocated")): + (source / "var/lib").unlink(missing_ok=True) + (source / "var/lib").symlink_to(target) + store = self.scratch / f"store-{index}" + storage.capture(source, store, epoch=CAPTURE_EPOCH) + for rules in ([], ["d /empty-dir - - -"]): + with self.subTest(target=target, rules=rules): + spec = finalize.ImageSpec(lower=[str(store)], tmpfiles=rules) + with finalize.snapshot( + spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] + ) as snapshot: + names = {obj["name"] for obj in snapshot.objects[1:]} + self.assertNotIn("relocated/database", names) + self.assertNotIn("relocated/database/contents", names) + self.assertIn("var/lib", names) + + def test_database_removal_updates_the_root_timestamp(self) -> None: + # Removing the last var subtree changes the root mtime. The direct snapshot path must + # match filesystem removal when an existing directory has a no-op tmpfiles rule. + source = self.scratch / "source" + for present in (True, False): + if not present: + shutil.rmtree(source / "var/lib/database") + os.utime(source, ns=(111, 111)) + storage.capture(source, self.store, epoch=CAPTURE_EPOCH) + for rules in ([], ["d /empty-dir - - -"]): + with self.subTest(present=present, rules=rules): + spec = finalize.ImageSpec(lower=self.spec["lower"], tmpfiles=rules) + with finalize.snapshot( + spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] + ) as snapshot: + self.assertEqual(snapshot.objects[0]["mTimeNSec"], EPOCH * 10**9) + self.assertNotIn("var", {obj["name"] for obj in snapshot.objects[1:]}) + + def test_missing_database_does_not_require_reconstruction(self) -> None: + # A cached snapshot can contain a device even when the current worker cannot create it. + # An absent database without symlink parents must still be archivable directly from blobs. + objects = uapi16.read(self.store / uapi16.MANIFEST) + objects.append( + {"name": "device", "type": "chr", "mode": 0o600, "mTimeNSec": 111, "major": 1, "minor": 3} + ) + uapi16.write_objects(objects, self.store / uapi16.MANIFEST) + out = self.scratch / "image.tar" + with mock.patch.object(storage, "root", side_effect=AssertionError("unexpected reconstruction")): + with finalize.snapshot( + self.spec, program="test", epoch=EPOCH, pkgdb_paths=["absent/database"] + ) as snapshot: + archive._archive(snapshot, out, "tar", EPOCH, "none") + with tarfile.open(out) as packed: + self.assertTrue(packed.getmember("./device").ischr()) + + def test_missing_database_leaves_nonempty_parent_times_unchanged(self) -> None: + # A missing database under a nonempty directory removes nothing. Its parent timestamps + # must remain unchanged rather than being set to the terminal epoch. + source = self.scratch / "source" + shutil.rmtree(source / "var/lib/database") + (source / "var/lib/keep").touch() + os.utime(source / "var/lib", ns=(333, 333)) + os.utime(source, ns=(111, 111)) + storage.capture(source, self.store, epoch=CAPTURE_EPOCH) + with finalize.snapshot( + self.spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] + ) as snapshot: + objects = {obj.get("name", "."): obj for obj in snapshot.objects} + self.assertEqual(objects["."]["mTimeNSec"], 111) + self.assertEqual(objects["var/lib"]["mTimeNSec"], 333) + + def test_cpio_listing_uses_terminal_timestamp_precision(self) -> None: + # Cpio finalization must round the listing to whole seconds and omit user attributes, + # while keeping the explicit hardlink group used by the archive. + out, listing = self.scratch / "image.cpio", self.scratch / "listing" + spec = specs.write( + self.scratch / "spec.json", + dict(self.spec) + | { + "out": str(out), + "manifest": str(listing), + "format": "cpio", + "compression": "none", + "pkgdb_paths": [], + }, + ) + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": str(EPOCH)}): + archive.main(["--spec", str(spec)]) + objects = {obj.get("name", "."): obj for obj in uapi16.read(listing)} + self.assertEqual(objects["a"]["mTimeNSec"], 0) + self.assertNotIn("extendedAttributes", objects["a"]) + self.assertEqual(objects["a"]["inodeToken"], objects["linked"]["inodeToken"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/image/compose.bzl b/image/compose.bzl index 4713ed92..64d7923c 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -15,6 +15,7 @@ load( "ImageArchiveInfo", "declare_image_archive", "declare_image_directory", + "image_directory_providers", ) load( ":boot.bzl", @@ -152,6 +153,7 @@ def _initrd_image_impl(ctx: AnalysisContext) -> list[Provider]: return image_providers( default_outputs = [cpio.archive], extra = [InitrdInfo(cpio = cpio, image = image), cpio], + sub_targets = {"manifest": [DefaultInfo(default_output = cpio.manifest)]}, image = image, ) @@ -236,6 +238,7 @@ def _archive_impl(ctx: AnalysisContext) -> list[Provider]: return image_providers( default_outputs = [archive.archive], extra = [archive], + sub_targets = {"manifest": [DefaultInfo(default_output = archive.manifest)]}, image = image, ) @@ -484,7 +487,8 @@ def _disk_impl(ctx: AnalysisContext) -> list[Provider]: "initrd": [ DefaultInfo( default_output = initrd_info.cpio.archive, - sub_targets = published_metadata_subtargets(initrd_info.image, "{}.initrd".format(basename), version), + sub_targets = published_metadata_subtargets(initrd_info.image, "{}.initrd".format(basename), version) + | {"manifest": [DefaultInfo(default_output = initrd_info.cpio.manifest)]}, ), initrd_info, ], @@ -539,7 +543,7 @@ def _disk_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets = dict(composed.sub_targets) sub_targets.update(manifest_sub_targets(manifests)) sub_targets.update(boot_sub_targets) - sub_targets["directory"] = [DefaultInfo(default_output = directory.directory), directory] + sub_targets["directory"] = image_directory_providers(directory) if bootable: sub_targets["ami"] = declare_ami( ctx.actions, @@ -594,7 +598,7 @@ def _disk_impl(ctx: AnalysisContext) -> list[Provider]: default_outputs = [raw], extra = [directory, composed.info, published] + boot_providers, image = top, - sub_targets = sub_targets | published_metadata_subtargets(top, basename, version), + sub_targets = published_metadata_subtargets(top, basename, version) | sub_targets, ) _DISK_ATTRS = ( diff --git a/image/ddi.bzl b/image/ddi.bzl index e56bbb49..17b4a5c0 100644 --- a/image/ddi.bzl +++ b/image/ddi.bzl @@ -439,10 +439,8 @@ def declare_ddi( ) sub_targets = manifest_sub_targets(manifests) if manifest != None: - # Qualified, unlike the provider field: a logical image publishes a `manifest` subtarget of - # its own (image.bzl:image_metadata_subtargets) listing its whole tree, and a composition - # merges both sets of subtargets into one namespace. sub_targets["disk-manifest"] = [DefaultInfo(default_output = manifest)] + sub_targets["manifest"] = [DefaultInfo(default_output = manifest)] if root_hash != None: sub_targets["roothash"] = [DefaultInfo(default_output = root_hash.hash), info] diff --git a/image/extension.bzl b/image/extension.bzl index 146995c5..db3415ab 100644 --- a/image/extension.bzl +++ b/image/extension.bzl @@ -165,7 +165,7 @@ def _declare_preset( default_outputs = [info.disk], extra = [info, published] + extra, image = image, - sub_targets = output.sub_targets | published_metadata_subtargets(image, basename, version, filename), + sub_targets = published_metadata_subtargets(image, basename, version, filename) | output.sub_targets, ) def declare_extension( diff --git a/image/finalize.py b/image/finalize.py index ff49fe24..5642ce75 100644 --- a/image/finalize.py +++ b/image/finalize.py @@ -27,6 +27,7 @@ import rootfs import storage +import uapi16 class ImageSpec(TypedDict): @@ -370,3 +371,61 @@ def locale_gen(tree: Path) -> None: # it wrote nothing. Only 0 means that the archive contains the locale. if rc != 0: fail(f"locale_gen: `{' '.join(cmd)}` failed (rc={rc})") + + +def _remove(objects: dict[str, uapi16.Object], relative: str, epoch: int) -> None: + path = Path(relative) + name = storage.resolve(objects, relative, follow=False) + removed = False + for child in list(objects): + if child == name or child.startswith(name + "/"): + del objects[child] + removed = True + while True: + path = path.parent + name = storage.resolve(objects, str(path)) + directory = objects.get(name) + if directory is None or directory["type"] != "dir": + return + if removed: + # Removing a file from a real tree sets the mtime of its directory to the current time. + # snapshot() clamps that time to the epoch. + directory["mTimeNSec"] = epoch * 10**9 + # remove_path() stops at a parent that is a symlink. + if name == "." or objects[storage.resolve(objects, str(path), follow=False)]["type"] == "lnk": + return + if any(child.startswith(name + "/") for child in objects): + return + del objects[name] + removed = True + + +@contextlib.contextmanager +def snapshot( + spec: ImageSpec, + *, + program: str, + epoch: int, + pkgdb_paths: Sequence[str] = (), +) -> Generator[storage.Snapshot]: + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + if spec["tmpfiles"]: + out = Path(scratch) / "snapshot" + with image(spec, program=program) as tree: + storage.capture(tree, out, spec["lower"], epoch=epoch) + result = storage.Snapshot([*spec["lower"], out]) + else: + result = storage.Snapshot(spec["lower"]) + for obj in result.objects: + if obj["type"] == "dir": + obj["mode"] = 0o1777 if obj.get("name") in _WORLD_WRITABLE else 0o755 + elif obj["type"] == "reg": + obj["mode"] = 0o755 if obj["mode"] & 0o111 else 0o644 + objects = storage.by_name(result.objects) + for relative in pkgdb_paths: + _remove(objects, relative, epoch) + result.objects = list(objects.values()) + uapi16.renumber(result.objects) + for obj in result.objects: + obj["mTimeNSec"] = min(obj.get("mTimeNSec", epoch * 10**9), epoch * 10**9) + yield result diff --git a/image/manifest_main.py b/image/manifest_main.py index d3b69f84..6057e7f3 100755 --- a/image/manifest_main.py +++ b/image/manifest_main.py @@ -24,8 +24,8 @@ def main(argv: list[str] | None = None) -> None: out = Path(spec["out"]) epoch = int(os.environ["SOURCE_DATE_EPOCH"]) - with finalize.image(spec, program="manifest") as tree: - written = uapi16.write(tree, out, epoch) + with finalize.snapshot(spec, program="manifest", epoch=epoch) as snapshot: + written = uapi16.write_listing(snapshot.objects, out) print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) diff --git a/image/uki.py b/image/uki.py index 4bbb1550..0eceecb6 100755 --- a/image/uki.py +++ b/image/uki.py @@ -199,7 +199,7 @@ def _microcode(tree: Path, scratch: Path, epoch: int, efi_arch: str) -> Path | N for path in root.rglob("*"): path.chmod(0o755 if path.is_dir() else 0o644) archive = scratch / "microcode.cpio" - cpio.pack_tree(root, archive, epoch, block_align=False) + cpio.pack_tree(root, archive, epoch) print(f"uki: microcode for {', '.join(packed)} in {archive.stat().st_size // 1024} KiB", file=sys.stderr) return archive diff --git a/uapi16/storage.py b/uapi16/storage.py index 7433eb8f..225df713 100644 --- a/uapi16/storage.py +++ b/uapi16/storage.py @@ -3,13 +3,12 @@ """Store complete filesystem snapshots independently of artifact filesystem metadata.""" -import base64 import os import stat import tempfile from collections.abc import Generator, Sequence from contextlib import ExitStack, contextmanager -from pathlib import Path +from pathlib import Path, PurePosixPath from util import clone_file, fail @@ -69,6 +68,12 @@ def content(self, obj: uapi16.Object) -> Path: return path +def add_blob(source: Path, destination: Path) -> None: + if not destination.exists(): + clone_file(source, destination) + destination.chmod(0o444) + + def capture(tree: Path, out: Path, parents: Sequence[str | Path] = (), *, epoch: int) -> None: """Write the snapshot of `tree` to `out`, with modification times clamped to `epoch`.""" parent = Snapshot(parents) @@ -90,10 +95,8 @@ def capture(tree: Path, out: Path, parents: Sequence[str | Path] = (), *, epoch: if obj["type"] == "reg": digest = obj["sha256"] obj["contents"] = [{"file": f"blobs/{digest}"}] - destination = blobs / digest - if digest not in parent.blobs and not destination.exists(): - clone_file(entry.path, destination) - destination.chmod(0o444) + if digest not in parent.blobs: + add_blob(entry.path, blobs / digest) inodes[entry.inode] = obj objects.append(obj) for obj in objects: @@ -102,14 +105,52 @@ def capture(tree: Path, out: Path, parents: Sequence[str | Path] = (), *, epoch: uapi16.write_objects(objects, out / uapi16.MANIFEST) +def save(snapshot: Snapshot, out: Path) -> None: + blobs = out / "blobs" + blobs.mkdir(parents=True, exist_ok=True) + for obj in snapshot.objects[1:]: + if obj["type"] == "reg": + add_blob(snapshot.content(obj), blobs / obj["sha256"]) + uapi16.write_objects(snapshot.objects, out / uapi16.MANIFEST) + + +def by_name(objects: Sequence[uapi16.Object]) -> dict[str, uapi16.Object]: + # The root object has no name. resolve() returns "." for it. + return {obj.get("name", "."): obj for obj in objects} + + +def resolve(objects: dict[str, uapi16.Object], name: str, *, follow: bool = True) -> str: + resolved = PurePosixPath() + parts = list(PurePosixPath(name).parts) + hops = 0 + while parts: + part = parts.pop(0) + if part == "..": + resolved = resolved.parent + continue + candidate = resolved / part + obj = objects.get(str(candidate)) + if obj is None or obj["type"] != "lnk" or (not parts and not follow): + resolved = candidate + continue + hops += 1 + if hops > 40: + fail(f"storage: too many levels of symbolic links in {name!r}") + target = PurePosixPath(uapi16.symlink_target(obj)) + if target.is_absolute(): + resolved = PurePosixPath() + target = target.relative_to("/") + parts = [*target.parts, *parts] + return str(resolved) + + def metadata(path: Path, obj: uapi16.Object) -> None: if "uid" in obj or "gid" in obj: os.chown(path, obj.get("uid", 0), obj.get("gid", 0), follow_symlinks=False) if obj["type"] != "lnk": path.chmod(obj["mode"]) for name, value in obj.get("extendedAttributes", {}).get("user", {}).items(): - data = value["text"].encode() if "text" in value else base64.b64decode(value["base64"]) - os.setxattr(path, "user." + name, data, follow_symlinks=False) + os.setxattr(path, "user." + name, uapi16.xattr_value(value), follow_symlinks=False) if "mTimeNSec" in obj: os.utime(path, ns=(obj["mTimeNSec"], obj["mTimeNSec"]), follow_symlinks=False) @@ -136,7 +177,7 @@ def restore(snapshot: Snapshot, tree: Path, *, hardlinks: bool | None = None) -> # Metadata restoration and writes must never change a cached blob's inode. clone_file(snapshot.content(obj), path) elif kind == "lnk": - path.symlink_to(os.fsdecode(base64.b64decode(obj["contents"][0]["literal"]))) + path.symlink_to(uapi16.symlink_target(obj)) elif kind == "fifo": os.mkfifo(path) elif kind in ("chr", "blk"): diff --git a/uapi16/uapi16.py b/uapi16/uapi16.py index 16fc9fbd..9db6c073 100644 --- a/uapi16/uapi16.py +++ b/uapi16/uapi16.py @@ -232,6 +232,25 @@ def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object return obj +def symlink_target(obj: Object) -> str: + return os.fsdecode(base64.b64decode(obj["contents"][0]["literal"])) + + +def xattr_value(value: Xattr) -> bytes: + return value["text"].encode() if "text" in value else base64.b64decode(value["base64"]) + + +def write_listing(objects: Sequence[Object], out: Path) -> int: + # The `contents` of a regular file names a blob, and the blob only exists inside a store. + listed: list[Object] = [] + for obj in objects: + copy = obj.copy() + if copy.get("type") != "lnk": + copy.pop("contents", None) + listed.append(copy) + return write_objects(listed, out) + + def _record(obj: Object) -> bytes: return ( _RS @@ -344,11 +363,15 @@ def merge(sources: Sequence[Path], out: Path) -> int: objects[str(obj["name"])] = obj offset += highest - tokens: dict[int, int] = {} - counts = Counter(obj.get("inodeToken") for obj in objects.values() if obj.get("inodeToken")) ordered = [objects[name] for name in sorted(objects, key=order)] - for obj in ordered: + renumber(ordered) + return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out) + + +def renumber(objects: Sequence[Object]) -> None: + tokens: dict[int, int] = {} + counts = Counter(obj.get("inodeToken") for obj in objects if obj.get("inodeToken")) + for obj in objects: token = obj.pop("inodeToken", None) if isinstance(token, int) and counts[token] > 1: obj["inodeToken"] = tokens.setdefault(token, len(tokens) + 1) - return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out) diff --git a/util.py b/util.py index 835be99d..0552da3a 100644 --- a/util.py +++ b/util.py @@ -241,7 +241,7 @@ def remove_path(path: Path, with_parents: bool = False) -> None: return for parent in path.parents: # `parent.parent` stops the walk at the filesystem root. - if parent == parent.parent or not parent.is_dir() or any(parent.iterdir()): + if parent == parent.parent or parent.is_symlink() or not parent.is_dir() or any(parent.iterdir()): break parent.rmdir() From cafc4dd3cf2b2b1da324a46eacb7efafe917b28c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 7 Nov 2026 17:27:46 +0100 Subject: [PATCH 5/7] image: Reconstruct roots with reflinks or data layers Image reconstruction copies every regular file before commands run. Let's use private reflinks when available and metadata over shared blobs otherwise. Declared hardlink groups share private inodes. Capture uses upper inode identity when OverlayFS reports different inode numbers for paths linked by commands. Signed-off-by: Daan De Meyer --- box/isolation.py | 16 +- box/isolation_test.py | 49 +++++ docs/design/architecture.md | 176 +++++++++++------- docs/user/images.md | 15 +- image/archive_test.py | 60 +++++- image/ddi.py | 32 ++-- image/finalize.py | 79 ++++---- image/finalize_test.py | 24 --- image/layer.py | 5 +- image/layer_test.py | 44 ++++- package/transaction.py | 4 +- uapi16/storage.py | 146 +++++++++++---- uapi16/storage_test.py | 358 +++++++++++++++++++++++++++++++----- util.py | 17 ++ util_test.py | 37 ++++ 15 files changed, 818 insertions(+), 244 deletions(-) diff --git a/box/isolation.py b/box/isolation.py index 7dc83f80..ebeffc12 100644 --- a/box/isolation.py +++ b/box/isolation.py @@ -562,9 +562,17 @@ def _tmpfs(mode: str = _TMPFS_DEFAULT_MODE) -> int: return _fsmount(fs) -def _overlay(lowers: tuple[str, ...], upper: str | None, work: str | None, attributes: int = 0) -> int: +def _overlay( + lowers: tuple[str, ...], + upper: str | None, + work: str | None, + attributes: int = 0, + *, + datadirs: tuple[str, ...] = (), +) -> int: """Return a detached overlay. Without `upper` and `work`, the kernel makes it read-only.""" layers = [("lowerdir+", lower) for lower in lowers] + layers += [("datadir+", data) for data in datadirs] layers += [(key, path) for key, path in (("upperdir", upper), ("workdir", work)) if path is not None] with Close(_fsopen("overlay")) as fs: _fsconfig(fs, FSCONFIG_SET_STRING, "source", "overlayfs") @@ -997,20 +1005,22 @@ def __init__( workdir: StrPath | None, target: StrPath, *, + datadirs: tuple[StrPath, ...] = (), strict_unmount: bool = False, ) -> None: super().__init__(target, strict_unmount=strict_unmount) if (upperdir is None) != (workdir is None): raise ValueError("Overlay takes an upperdir and a workdir, or neither") - if upperdir is None and len(lowerdirs) < 2: + if upperdir is None and len(lowerdirs) + len(datadirs) < 2: raise ValueError("a read-only Overlay needs at least two lowerdirs") self.lowerdirs = tuple(os.fspath(path) for path in lowerdirs) + self.datadirs = tuple(os.fspath(path) for path in datadirs) self.upperdir = None if upperdir is None else os.fspath(upperdir) self.workdir = None if workdir is None else os.fspath(workdir) @override def _open(self) -> int: - return _overlay(self.lowerdirs, self.upperdir, self.workdir) + return _overlay(self.lowerdirs, self.upperdir, self.workdir, datadirs=self.datadirs) Filesystem = Bind | File | Devices | Tmpfs | Symlink | Overlay diff --git a/box/isolation_test.py b/box/isolation_test.py index 833b004f..e9a7b0df 100644 --- a/box/isolation_test.py +++ b/box/isolation_test.py @@ -353,6 +353,55 @@ def test_overlay_paths_may_contain_option_separators(self) -> None: (target / "generated").write_text("output") self.assertEqual((upper / "generated").read_text(), "output") + def test_overlay_reads_data_layers_without_changing_blobs(self) -> None: + # Sparse lower files provide distinct metadata for shared data in two hidden data layers. + # Reading must not copy data up; chmod and writes must leave the data layers unchanged. + with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: + root = Path(directory) + lower, first, second, upper, work = ( + root / name for name in ("lower", "data:one", "data,two\\", "upper", "work") + ) + for path in (lower, first, second, upper, work): + path.mkdir() + (first / "shared").write_bytes(b"first") + (second / "shared").write_bytes(b"other") + (second / "inherited").write_bytes(b"parent") + for name, redirect, size, mode, timestamp in ( + ("a", "shared", 5, 0o640, 111), + ("b", "shared", 5, 0o755, 222), + ("c", "inherited", 6, 0o644, 333), + ): + path = lower / name + with path.open("wb") as stream: + stream.truncate(size) + os.setxattr(path, "user.overlay.metacopy", b"") + os.setxattr(path, "user.overlay.redirect", ("/" + redirect).encode()) + path.chmod(mode) + os.utime(path, ns=(timestamp, timestamp)) + before = (first / "shared").stat() + target = root / "target" + with isolation.Overlay( + (lower,), upper, work, target, datadirs=(first, second), strict_unmount=True + ) as mounted: + mounted.mount() + self.assertEqual({path.name for path in target.iterdir()}, {"a", "b", "c"}) + self.assertEqual((target / "a").read_bytes(), b"first") + self.assertEqual((target / "b").read_bytes(), b"first") + self.assertEqual((target / "c").read_bytes(), b"parent") + self.assertFalse((target / "a").samefile(target / "b")) + self.assertEqual((target / "a").stat().st_mode & 0o777, 0o640) + self.assertEqual((target / "b").stat().st_mtime_ns, 222) + self.assertFalse((upper / "a").exists()) + (target / "a").chmod(0o600) + self.assertTrue((upper / "a").exists()) + self.assertEqual((upper / "a").read_bytes(), b"first") + (target / "a").write_bytes(b"changed") + self.assertEqual((target / "b").read_bytes(), b"first") + self.assertEqual((first / "shared").read_bytes(), b"first") + self.assertEqual((first / "shared").stat().st_mode, before.st_mode) + self.assertEqual((first / "shared").stat().st_mtime_ns, before.st_mtime_ns) + self.assertEqual((second / "shared").read_bytes(), b"other") + def test_overlay_without_an_upper_is_read_only(self) -> None: with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: root = Path(directory) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 1b6cdaa1..b90aa0f1 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -1112,22 +1112,59 @@ package's directory does not exist when the tests run. ### Filesystem layer representation -Buck directory artifacts cannot faithfully store overlay whiteout devices, opaque-directory xattrs, or a -backslash in a path component. tine stores filesystem deltas in a regular-file representation: - -- `.wh.` represents a whiteout; -- `.wh..wh..opq` represents an opaque directory; -- `.esc.` represents a path component Buck cannot store. - -`rootfs.capture()` translates a native overlay upper into this form after unmounting. Before a later mount, -`rootfs.rootfs()` constructs sparse sidecar layers that translate the stored markers back to native -overlayfs whiteouts/xattrs and thaw escaped names. Stored layers remain ordinary Buck artifacts and can -therefore move through its CAS. - -Directory modes are made traversable so Buck can materialize/delete them, and Buck's artifact model does -not preserve general ownership, capabilities, xattrs, or every mode bit. Authored tmpfiles snippets can -recreate paths, modes, and xattrs during terminal assembly. Ownership is deliberately normalized to uid/gid -zero rather than reconstructed. SELinux labels are not currently produced. +An image-producing action writes a complete UAPI.16 snapshot before Buck publishes its outputs. Logical +paths, inode types, modes, nanosecond modification times, user extended attributes, symlink targets and +explicit hardlink groups are in the manifest. Regular-file contents are immutable SHA-256 blobs. Each store +contains only blobs absent from its parents; the ordered store list remains an explicit input to every +consumer. Content references use `blobs/` in the combined store namespace, resolved in store order. +A store with parents therefore needs its declared parent stores to resolve its references. Standalone +directory exports include every referenced blob. + +Equal bytes share storage without sharing an inode. Reproducible `inodeToken` counters identify explicit +hardlink groups independently of content digests. A metadata-only change changes the manifest and therefore +Buck's input digest. Capture reads the visible merged filesystem after scriptlets and authored operations; +deletions disappear from the next snapshot, including native OverlayFS whiteouts, opaque directories and +file/directory replacements. Logical names never become the file names of a snapshot store. Capture clamps +modification times to the action's `SOURCE_DATE_EPOCH`, so the same install at another time writes the same +manifest. + +Reconstruction first reflinks one blob into the temporary directory. If that works, it clones every file into +a private root. Each independent file gets its own inode with shared data extents, and only declared hardlink +groups share an inode. Metadata restoration and later writes leave cached blobs unchanged. A read-only root +is a read-only bind mount of the private root. Reconstruction tests a clone instead of checking the +filesystem type. Separate btrfs filesystems cannot reflink between each other, and other filesystems support +reflinks too. + +If the blob cannot be reflinked, reconstruction uses OverlayFS. Each independent file has a sparse metadata +inode in the normal lower layer and reads contents from the blob stores as data-only lower layers. Explicit +hardlink groups start as private clones or copies in the upper, with aliases linked to that private inode. +`index=off` would break a lower hardlink group on copy-up. Metadata restoration finishes before mounting. +Directory times are restored after their contents, and symlink metadata is applied without following the +link. + +On the overlay path, metadata changes and data writes copy independent files into the upper. `metacopy=off` +copies file data even for chmod; the kernel attempts a reflink and falls back to a byte copy. An unprivileged +mount cannot enable inode indexing. A command that creates a hardlink to an inherited independent file can +see different inode numbers for the source and alias even though writes affect both. Snapshot capture scans +the visible tree and uses the upper inode to identify the actual hardlink group. The complete snapshot +retains the group when the root is reconstructed or exported. + +Read-only overlays use only metadata inodes in the normal lower, with aliases linked for declared groups. +Boot selection, UKI assembly, artifact extraction and SBOM scanning select between private reflinks and +read-only overlays. A terminal reader with authored tmpfiles rules gets a writable root, because the rules +write into it. Repart uses full private files because it reads inode identity without access to the upper. +Parent release inspection reads the os-release file from the parent snapshot without reconstructing a root. +Temporary roots are removed when the operation exits, including on failure. There is no persistent +materialized-root cache. + +Boxes and other directory-layer callers of `rootfs` retain their direct mounting path and marker encoding: +`.wh.` represents a whiteout, `.wh..wh..opq` an opaque directory and `.esc.` an +unstorable component. Ordinary box startup does not parse image manifests or reconstruct image roots. + +Ownership remains uid/gid zero. Terminal outputs apply the image permission policy before authored tmpfiles +rules and clamp timestamps to the assembly epoch. Permission normalization prepares the snapshot before +filesystem reconstruction, so a read-only consumer needs no chmod or copy-up. Security labels, capabilities +and named ACL entries are unsupported. ### Selecting a distribution @@ -1195,35 +1232,34 @@ whether the key is set or not, so CI does not depend on it. ### Image construction The `image.layer` rule creates either an initial image from a package manager, or a derived image from -`parent`. `image.ImageInfo` carries the package manager, the box the image runs in, an ordered stack of -filesystem deltas, deferred tmpfiles snippets, and the canonical lazy package-database and SBOM artifacts. A +`parent`. `image.ImageInfo` carries the package manager, the box the image runs in, an ordered list of +snapshot stores, deferred tmpfiles snippets, and the canonical lazy package-database and SBOM artifacts. A derived image inherits the construction configuration but declares fresh metadata for its completed stack, so one composition cannot silently switch package sources or tooling environments between stages. -Each `image.layer` call applies one ordered operation sequence in one action and persists exactly one overlay -upper. A layer's `packages` and `package_sets` are rule attributes rather than operations: it installs them -as one request before any operation runs, while `run`, `copy`, `mkdir`, `symlink`, and `remove` mutate the -same root afterwards. A package set resolves its symbolic name through the image's package manager during -analysis and joins the concrete names in the same request, deduplicated. Installation is a property of the -layer rather than a position in it because a layer resolves one closure at analysis time over the stack below -it: a second install could only ever be solved blind to what the first one added, so there is nothing an -ordering could mean. Making it an attribute is also what lets operation sequences compose, since two lists -that both need packages no longer collide, and what lets a sequence name a package set and extra packages -together, which a set alone cannot express because its members are known only during analysis. Installing -against the result of an earlier install is a second layer, which is where a closure resolved over that -result becomes available. `copy` introduces a declared Buck artifact at an absolute image path, preserving -its position relative to the other operations. `run` executes the image's own tools in a chroot by default; -`chroot = False` instead executes box tooling with the image available at `/buildroot`. Its `env` argument -overlays variables on the box or image environment for that command. Package installation runs outside the -chroot. `copy`, `merge_os_release`, `mkdir`, `symlink`, `write_file` and `remove` run inside it, so ordinary -absolute symlinks resolve relative to the image. The `image.layer()` declaration macro recursively flattens -operation lists, -allowing reusable helpers to return ordered groups of operations. Every rule that accepts operations is -wrapped in a declaration macro that flattens them the same way, so a helper returning an ordered group -works identically in `image.layer()` and in a composition; the underlying attribute stays a flat, typed -list so Buck can track every embedded source dependency. An `image.install()` target carries `packages` -beside its operations, so a reusable target declares what it needs and `image.ops.install_from()` folds that -into the installing layer's own request. +Each `image.layer` call applies one ordered operation sequence in one action and persists one complete +snapshot with its new blobs. A layer's `packages` and `package_sets` are rule attributes rather than +operations: it installs them as one request before any operation runs, while `run`, `copy`, `mkdir`, +`symlink`, and `remove` mutate the same root afterwards. A package set resolves its symbolic name through the +image's package manager during analysis and joins the concrete names in the same request, deduplicated. +Installation is a property of the layer rather than a position in it because a layer resolves one closure at +analysis time over the stack below it: a second install could only ever be solved blind to what the first one +added, so there is nothing an ordering could mean. Making it an attribute is also what lets operation +sequences compose, since two lists that both need packages no longer collide, and what lets a sequence name a +package set and extra packages together, which a set alone cannot express because its members are known only +during analysis. Installing against the result of an earlier install is a second layer, which is where a +closure resolved over that result becomes available. `copy` introduces a declared Buck artifact at an +absolute image path, preserving its position relative to the other operations. `run` executes the image's own +tools in a chroot by default; `chroot = False` instead executes box tooling with the image available at +`/buildroot`. Its `env` argument overlays variables on the box or image environment for that command. Package +installation runs outside the chroot. `copy`, `merge_os_release`, `mkdir`, `symlink`, `write_file` and +`remove` run inside it, so ordinary absolute symlinks resolve relative to the image. The `image.layer()` +declaration macro recursively flattens operation lists, allowing reusable helpers to return ordered groups of +operations. Every rule that accepts operations is wrapped in a declaration macro that flattens them the same +way, so a helper returning an ordered group works identically in `image.layer()` and in a composition; the +underlying attribute stays a flat, typed list so Buck can track every embedded source dependency. An +`image.install()` target carries `packages` beside its operations, so a reusable target declares what it +needs and `image.ops.install_from()` folds that into the installing layer's own request. `install_langs` narrows the install to the translated files of the named languages, and `install_docs` drops documentation while keeping licenses. Neither ever enters the tree, so the package database records @@ -1232,7 +1268,7 @@ sequence, not the logical image's lifetime: an initrd needs neither, while the r want both. Each package system implements them in its own driver. A sequence that installs nothing ignores both, so a composition can forward them to a layer whose operations happen to skip installation. -An `image.layer` exposes its newly persisted delta, the full stack and canonical metadata through +An `image.layer` exposes its snapshot, parent stores and canonical metadata through `image.ImageInfo`, and the same metadata through lazy `[manifest]`, `[pkgdb]` and `[sbom]` subtargets. `image.ImageSbomInfo` gives typed consumers only the two SBOM formats. Declaring a logical image declares its metadata with it, because they are intrinsic metadata facets of every logical image: an @@ -1294,23 +1330,25 @@ Every terminal driver receives the ordered layer stack and the deferred tmpfiles extension receives only the snippets that it adds to its base, because a snippet of the base would create a file of the base in the extension. The driver applies the snippets with `systemd-tmpfiles --root` before reading or emitting image content; a missing tool is an error whenever finalization is needed. The directives -run against a disposable overlay upper and can create paths or restore modes and xattrs. Image-shipped +run against a temporary root and can create paths or restore modes and xattrs. Image-shipped tmpfiles configuration is not applied implicitly; enabling it will be an explicit output option once its single-UID/GID behavior is defined. Ownership and named ACL entries are deliberately unsupported: all archive entries use uid/gid zero. -Tar uses deterministic PAX archives and stores Linux xattrs using `SCHILY.xattr.*` headers. The newc cpio -format has no general xattr representation. Tar/cpio entries are ordered and mtimes are clamped to the fixed -assembly epoch. The cpio reader/writer aligns regular-file payloads and uses `copy_file_range` when possible +Tar uses deterministic PAX archives and stores user xattrs using `SCHILY.xattr.user.*` headers. The newc cpio +format has no general xattr representation. Tar/cpio entries are ordered and encode explicit hardlinks +independently of content identity. Linux's initramfs unpacker recreates hardlinked symlinks as separate +symlinks. Mtimes are clamped to the fixed assembly epoch. Tar retains nanoseconds; newc cpio uses whole +seconds. The cpio reader/writer aligns regular-file payloads and uses `copy_file_range` when possible so large archives can share extents on reflink-capable filesystems. `compression = "zstd"` compresses the finished archive in the same action, so the uncompressed form never becomes a Buck artifact; zstd's multi-threaded output is byte-identical to its single-threaded output, so this stays reproducible. The initrd -uses it, while the kernel-modules cpio that `uki.py` appends stays raw because the distribution already -ships each module compressed; the kernel unpacks the concatenation as independently compressed segments. +uses it, while the kernel-modules cpio that `uki.py` appends stays raw because the distribution already ships +each module compressed; the kernel unpacks the concatenation as independently compressed segments. `declare_ddi()` in `ddi.bzl` deliberately distinguishes `definitions` from `partitions`. Definitions describe -new partitions to populate directly from `image.ImageInfo`: repart mounts the delta stack with a disposable -overlay upper instead of first copying a directory artifact. Partition inputs are `image.DdiInfo` outputs +new partitions to populate directly from `image.ImageInfo`: repart reads a temporary root reconstructed +from the snapshot and blobs. Partition inputs are `image.DdiInfo` outputs from an earlier split call; their blocks are copied into the new disk with their resolved type and UUID preserved. Calls emit a disk by default. `split = True` additionally exposes each newly defined partition with normalized metadata alongside its block artifact; `disk = False` makes such a call partition-only. No @@ -1373,18 +1411,16 @@ never means restating what it already does, and `install_docs` and `install_lang image wants, since documentation and translations only cost an initrd boot memory. It generates before it prunes, so the hardware database it ships is compiled from the sources it then drops. The rule archives the image into the cpio itself, at the `compression` it declares, and returns both as `image.InitrdInfo` -alongside the image's own providers. That cpio omits the package database from the paths the image's -package system declares: nothing in an initrd resolves a dependency or verifies a package, and the kernel -unpacks the whole cpio into tmpfs, so shipping it would only cost boot memory. The `[manifest]`, `[pkgdb]` -and `[sbom]` views -read the tree rather than the archive, so they still describe the complete installed set. -A bootable `image.disk` consumes `image.InitrdInfo` rather than a bare `image.ImageInfo`, so the archive is -the initrd target's own output and its compression is declared where the initrd is. A composition given -no `initrd` declares `.initrd` for itself through the same macro, inheriting its package manager, -version, and distribution; the default is therefore an ordinary target that can be built and inspected on -its own rather than an anonymous step inside the disk image. The composition publishes the provider it was -given from `[initrd]` and returns the same -instance directly, so both interfaces describe exactly the same initrd. +alongside the image's own providers. That cpio omits the package database from the paths the image's package +system declares: nothing in an initrd resolves a dependency or verifies a package, and the kernel unpacks the +whole cpio into tmpfs, so shipping it would only cost boot memory. The `[manifest]` view describes the cpio +after package-database removal. The `[pkgdb]` and `[sbom]` views describe the complete installed set. A +bootable `image.disk` consumes `image.InitrdInfo` rather than a bare `image.ImageInfo`, so the archive is the +initrd target's own output and its compression is declared where the initrd is. A composition given no +`initrd` declares `.initrd` for itself through the same macro, inheriting its package manager, version, +and distribution; the default is therefore an ordinary target that can be built and inspected on its own +rather than an anonymous step inside the disk image. The composition publishes the provider it was given from +`[initrd]` and returns the same instance directly, so both interfaces describe exactly the same initrd. `uki.py` appends the kernel-modules cpio and runs `ukify`. On x86 it also packs the microcode the image ships under `/usr/lib/firmware/{amd,intel}-ucode` into a `.ucode` section: one concatenated file per vendor @@ -1675,11 +1711,13 @@ spec schema. Both it and the snapshot driver keep `--out` there too: the box loc targets let a caller name the file to write, or `-` for stdout when it has nowhere to write one, and one calling convention per driver beats splitting the destination by verb. -### Store image layers as deltas +### Store image snapshots with shared blobs -Copying a complete root for every image step scales with total image size rather than change size. Ordered -overlay deltas let child layers and terminal outputs reuse their ancestors. Encoding whiteouts and opaque -directories as regular files keeps those deltas compatible with Buck's artifact/CAS model. +Complete manifests describe visible filesystem state independently of Buck's metadata and filename limits. +Each action stores only newly introduced content blobs and retains its parent stores as Buck inputs. +Temporary reconstruction restores inode identity and metadata before commands run. Archive writers read +manifests and blobs directly when no authored tmpfiles rules require a filesystem. Directory exports restore +their authoritative snapshot after final Buck materialization. ### Prefer exact transactions over package-manager network access @@ -1853,8 +1891,8 @@ Near-term image gaps are: - richer ordered operations for setting file metadata directly; - deciding whether package installation and image tooling eventually need distinct compatible boxes. -The layer model should remain ordered operations captured as deltas. A provides/requires feature solver is -unnecessary unless real composition requirements appear. +The layer model should remain ordered operations captured as complete snapshots. A provides/requires feature +solver is unnecessary unless real composition requirements appear. ### Scale, configuration, and testing diff --git a/docs/user/images.md b/docs/user/images.md index f7063042..5369d94e 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: MPL-2.0 # Building images How to declare, build, and run OS images. This is the user guide. The architecture behind it (the -provider vocabulary, the delta layer model, and disk composition internals) is documented in +provider vocabulary, image snapshots, and disk composition internals) is documented in [architecture.md](../design/architecture.md). ## Restrictions @@ -21,6 +21,16 @@ writable directories such as `/etc` and `/var`. If you must have files with different permissions in your image, the escape hatch is the `tmpfiles` attribute of an image or composition. +Image operations see the original supported modes, modification times, user extended attributes and explicit +hardlinks from their parent snapshot. Permission normalization applies when producing terminal outputs, +before authored `tmpfiles` rules. Snapshots keep nanosecond timestamps, and capture clamps them to +`SOURCE_DATE_EPOCH`. Cpio stores whole seconds and has no extended attributes. Linux's +initramfs unpacker recreates hardlinked symlinks as separate symlinks. + +Image paths must satisfy [UAPI.16](https://uapi-group.org/specifications/specs/file_manifest/): normalized +relative UTF-8 paths without control characters. The top-level names `Uapi16Manifest` and +`Uapi16Manifest.*` are reserved. Backslashes and literal `.wh.*` and `.esc.*` names work without escaping. + A layer that adds a program or shared library which would run on an executable stack fails to build: one whose `PT_GNU_STACK` program header is executable, or which has none on x86, where glibc then falls back to an executable stack. @@ -398,7 +408,8 @@ at all. `install_docs = False` likewise installs no documentation, keeping the l Logical images and terminal outputs are separate rule families. Terminal rules merge the layer stack only when needed: -- `image.directory` materializes a Buck directory artifact and provides `image.ImageDirectoryInfo`; +- `image.directory` builds an authoritative manifest and content blobs and provides + `image.ImageDirectoryInfo`; - `image.uki` builds the unified kernel image for the image's single installed kernel from one or more cpio `image.ImageArchiveInfo` dependencies and provides `image.UkiInfo`, named `__.efi` (defaults: target name and `0`; systemd architecture spelling, e.g. `x86-64`), the shape diff --git a/image/archive_test.py b/image/archive_test.py index 4e595db2..43fbd5dd 100644 --- a/image/archive_test.py +++ b/image/archive_test.py @@ -51,9 +51,61 @@ def setUp(self) -> None: (source / "var/lib/database").mkdir(parents=True) (source / "var/lib/database/contents").write_text("database") self.store = self.scratch / "store" - storage.capture(source, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(source), self.store, epoch=CAPTURE_EPOCH) self.spec = finalize.ImageSpec(lower=[str(self.store)], tmpfiles=[]) + def test_archives_keep_hardlinks_created_by_commands(self) -> None: + # A command creates a hardlink to an inherited independent file. Upper-aware capture + # must preserve the group in tar and cpio without linking the equal-content file a. + child = self.scratch / "child" + with storage.root([self.store]) as restored: + tree = restored.path + os.link(tree / "b", tree / "new-link") + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + spec = finalize.ImageSpec(lower=[str(self.store), str(child)], tmpfiles=[]) + + # The archive writers must group paths by the captured inodeToken rather than blob digest. + with finalize.snapshot(spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, self.scratch / "image.tar", "tar", EPOCH, "none") + archive._archive(snapshot, self.scratch / "image.cpio", "cpio", EPOCH, "none") + with tarfile.open(self.scratch / "image.tar") as packed: + self.assertTrue(packed.getmember("./new-link").islnk()) + self.assertEqual(packed.getmember("./new-link").linkname, "./b") + self.assertTrue(packed.getmember("./b").isreg()) + entries = {entry.name: entry for entry in cpio.read((self.scratch / "image.cpio").read_bytes())} + self.assertEqual(entries["b"].ino, entries["new-link"].ino) + self.assertNotEqual(entries["a"].ino, entries["b"].ino) + + def test_terminal_readers_normalize_metadata_without_copying_contents(self) -> None: + # A terminal reader needs normalized modes and explicit hardlinks before it reads files. + # Preparing metadata before the read-only mount must avoid every blob copy. + (self.scratch / "source/tmp").mkdir(mode=0o700) + (self.scratch / "source/var/tmp").symlink_to("../empty-dir") + storage.capture(storage.Root(self.scratch / "source"), self.store, epoch=CAPTURE_EPOCH) + with mock.patch.object(storage, "clone_file", side_effect=AssertionError("unexpected copy")): + with finalize.image(self.spec, program="test", strategy="overlay") as tree: + self.assertEqual(stat.S_IMODE((tree / "a").stat().st_mode), 0o644) + self.assertEqual(stat.S_IMODE((tree / "b").stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((tree / "tmp").stat().st_mode), 0o1777) + self.assertTrue((tree / "var/tmp").is_symlink()) + self.assertEqual(stat.S_IMODE((tree / "empty-dir").stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((tree / "fifo").stat().st_mode), 0o600) + self.assertTrue((tree / "a").samefile(tree / "linked")) + self.assertFalse((tree / "a").samefile(tree / "b")) + self.assertEqual((tree / "a").read_text(), "contents") + self.assertEqual((tree / "a").stat().st_mtime_ns, 123) + + def test_terminal_readers_apply_tmpfiles_before_reading(self) -> None: + # Authored tmpfiles rules need a writable root even for a terminal reader. Their modes + # must override normalization and their new files must be present when the reader runs. + spec = finalize.ImageSpec( + lower=self.spec["lower"], tmpfiles=["z /a 0600 - - -", "f /created 0640 - - - authored"] + ) + with finalize.image(spec, program="test") as tree: + self.assertEqual(stat.S_IMODE((tree / "a").stat().st_mode), 0o600) + self.assertEqual((tree / "created").read_text(), "authored") + self.assertTrue((tree / "a").samefile(tree / "linked")) + def test_archive_writers_read_blobs_without_reconstruction(self) -> None: # A terminal without tmpfiles can package its snapshot directly. Tar must preserve # declared links, independent equal files, original nanoseconds and user attributes. @@ -288,7 +340,7 @@ def test_database_removal_resolves_intermediate_symlinks(self) -> None: (source / "var/lib").unlink(missing_ok=True) (source / "var/lib").symlink_to(target) store = self.scratch / f"store-{index}" - storage.capture(source, store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(source), store, epoch=CAPTURE_EPOCH) for rules in ([], ["d /empty-dir - - -"]): with self.subTest(target=target, rules=rules): spec = finalize.ImageSpec(lower=[str(store)], tmpfiles=rules) @@ -308,7 +360,7 @@ def test_database_removal_updates_the_root_timestamp(self) -> None: if not present: shutil.rmtree(source / "var/lib/database") os.utime(source, ns=(111, 111)) - storage.capture(source, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(source), self.store, epoch=CAPTURE_EPOCH) for rules in ([], ["d /empty-dir - - -"]): with self.subTest(present=present, rules=rules): spec = finalize.ImageSpec(lower=self.spec["lower"], tmpfiles=rules) @@ -343,7 +395,7 @@ def test_missing_database_leaves_nonempty_parent_times_unchanged(self) -> None: (source / "var/lib/keep").touch() os.utime(source / "var/lib", ns=(333, 333)) os.utime(source, ns=(111, 111)) - storage.capture(source, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(source), self.store, epoch=CAPTURE_EPOCH) with finalize.snapshot( self.spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] ) as snapshot: diff --git a/image/ddi.py b/image/ddi.py index 26a3a338..15da8294 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -260,14 +260,15 @@ def _extension_seed(identity: str, name: str, releases: list[Release]) -> uuid.U return uuid.uuid5(_EXTENSION_SEED_NAMESPACE, digest.hexdigest()) -def _os_release(tree: Path) -> dict[str, str]: +def _os_release(snapshot: storage.Snapshot) -> dict[str, str]: """Parse the parent image's os-release for strict extension matching.""" + objects = storage.by_name(snapshot.objects) for candidate in ("usr/lib/os-release", "etc/os-release"): - path = tree / candidate - if not path.is_file(): + obj = objects.get(storage.resolve(objects, candidate)) + if obj is None or obj["type"] != "reg": continue fields = {} - for line in path.read_text().splitlines(): + for line in snapshot.content(obj).read_text().splitlines(): key, sep, value = line.strip().partition("=") if key and sep and not key.startswith("#"): fields[key] = value.strip("\"'") @@ -290,21 +291,18 @@ def pin_release(parent: dict[str, str], release: dict[str, str]) -> dict[str, st return strict | release -def _extension_layers(spec: Spec, extension: ExtensionSpec) -> tuple[list[Release], list[str | Path]]: - """Pin release fields to the parent and retain every store needed to resolve contents.""" +def _extension_releases(spec: Spec, extension: ExtensionSpec) -> list[Release]: releases = extension["releases"] parent = extension["parent"] - lowers: list[str | Path] = list(spec["lower"]) if not parent: - return releases, lowers - with storage.root(lowers[:parent]) as tree: - fields = _os_release(tree) + return releases + fields = _os_release(storage.Snapshot(spec["lower"][:parent])) if "ID" not in fields: util.fail("ddi: the parent os-release lacks ID") pinned = [ Release(path=release["path"], fields=pin_release(fields, release["fields"])) for release in releases ] - return pinned, lowers + return pinned def empty_directories(tree: Path, directories: list[str]) -> None: @@ -468,13 +466,15 @@ def main(argv: list[str] | None = None) -> None: for index, partition in enumerate(partitions, start=len(written)) ] extension = spec["extension"] - releases: list[Release] = [] - lowers = None - if extension: - releases, lowers = _extension_layers(spec, extension) + releases = _extension_releases(spec, extension) if extension else [] with ( finalize.image( - spec, program="ddi", lowers=lowers, binds=binds, base=extension["parent"] if extension else 0 + spec, + program="ddi", + binds=binds, + base=extension["parent"] if extension else 0, + readonly=False, + strategy="copy", ) as tree, tempfile.TemporaryDirectory(prefix="repart.") as scratch_dir, ): diff --git a/image/finalize.py b/image/finalize.py index 5642ce75..2dd04f8c 100644 --- a/image/finalize.py +++ b/image/finalize.py @@ -15,7 +15,6 @@ import gzip import os import shutil -import stat import subprocess import sys import tempfile @@ -37,53 +36,49 @@ class ImageSpec(TypedDict): tmpfiles: list[str] -def _normalize_mode(path: Path) -> None: - """Normalize file permissions for path. - - See docs/user/images.md. - """ - mode = path.lstat().st_mode - desired = 0o755 if stat.S_ISDIR(mode) or mode & 0o111 else 0o644 - if (stat.S_ISDIR(mode) or stat.S_ISREG(mode)) and stat.S_IMODE(mode) != desired: - path.chmod(desired) - - -def normalize_modes(tree: Path) -> None: - _normalize_mode(tree) - for path in tree.rglob("*"): - _normalize_mode(path) - - _WORLD_WRITABLE = ("tmp", "var/tmp") -def world_writable(tree: Path) -> None: - """Apply the temporary-directory exception to the image permission policy.""" - for name in _WORLD_WRITABLE: - path = tree / name - if path.is_dir() and not path.is_symlink() and stat.S_IMODE(path.stat().st_mode) != 0o1777: - path.chmod(0o1777) - - @contextlib.contextmanager def image( spec: ImageSpec, *, program: str, - lowers: Sequence[str | Path] | None = None, binds: Sequence[tuple[str | Path, str | Path]] | None = None, base: int = 0, + readonly: bool = True, + strategy: storage.Strategy = "auto", ) -> Generator[Path]: """Mount the image stack the spec names and yield the finalized tree.""" + with _root( + spec, program=program, binds=binds, base=base, readonly=readonly, strategy=strategy + ) as restored: + yield restored.path + + +@contextlib.contextmanager +def _root( + spec: ImageSpec, + *, + program: str, + binds: Sequence[tuple[str | Path, str | Path]] | None = None, + base: int = 0, + readonly: bool = True, + strategy: storage.Strategy = "auto", +) -> Generator[storage.Root]: with ( - storage.root(spec["lower"] if lowers is None else lowers, base=base) as restored, - rootfs.rootfs("/buildroot", bind=restored, binds=binds) as tree, + storage.root( + spec["lower"], + base=base, + # Authored tmpfiles rules write into the root. They fail on a read-only root. + readonly=readonly and not spec["tmpfiles"], + strategy=strategy, + prepare=normalize_snapshot, + ) as restored, + rootfs.rootfs("/buildroot", bind=restored.path, binds=binds) as tree, ): - normalize_modes(tree) - # Before the snippets, so an image that states a mode of its own still wins. - world_writable(tree) apply_tmpfiles(tree, spec["tmpfiles"], program=program) - yield tree + yield storage.Root(tree, restored.upper) def apply_tmpfiles(tree: Path, snippets: list[str], *, program: str) -> None: @@ -373,6 +368,14 @@ def locale_gen(tree: Path) -> None: fail(f"locale_gen: `{' '.join(cmd)}` failed (rc={rc})") +def normalize_snapshot(snapshot: storage.Snapshot) -> None: + for obj in snapshot.objects: + if obj["type"] == "dir": + obj["mode"] = 0o1777 if obj.get("name") in _WORLD_WRITABLE else 0o755 + elif obj["type"] == "reg": + obj["mode"] = 0o755 if obj["mode"] & 0o111 else 0o644 + + def _remove(objects: dict[str, uapi16.Object], relative: str, epoch: int) -> None: path = Path(relative) name = storage.resolve(objects, relative, follow=False) @@ -411,16 +414,12 @@ def snapshot( with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: if spec["tmpfiles"]: out = Path(scratch) / "snapshot" - with image(spec, program=program) as tree: - storage.capture(tree, out, spec["lower"], epoch=epoch) + with _root(spec, program=program, readonly=False) as restored: + storage.capture(restored, out, spec["lower"], epoch=epoch) result = storage.Snapshot([*spec["lower"], out]) else: result = storage.Snapshot(spec["lower"]) - for obj in result.objects: - if obj["type"] == "dir": - obj["mode"] = 0o1777 if obj.get("name") in _WORLD_WRITABLE else 0o755 - elif obj["type"] == "reg": - obj["mode"] = 0o755 if obj["mode"] & 0o111 else 0o644 + normalize_snapshot(result) objects = storage.by_name(result.objects) for relative in pkgdb_paths: _remove(objects, relative, epoch) diff --git a/image/finalize_test.py b/image/finalize_test.py index e24d7271..18740743 100644 --- a/image/finalize_test.py +++ b/image/finalize_test.py @@ -224,27 +224,3 @@ def test_writes_into_the_archive_of_the_image(self) -> None: "en_US.UTF-8", ], ) - - -class TestWorldWritable(unittest.TestCase): - @override - def setUp(self) -> None: - self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="finalize."))) - - def test_restores_the_mode_a_layer_could_not_carry(self) -> None: - for name in ("tmp", "var/tmp"): - (self.tree / name).mkdir(parents=True, mode=0o755) - finalize.world_writable(self.tree) - for name in ("tmp", "var/tmp"): - self.assertEqual((self.tree / name).stat().st_mode & 0o7777, 0o1777) - - def test_leaves_an_image_that_ships_neither_alone(self) -> None: - finalize.world_writable(self.tree) - self.assertFalse((self.tree / "tmp").exists()) - - def test_does_not_chmod_through_a_symlink(self) -> None: - (self.tree / "var").mkdir() - (self.tree / "elsewhere").mkdir(mode=0o755) - (self.tree / "var/tmp").symlink_to("../elsewhere") - finalize.world_writable(self.tree) - self.assertEqual((self.tree / "elsewhere").stat().st_mode & 0o7777, 0o755) diff --git a/image/layer.py b/image/layer.py index e0360dbd..dfac79b6 100755 --- a/image/layer.py +++ b/image/layer.py @@ -293,9 +293,10 @@ def main(argv: list[str] | None = None) -> None: lower = spec["lower"] with ( - storage.root(lower) as tree, + storage.root(lower) as restored, tempfile.TemporaryDirectory(prefix="layer.", dir="/var/tmp") as scratch, ): + tree = restored.path with rootfs.rootfs("/buildroot", bind=tree, apivfs=True, strict_unmount=True) as target: if install is not None: _install(install, target, Path(scratch)) @@ -306,7 +307,7 @@ def main(argv: list[str] | None = None) -> None: util.fail( "image: executable stacks are not allowed, see docs/user/images.md:\n" + "\n".join(stacks) ) - storage.capture(tree, out, lower, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) + storage.capture(restored, out, lower, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) print( f"image: applied {len(operations)} ops over {len(lower)} parent store(s) -> {out}", file=sys.stderr ) diff --git a/image/layer_test.py b/image/layer_test.py index 7e3ace60..049b5c14 100644 --- a/image/layer_test.py +++ b/image/layer_test.py @@ -90,7 +90,7 @@ def test_a_derived_layer_exposes_a_readonly_project_only_to_copy_and_run(self) - (lower / "cache.pyc").write_text("image cache", encoding="utf-8") (self.project / "cache.pyc").write_text("cache", encoding="utf-8") stored = self.scratch / "stored" - storage.capture(lower, stored, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(lower), stored, epoch=CAPTURE_EPOCH) def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: self.assertEqual(cwd, layer.PROJECT) @@ -111,7 +111,8 @@ def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: ) command.assert_called_once() - with storage.root([stored, self.out]) as tree: + with storage.root([stored, self.out]) as restored: + tree = restored.path self.assertEqual((tree / "copied").read_text(encoding="utf-8"), "copied") self.assertFalse((tree / "cache.pyc").exists()) self.assertTrue((self.project / "cache.pyc").exists()) @@ -126,7 +127,7 @@ def test_commands_see_inherited_metadata_and_can_override_it(self) -> None: os.utime(source / "file", ns=(123, 123)) os.link(source / "file", source / "linked") parent = self.scratch / "parent" - storage.capture(source, parent, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(source), parent, epoch=CAPTURE_EPOCH) copied = self.scratch / "copied-parent" shutil.copytree(parent, copied, copy_function=shutil.copyfile) @@ -140,7 +141,8 @@ def command(*args: object) -> None: with mock.patch.object(layer, "_run", side_effect=command): self._build([["run", ["command"], {}, False]], copied) - with storage.root([copied, self.out]) as tree: + with storage.root([copied, self.out]) as restored: + tree = restored.path self.assertEqual(stat.S_IMODE((tree / "file").stat().st_mode), 0o600) self.assertEqual((tree / "linked").stat().st_mtime_ns, 456) self.assertEqual(list((self.out / "blobs").iterdir()), []) @@ -165,6 +167,28 @@ def command(*args: object) -> None: if Path("/buildroot").is_mount(): umount2("/buildroot", 0) + def test_commands_create_hardlinks_to_inherited_files(self) -> None: + # A command links an inherited independent file and changes its data and mode. Capture + # must use upper inode identity so the next image keeps the new aliases hardlinked. + source = self.scratch / "source-root" + source.mkdir() + (source / "file").write_text("contents") + parent = self.scratch / "parent" + storage.capture(storage.Root(source), parent, epoch=CAPTURE_EPOCH) + + def command(*args: object) -> None: + os.link("/buildroot/file", "/buildroot/linked") + Path("/buildroot/file").write_text("changed") + Path("/buildroot/linked").chmod(0o600) + + with mock.patch.object(layer, "_run", side_effect=command): + self._build([["run", ["command"], {}, False]], parent) + with storage.root([parent, self.out]) as restored: + tree = restored.path + self.assertTrue((tree / "file").samefile(tree / "linked")) + self.assertEqual((tree / "linked").read_text(), "changed") + self.assertEqual(stat.S_IMODE((tree / "file").stat().st_mode), 0o600) + class TestRemove(unittest.TestCase): def test_plain_path_is_also_a_glob(self) -> None: @@ -225,8 +249,8 @@ def test_modes(self) -> None: # Terminal policy normalizes regular files and directories without following links or # changing special-file modes. Authored tmpfiles rules run after this normalization. with tempfile.TemporaryDirectory(prefix="layer-test.", dir="/var/tmp") as scratch: - tree = Path(scratch) - tree.chmod(0o700) + tree = Path(scratch) / "source" + tree.mkdir(mode=0o700) (tree / "sub").mkdir() (tree / "sub").chmod(0o1777) (tree / "sub/file").write_text("file") @@ -240,7 +264,12 @@ def test_modes(self) -> None: (tree / "link").symlink_to("sub/secret") os.mkfifo(tree / "fifo", 0o600) - finalize.normalize_modes(tree) + store = Path(scratch) / "store" + storage.capture(storage.Root(tree), store, epoch=CAPTURE_EPOCH) + snapshot = storage.Snapshot([store]) + finalize.normalize_snapshot(snapshot) + tree = Path(scratch) / "normalized" + storage.restore(snapshot, tree) self.assertEqual(tree.stat().st_mode & 0o7777, 0o755) self.assertEqual((tree / "sub").stat().st_mode & 0o7777, 0o755) @@ -249,7 +278,6 @@ def test_modes(self) -> None: self.assertEqual((tree / "sub/script").stat().st_mode & 0o7777, 0o755) self.assertEqual((tree / "sub/mount").stat().st_mode & 0o7777, 0o755) self.assertTrue((tree / "link").is_symlink()) - # not content: whatever it is, it is not ours to rewrite self.assertEqual((tree / "fifo").lstat().st_mode & 0o7777, 0o600) diff --git a/package/transaction.py b/package/transaction.py index 26272f22..913230cd 100644 --- a/package/transaction.py +++ b/package/transaction.py @@ -182,8 +182,8 @@ def installed_root(lowers: Sequence[str]) -> Generator[Path]: # A snapshot store contains `uapi16.MANIFEST` at its top level. A box layer is a plain directory # tree, and no package installs a file of that name into the root. if (Path(lowers[-1]) / uapi16.MANIFEST).exists(): - with storage.root(lowers) as tree: - yield tree + with storage.root(lowers) as restored: + yield restored.path else: with rootfs.rootfs("/installroot", lowers=lowers) as tree: yield tree diff --git a/uapi16/storage.py b/uapi16/storage.py index 225df713..c9fc82ab 100644 --- a/uapi16/storage.py +++ b/uapi16/storage.py @@ -4,16 +4,21 @@ """Store complete filesystem snapshots independently of artifact filesystem metadata.""" import os +import re import stat import tempfile -from collections.abc import Generator, Sequence +from collections.abc import Callable, Generator, Sequence from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, replace from pathlib import Path, PurePosixPath +from typing import Literal -from util import clone_file, fail +from util import ReflinkUnavailable, clone_file, fail, reflink import uapi16 -from isolation import Overlay +from isolation import Bind, Overlay + +type Strategy = Literal["auto", "copy", "overlay"] class Snapshot: @@ -54,34 +59,50 @@ def _validate(self) -> None: def content(self, obj: uapi16.Object) -> Path: digest = obj["sha256"] - if ( - not isinstance(digest, str) - or len(digest) != 64 - or any(c not in "0123456789abcdef" for c in digest) - ): + if not isinstance(digest, str) or not re.fullmatch("[0-9a-f]{64}", digest): fail(f"storage: invalid SHA-256 digest {digest!r}") if digest not in self.blobs: fail(f"storage: missing blob {digest} for {obj['name']!r}") path = self.blobs[digest] - if not stat.S_ISREG(path.lstat().st_mode) or path.stat().st_size != obj["size"]: + st = path.lstat() + if not stat.S_ISREG(st.st_mode) or st.st_size != obj["size"]: fail(f"storage: invalid blob {digest} for {obj['name']!r}") return path +@dataclass(frozen=True) +class Root: + path: Path + upper: Path | None = None + + def entries(self) -> Generator[uapi16.Entry]: + for entry in uapi16.entries(self.path): + if self.upper is not None and entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode): + # OverlayFS can report different origin inodes for paths linked after copy-up. + # The upper inode identifies the hardlink group that commands actually modified. + try: + st = (self.upper / entry.name).lstat() + except FileNotFoundError: + pass + else: + entry = replace(entry, st=st) + yield entry + + def add_blob(source: Path, destination: Path) -> None: if not destination.exists(): clone_file(source, destination) destination.chmod(0o444) -def capture(tree: Path, out: Path, parents: Sequence[str | Path] = (), *, epoch: int) -> None: +def capture(tree: Root, out: Path, parents: Sequence[str | Path] = (), *, epoch: int) -> None: """Write the snapshot of `tree` to `out`, with modification times clamped to `epoch`.""" parent = Snapshot(parents) blobs = out / "blobs" blobs.mkdir(parents=True, exist_ok=True) - found = list(uapi16.entries(tree)) + found = list(tree.entries()) tokens = uapi16.hardlink_tokens(found) - root = uapi16.file_object(uapi16.Entry("", tree, tree.lstat()), epoch=epoch) + root = uapi16.file_object(uapi16.Entry("", tree.path, tree.path.lstat()), epoch=epoch) root.pop("name") root["mediaType"] = uapi16.MEDIA_TYPE objects = [root] @@ -155,27 +176,52 @@ def metadata(path: Path, obj: uapi16.Object) -> None: os.utime(path, ns=(obj["mTimeNSec"], obj["mTimeNSec"]), follow_symlinks=False) -def restore(snapshot: Snapshot, tree: Path, *, hardlinks: bool | None = None) -> None: - """Restore private inodes, optionally selecting linked or independent files.""" +def restore( + snapshot: Snapshot, + tree: Path, + *, + metacopy: bool = False, + upper: Path | None = None, +) -> None: + """Restore private files or prepare metadata and hardlinks for OverlayFS.""" tree.mkdir(parents=True, exist_ok=True) groups: dict[int, Path] = {} directories = [(tree, snapshot.objects[0])] + linked_parents = set() + if upper is not None: + upper.mkdir() + directories.append((upper, snapshot.objects[0])) + linked_parents = { + str(parent) + for obj in snapshot.objects[1:] + if obj.get("inodeToken") + for parent in Path(obj["name"]).parents + } for obj in snapshot.objects[1:]: - path = tree / obj["name"] kind = obj["type"] token = obj.get("inodeToken") + # index=off breaks lower hardlinks on copy-up, so writable groups start in the upper. + private = upper is not None and token + path = (upper if private else tree) / obj["name"] if kind == "dir": path.mkdir() directories.append((path, obj)) - continue - if hardlinks is not None and bool(token) != hardlinks: + if upper is not None and obj["name"] in linked_parents: + (upper / obj["name"]).mkdir() + directories.append((upper / obj["name"], obj)) continue if token and token in groups: os.link(groups[token], path, follow_symlinks=False) continue if kind == "reg": - # Metadata restoration and writes must never change a cached blob's inode. - clone_file(snapshot.content(obj), path) + if metacopy and not private: + with path.open("wb") as stream: + stream.truncate(obj["size"]) + os.setxattr(path, "user.overlay.metacopy", b"") + os.setxattr(path, "user.overlay.redirect", ("/" + obj["sha256"]).encode()) + else: + # Metadata restoration and writes must never change a cached blob's inode. + clone_file(snapshot.content(obj), path) elif kind == "lnk": path.symlink_to(uapi16.symlink_target(obj)) elif kind == "fifo": @@ -209,8 +255,28 @@ def _identities(objects: Sequence[uapi16.Object]) -> dict[str, tuple[uapi16.Obje return identities +def _reflinks(snapshot: Snapshot, scratch: Path) -> bool: + # clone_file() reflinks through copy_file_range() where the filesystem supports reflinks. On other + # filesystems it copies every byte, and copying every blob is slower than an overlay mount. + blob = next(iter(snapshot.blobs.values()), None) + if blob is None: + return True + try: + reflink(blob, scratch / "reflink") + except ReflinkUnavailable: + return False + return True + + @contextmanager -def root(stores: Sequence[str | Path], *, base: int = 0) -> Generator[Path]: +def root( + stores: Sequence[str | Path], + *, + base: int = 0, + readonly: bool = False, + strategy: Strategy = "auto", + prepare: Callable[[Snapshot], None] | None = None, +) -> Generator[Root]: snapshot = Snapshot(stores) if base: inherited = _identities(Snapshot(stores[:base]).objects[1:]) @@ -221,14 +287,32 @@ def root(stores: Sequence[str | Path], *, base: int = 0) -> Generator[Path]: snapshot.objects = [snapshot.objects[0]] + [ obj for obj in snapshot.objects[1:] if obj["name"] in changed or obj["name"] in needed ] - with ExitStack() as stack: - scratch = Path( - stack.enter_context(tempfile.TemporaryDirectory(prefix="snapshot-root.", dir="/var/tmp")) - ) - lower, upper, work, merged = (scratch / name for name in ("lower", "upper", "work", "root")) - restore(snapshot, lower, hardlinks=False) - # index=off splits lower hardlinks on copy-up. Upper hardlinks keep their inode on writes. - restore(snapshot, upper, hardlinks=True) - work.mkdir() - stack.enter_context(Overlay((lower,), upper, work, merged)).mount() - yield merged + if prepare is not None: + prepare(snapshot) + # Overlay refuses a read-only mount with one lower layer and no data layer. An empty snapshot + # has no blob directories to use as data layers. + if not snapshot.stores: + strategy = "copy" + with tempfile.TemporaryDirectory(prefix="snapshot-root.", dir="/var/tmp") as temporary: + scratch = Path(temporary) + tree = scratch / "root" + if strategy == "auto": + strategy = "copy" if _reflinks(snapshot, scratch) else "overlay" + if strategy == "copy": + restore(snapshot, tree) + with ExitStack() as stack: + if readonly: + bound = stack.enter_context(Bind(tree, tree, readonly=True)) + bound.mount() + yield Root(tree) + return + lower = scratch / "lower" + upper = None if readonly else scratch / "upper" + work = None if readonly else scratch / "work" + restore(snapshot, lower, metacopy=True, upper=upper) + if work is not None: + work.mkdir() + datadirs = tuple(store / "blobs" for store in snapshot.stores) + with Overlay((lower,), upper, work, tree, datadirs=datadirs) as mounted: + mounted.mount() + yield Root(tree, upper) diff --git a/uapi16/storage_test.py b/uapi16/storage_test.py index f5747006..0beb1f15 100644 --- a/uapi16/storage_test.py +++ b/uapi16/storage_test.py @@ -3,6 +3,7 @@ """Snapshot storage and reconstruction regressions.""" +import errno import os import shutil import socket @@ -11,9 +12,13 @@ import unittest from pathlib import Path from typing import override +from unittest import mock + +from util import ReflinkUnavailable, clone_file import storage import uapi16 +from isolation import Overlay # Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. CAPTURE_EPOCH = 2**31 @@ -46,22 +51,31 @@ def test_roundtrip_after_artifact_metadata_is_lost(self) -> None: os.utime(self.tree / "link", ns=(333, 333), follow_symlinks=False) os.utime(self.tree / "empty", ns=(444, 444)) os.setxattr(self.tree / "a", "user.example", b"\x00\xff") - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) copied = self.scratch / "copied" shutil.copytree(self.store, copied, copy_function=shutil.copyfile) - restored = self.scratch / "restored" - storage.restore(storage.Snapshot([copied]), restored) - self.assertEqual((restored / "a").stat().st_mtime_ns, 111) - self.assertEqual(stat.S_IMODE((restored / "a").stat().st_mode), 0o640) - self.assertEqual((restored / "b").stat().st_mtime_ns, 1800000000000000222) - self.assertEqual(stat.S_IMODE((restored / "b").stat().st_mode), 0o755) - self.assertTrue((restored / "a").samefile(restored / "linked")) - self.assertFalse((restored / "a").samefile(restored / "b")) - self.assertEqual(os.getxattr(restored / "a", "user.example"), b"\x00\xff") - self.assertEqual(os.readlink(restored / "link"), "/a") - self.assertEqual((restored / "link").lstat().st_mtime_ns, 333) - self.assertEqual((restored / "empty").stat().st_mtime_ns, 444) - self.assertEqual({p.name for p in restored.iterdir()}, {p.name for p in self.tree.iterdir()}) + materialized = self.scratch / "restored" + storage.restore(storage.Snapshot([copied]), materialized) + with ( + storage.root([copied], readonly=True) as readonly, + storage.root([copied]) as restored, + ): + for tree in (materialized, readonly.path, restored.path): + with self.subTest(tree=tree): + self.assertEqual((tree / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((tree / "a").stat().st_mode), 0o640) + self.assertEqual((tree / "b").stat().st_mtime_ns, 1800000000000000222) + self.assertEqual(stat.S_IMODE((tree / "b").stat().st_mode), 0o755) + self.assertTrue((tree / "a").samefile(tree / "linked")) + self.assertFalse((tree / "a").samefile(tree / "b")) + self.assertEqual(os.getxattr(tree / "a", "user.example"), b"\x00\xff") + self.assertEqual(os.readlink(tree / "link"), "/a") + self.assertEqual((tree / "link").lstat().st_mtime_ns, 333) + self.assertEqual((tree / "empty").stat().st_mtime_ns, 444) + self.assertEqual({p.name for p in tree.iterdir()}, {p.name for p in self.tree.iterdir()}) + child = self.scratch / "child" + storage.capture(restored, child, [copied], epoch=CAPTURE_EPOCH) + self.assertEqual((child / uapi16.MANIFEST).read_bytes(), (copied / uapi16.MANIFEST).read_bytes()) def test_capture_clamps_times_to_the_epoch(self) -> None: # A package manager writes the time of the install into every file and directory that it @@ -69,21 +83,29 @@ def test_capture_clamps_times_to_the_epoch(self) -> None: # installs at different times write the same manifest. Capture a tree with one file before and # one file after the epoch, and check the times in the manifest. self.files() - storage.capture(self.tree, self.store, epoch=1700000000) + storage.capture(storage.Root(self.tree), self.store, epoch=1700000000) objects = {obj.get("name"): obj for obj in storage.Snapshot([self.store]).objects} self.assertEqual(objects["a"]["mTimeNSec"], 111) self.assertEqual(objects["b"]["mTimeNSec"], 1700000000 * 10**9) - def test_overlay_writes_and_metadata_leave_blobs_unchanged(self) -> None: - # Declared links start in the upper because index=off splits lower links on writes. - # Equal-content independent files must copy up separately without changing the blob. + def test_writes_and_metadata_leave_blobs_unchanged(self) -> None: + # Writes and metadata changes in a restored root must preserve declared hardlinks, keep + # equal-content independent files separate and leave the cached blob unchanged. self.files() - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) blob = next((self.store / "blobs").iterdir()) before = blob.read_bytes(), blob.stat() - with storage.root([self.store]) as root: + with storage.root([self.store]) as restored: + root = restored.path self.assertTrue((root / "a").samefile(root / "linked")) self.assertFalse((root / "a").samefile(root / "b")) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + self.assertEqual((root / "b").stat().st_mtime_ns, 1800000000000000222) + (root / "b").chmod(0o604) + os.chown(root / "b", 0, 0, follow_symlinks=False) + os.utime(root / "b", ns=(444, 444)) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((root / "a").stat().st_mode), 0o640) (root / "a").write_text("linked write") (root / "linked").chmod(0o600) os.utime(root / "a", ns=(555, 555)) @@ -98,33 +120,223 @@ def test_overlay_writes_and_metadata_leave_blobs_unchanged(self) -> None: self.assertEqual(blob.stat().st_mode, before[1].st_mode) self.assertEqual(blob.stat().st_uid, before[1].st_uid) + def test_readonly_reconstruction_never_copies_blob_data(self) -> None: + # Sparse lower metadata files restore independent files and declared hardlinks without + # copying any data. Reads preserve metadata, while writes and chmod fail with EROFS. + self.files() + (self.tree / "nested").mkdir() + (self.tree / "independent").write_text("identical") + (self.tree / "empty").touch() + os.link(self.tree / "a", self.tree / "nested/linked") + os.utime(self.tree / "nested", ns=(999, 999)) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with mock.patch.object(storage, "clone_file", wraps=storage.clone_file) as copy: + with storage.root([self.store], readonly=True, strategy="overlay") as restored: + root = restored.path + self.assertEqual(copy.call_count, 0) + self.assertTrue((root / "a").samefile(root / "nested/linked")) + self.assertFalse((root / "b").samefile(root / "independent")) + self.assertEqual((root / "independent").read_text(), "identical") + self.assertEqual((root / "empty").read_bytes(), b"") + self.assertEqual((root / "nested").stat().st_mtime_ns, 999) + self.assertEqual(copy.call_count, 0) + for operation in ( + lambda: (root / "a").write_text("changed"), + lambda: (root / "independent").chmod(0o600), + ): + with self.assertRaises(OSError) as error: + operation() + self.assertEqual(error.exception.errno, errno.EROFS) + + def test_commands_can_create_new_hardlinks(self) -> None: + # A command creates a hardlink to an inherited independent file. Writes must update the + # alias, and capturing the result must record a hardlink group without changing its blob. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root([self.store], strategy="overlay") as restored: + root = restored.path + os.link(root / "b", root / "new-link") + (root / "b").write_text("new group") + (root / "new-link").chmod(0o600) + os.utime(root / "new-link", ns=(777, 777)) + self.assertEqual((root / "new-link").read_text(), "new group") + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o600) + entries = {entry.name: entry for entry in restored.entries()} + self.assertEqual(entries["b"].inode, entries["new-link"].inode) + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child]) as restored: + root = restored.path + self.assertTrue((root / "b").samefile(root / "new-link")) + self.assertEqual((root / "new-link").read_text(), "new group") + self.assertEqual((root / "b").stat().st_mtime_ns, 777) + self.assertEqual((root / "a").read_text(), "identical") + + def test_extension_compares_hardlink_membership(self) -> None: + # An earlier new group changes token numbers without changing inherited files. Extension + # selection must omit those files and include every member when a group gains an alias. + (self.tree / "z").write_text("inherited") + os.link(self.tree / "z", self.tree / "z-linked") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + (self.tree / "a").write_text("new group") + os.link(self.tree / "a", self.tree / "a-linked") + child = self.scratch / "child" + storage.capture(storage.Root(self.tree), child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child], base=1, strategy="copy") as restored: + self.assertEqual({p.name for p in restored.path.iterdir()}, {"a", "a-linked"}) + self.assertTrue((restored.path / "a").samefile(restored.path / "a-linked")) + + # A new alias changes group membership even when the original members keep the same token. + (self.tree / "a").unlink() + (self.tree / "a-linked").unlink() + os.link(self.tree / "z", self.tree / "new-alias") + alias = self.scratch / "alias" + storage.capture(storage.Root(self.tree), alias, [self.store], epoch=CAPTURE_EPOCH) + with storage.root([self.store, alias], base=1, strategy="copy") as restored: + self.assertEqual({p.name for p in restored.path.iterdir()}, {"z", "z-linked", "new-alias"}) + self.assertTrue((restored.path / "z").samefile(restored.path / "new-alias")) + self.assertTrue((restored.path / "z-linked").samefile(restored.path / "new-alias")) + + def test_writable_reconstruction_copies_only_declared_groups(self) -> None: + # A writable overlay needs private data only for declared hardlink groups. Independent + # files must read from blobs until chmod or a write copies the affected path into the upper. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with mock.patch.object(storage, "clone_file", wraps=storage.clone_file) as copy: + with storage.root([self.store], strategy="overlay") as restored: + root = restored.path + assert restored.upper is not None + self.assertEqual(copy.call_count, 1) + self.assertTrue((restored.upper / "a").samefile(restored.upper / "linked")) + self.assertFalse((restored.upper / "b").exists()) + self.assertEqual((root / "b").read_text(), "identical") + self.assertFalse((restored.upper / "b").exists()) + (root / "b").chmod(0o600) + self.assertTrue((restored.upper / "b").exists()) + self.assertEqual(copy.call_count, 1) + + def test_available_reflinks_use_private_inodes_without_overlay(self) -> None: + # Simulate successful reflinks with private files on the test filesystem. Automatic + # reconstruction must preserve inode independence, hardlinks and metadata without OverlayFS. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + blob = next((self.store / "blobs").iterdir()) + before = blob.read_bytes(), blob.stat() + + for readonly in (False, True): + with ( + self.subTest(readonly=readonly), + mock.patch.object(storage, "reflink", side_effect=clone_file), + mock.patch.object(storage, "Overlay", side_effect=AssertionError("unexpected overlay")), + storage.root([self.store], readonly=readonly) as restored, + ): + root = restored.path + self.assertIsNone(restored.upper) + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertFalse((root / "a").samefile(root / "b")) + self.assertFalse((root / "a").samefile(blob)) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o755) + if readonly: + with self.assertRaises(OSError) as error: + (root / "a").write_text("changed") + self.assertEqual(error.exception.errno, errno.EROFS) + else: + os.link(root / "b", root / "new-link") + self.assertTrue((root / "b").samefile(root / "new-link")) + (root / "a").chmod(0o600) + (root / "linked").write_text("changed") + self.assertEqual((root / "a").read_text(), "changed") + self.assertEqual((root / "b").read_text(), "identical") + self.assertEqual(blob.read_bytes(), before[0]) + self.assertEqual(blob.stat().st_mode, before[1].st_mode) + self.assertEqual(blob.stat().st_mtime_ns, before[1].st_mtime_ns) + + def test_unavailable_reflinks_fall_back_to_overlay(self) -> None: + # Automatic reconstruction reflinks one blob to test the filesystem. When that fails, it + # must reconstruct an overlay with the original contents and metadata. + self.files() + (self.tree / "b").write_text("different contents") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + + with mock.patch.object( + storage, "reflink", side_effect=ReflinkUnavailable(errno.EXDEV, "different filesystems") + ) as reflink: + with storage.root([self.store]) as restored: + root = restored.path + self.assertIsNotNone(restored.upper) + self.assertEqual(reflink.call_count, 1) + self.assertEqual((root / "a").read_text(), "identical") + self.assertEqual((root / "b").read_text(), "different contents") + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + + def test_metadata_failure_does_not_select_another_backend(self) -> None: + # An unsupported metadata operation is a reconstruction error rather than evidence that + # reflinks are unavailable. Report the metadata failure without attempting an overlay. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + + with ( + mock.patch.object(storage, "reflink", side_effect=clone_file), + mock.patch.object(storage, "metadata", side_effect=OSError(errno.EOPNOTSUPP, "metadata")), + mock.patch.object(storage, "Overlay", side_effect=AssertionError("unexpected overlay")), + self.assertRaises(OSError) as error, + ): + with storage.root([self.store]): + pass + self.assertEqual(error.exception.errno, errno.EOPNOTSUPP) + + def test_independent_hardlink_groups_can_share_one_blob(self) -> None: + # Two declared groups have equal contents but different metadata. Each group must keep + # its own inode when chmod and writes modify the other group in the temporary root. + self.files() + os.link(self.tree / "b", self.tree / "second-link") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + self.assertEqual(len(list((self.store / "blobs").iterdir())), 1) + with storage.root([self.store]) as restored: + root = restored.path + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertTrue((root / "b").samefile(root / "second-link")) + self.assertFalse((root / "a").samefile(root / "b")) + (root / "linked").chmod(0o600) + (root / "a").write_text("first group") + self.assertEqual((root / "second-link").read_text(), "identical") + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o755) + (root / "second-link").write_text("second group") + self.assertEqual((root / "b").read_text(), "second group") + self.assertEqual((root / "linked").read_text(), "first group") + def test_reuse_and_metadata_only_changes(self) -> None: # Changing only metadata must alter the complete manifest while reusing the parent's blob. self.files() - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store]) as root: + with storage.root([self.store]) as restored: + root = restored.path (root / "b").chmod(0o600) os.utime(root / "b", ns=(999, 999)) - storage.capture(root, child, [self.store], epoch=CAPTURE_EPOCH) + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) self.assertEqual(list((child / "blobs").iterdir()), []) self.assertNotEqual( (child / uapi16.MANIFEST).read_bytes(), (self.store / uapi16.MANIFEST).read_bytes() ) - with storage.root([self.store, child]) as root: + with storage.root([self.store, child], readonly=True) as restored: + root = restored.path self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o600) self.assertEqual((root / "b").stat().st_mtime_ns, 999) - def test_deletions_opacity_and_replacements(self) -> None: + def test_deletions_and_replacements(self) -> None: # Delete inherited paths, replace files and directories, and rename a fresh directory over - # an inherited directory. Capture the merged view so no native overlay markers persist. + # an inherited directory. Each child snapshot must describe the complete resulting tree. (self.tree / "dir").mkdir() (self.tree / "dir/old").write_text("old") (self.tree / "deleted").write_text("deleted") (self.tree / "file").write_text("file") - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store]) as root: + with storage.root([self.store]) as restored: + root = restored.path (root / "deleted").unlink() shutil.rmtree(root / "dir") (root / "fresh").mkdir() @@ -133,42 +345,100 @@ def test_deletions_opacity_and_replacements(self) -> None: (root / "file").unlink() (root / "file").mkdir() (root / "file/child").write_text("child") - storage.capture(root, child, [self.store], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child]) as root: + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child]) as restored: + root = restored.path self.assertFalse((root / "deleted").exists()) self.assertEqual([p.name for p in (root / "dir").iterdir()], ["new"]) self.assertEqual((root / "file/child").read_text(), "child") shutil.rmtree(root / "file") (root / "file").write_text("replacement") grandchild = self.scratch / "grandchild" - storage.capture(root, grandchild, [self.store, child], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child, grandchild]) as root: + storage.capture(restored, grandchild, [self.store, child], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child, grandchild]) as restored: + root = restored.path self.assertEqual((root / "file").read_text(), "replacement") self.assertEqual({p.name for p in child.iterdir()}, {"blobs", uapi16.MANIFEST}) self.assertNotIn(b"overlay.opaque", (child / uapi16.MANIFEST).read_bytes()) + def test_capture_native_overlay_changes(self) -> None: + # An external overlay can introduce native whiteouts and opaque directories. Capturing + # its visible filesystem must omit deleted paths and overlay markers from the snapshot. + (self.tree / "opaque").mkdir() + (self.tree / "opaque/old").write_text("old") + (self.tree / "deleted").write_text("deleted") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + lower, upper, work, merged = (self.scratch / name for name in ("lower", "upper", "work", "merged")) + storage.restore(storage.Snapshot([self.store]), lower) + upper.mkdir() + work.mkdir() + child = self.scratch / "child" + with Overlay((lower,), upper, work, merged) as overlay: + overlay.mount() + (merged / "deleted").unlink() + shutil.rmtree(merged / "opaque") + (merged / "opaque").mkdir() + (merged / "opaque/new").write_text("new") + self.assertEqual(os.getxattr(upper / "opaque", "user.overlay.opaque"), b"y") + storage.capture(storage.Root(merged, upper), child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root([self.store, child]) as restored: + root = restored.path + self.assertFalse((root / "deleted").exists()) + self.assertEqual([path.name for path in (root / "opaque").iterdir()], ["new"]) + self.assertNotIn(b"overlay.opaque", (child / uapi16.MANIFEST).read_bytes()) + self.assertEqual({path.name for path in child.iterdir()}, {"blobs", uapi16.MANIFEST}) + + def test_failed_commands_remove_the_temporary_root(self) -> None: + # A command failure must remove the temporary root while retaining the reusable snapshot. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + root: Path | None = None + with self.assertRaisesRegex(RuntimeError, "command failed"): + with storage.root([self.store]) as restored: + root = restored.path + (root / "a").write_text("changed") + raise RuntimeError("command failed") + assert root is not None + self.assertFalse(root.exists()) + with storage.root([self.store]) as restored: + root = restored.path + self.assertEqual((root / "a").read_text(), "identical") + def test_child_stores_only_new_contents(self) -> None: # A child reuses every inherited content digest and writes one blob for two new equal files. self.files() - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store]) as root: + with storage.root([self.store]) as restored: + root = restored.path (root / "new").write_text("new contents") (root / "new-independent").write_text("new contents") - storage.capture(root, child, [self.store], epoch=CAPTURE_EPOCH) + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) self.assertEqual(len(list((child / "blobs").iterdir())), 1) - with storage.root([self.store, child]) as root: + with storage.root([self.store, child], readonly=True) as restored: + root = restored.path self.assertEqual((root / "a").read_text(), "identical") self.assertFalse((root / "new").samefile(root / "new-independent")) + def test_readonly_empty_snapshot(self) -> None: + # An image without parent stores still needs an empty, read-only root with its default mode. + with storage.root([], readonly=True) as restored: + root = restored.path + self.assertEqual(list(root.iterdir()), []) + self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o755) + with self.assertRaises(OSError) as error: + (root / "new").touch() + self.assertEqual(error.exception.errno, errno.EROFS) + def test_device_nodes(self) -> None: # Preserve device numbers when the namespace permits mknod; unprivileged hosts may deny it. try: os.mknod(self.tree / "device", stat.S_IFCHR | 0o600, os.makedev(1, 3)) except PermissionError: self.skipTest("the sandbox does not permit device nodes") - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) - with storage.root([self.store]) as root: + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with storage.root([self.store]) as restored: + root = restored.path attributes = (root / "device").lstat() self.assertTrue(stat.S_ISCHR(attributes.st_mode)) self.assertEqual(attributes.st_rdev, os.makedev(1, 3)) @@ -178,11 +448,13 @@ def test_special_files(self) -> None: os.mkfifo(self.tree / "fifo", 0o600) with socket.socket(socket.AF_UNIX) as sock: sock.bind(str(self.tree / "sock")) - storage.capture(self.tree, self.store, epoch=CAPTURE_EPOCH) - with storage.root([self.store]) as root: - self.assertTrue(stat.S_ISFIFO((root / "fifo").lstat().st_mode)) - self.assertTrue(stat.S_ISSOCK((root / "sock").lstat().st_mode)) - self.assertEqual(stat.S_IMODE((root / "fifo").stat().st_mode), 0o600) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + for readonly in (False, True): + with self.subTest(readonly=readonly), storage.root([self.store], readonly=readonly) as restored: + root = restored.path + self.assertTrue(stat.S_ISFIFO((root / "fifo").lstat().st_mode)) + self.assertTrue(stat.S_ISSOCK((root / "sock").lstat().st_mode)) + self.assertEqual(stat.S_IMODE((root / "fifo").stat().st_mode), 0o600) if __name__ == "__main__": diff --git a/util.py b/util.py index 0552da3a..1a8276cb 100644 --- a/util.py +++ b/util.py @@ -5,6 +5,7 @@ import bz2 import errno +import fcntl import gzip import http.client import io @@ -32,6 +33,10 @@ _FETCH_ATTEMPTS = 4 +class ReflinkUnavailable(OSError): + pass + + def fail(message: str) -> NoReturn: sys.exit(f"tine: {message}") @@ -208,6 +213,18 @@ def clone_file(src: Path, dst: Path, allow_link: bool = False) -> None: shutil.copymode(src, dst) +def reflink(src: Path, dst: Path) -> None: + """Raise ReflinkUnavailable instead of copying bytes when the filesystem cannot clone src.""" + with open(src, "rb") as source, open(dst, "wb") as destination: + try: + fcntl.ioctl(destination.fileno(), fcntl.FICLONE, source.fileno()) + except OSError as error: + if error.errno in _COPY_FALLBACK_ERRNOS | {errno.EACCES, errno.EBADF, errno.ENOTTY, errno.EPERM}: + raise ReflinkUnavailable(error.errno, error.strerror) from error + raise + shutil.copymode(src, dst) + + def named_files(source: Path, name: str) -> list[Path]: """Find files called `name` under `source`, and return their paths relative to `source`.""" found: list[Path] = [] diff --git a/util_test.py b/util_test.py index 188e4f8c..b6396346 100644 --- a/util_test.py +++ b/util_test.py @@ -6,10 +6,12 @@ buck test tine//:test """ +import errno import stat import tempfile import unittest from pathlib import Path +from unittest import mock import util @@ -82,3 +84,38 @@ def test_finds_nested_files_without_following_links(self) -> None: (source / "dangling").symlink_to("missing") self.assertEqual(util.named_files(source, "wanted"), [Path("nested/wanted"), Path("wanted")]) + + +class TestReflink(unittest.TestCase): + def test_reflink_mode_errors_are_not_unavailability(self) -> None: + # A successful reflink followed by an unsupported mode change must report the metadata + # error. A caller must not mistake that failure for missing reflink support. + root = scratch(self) + source, destination = root / "source", root / "destination" + source.write_bytes(b"cached contents") + with ( + mock.patch.object(util.fcntl, "ioctl", return_value=0), + mock.patch.object(util.shutil, "copymode", side_effect=OSError(errno.EOPNOTSUPP, "mode")), + self.assertRaises(OSError) as error, + ): + util.reflink(source, destination) + self.assertNotIsInstance(error.exception, util.ReflinkUnavailable) + self.assertEqual(error.exception.errno, errno.EOPNOTSUPP) + + def test_reflink_never_falls_back_to_a_byte_copy(self) -> None: + # A caller choosing a storage strategy must distinguish an unsupported reflink from + # a successful clone. Reject unsupported or denied clones without copying any bytes. + root = scratch(self) + source, destination = root / "source", root / "destination" + source.write_bytes(b"cached contents") + for number in (errno.EXDEV, errno.EOPNOTSUPP, errno.EPERM): + with ( + self.subTest(errno=number), + mock.patch.object(util.fcntl, "ioctl", side_effect=OSError(number, "unavailable")), + mock.patch.object(util, "copy_range") as copy, + self.assertRaises(OSError) as error, + ): + util.reflink(source, destination) + self.assertEqual(error.exception.errno, number) + copy.assert_not_called() + self.assertEqual(source.read_bytes(), b"cached contents") From b9461b06ef9700c0ef6b6595207f4f2ecfe04b80 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 12 Nov 2026 10:15:33 +0100 Subject: [PATCH 6/7] box: Build box layers from snapshots Box layers lose modes, hardlinks and xattrs when Buck stores them, so a child box installs over a parent that differs from what the package manager wrote. Let's capture every box layer as a snapshot and install a child box over its restored parent snapshot. The sandbox keeps mounting one plain directory per layer, which we export from the snapshot with only what Buck stores, so action startup doesn't change. Signed-off-by: Daan De Meyer --- box/build.bzl | 27 ++++++---- docs/design/architecture.md | 2 +- image/archive_test.py | 2 +- image/finalize.py | 8 +-- image/layer.py | 2 +- image/layer_test.py | 6 +-- package/BUCK | 2 +- package/installer.py | 33 +++++++++++- package/installer_test.py | 32 ++++++++--- package/transaction.py | 2 +- rootfs/rootfs.py | 8 +-- uapi16/BUCK | 2 +- uapi16/storage.py | 58 ++++++++++++-------- uapi16/storage_test.py | 105 +++++++++++++++++++++++++++--------- 14 files changed, 208 insertions(+), 81 deletions(-) diff --git a/box/build.bzl b/box/build.bzl index fc9cea09..b990f87b 100644 --- a/box/build.bzl +++ b/box/build.bzl @@ -56,6 +56,8 @@ _BoxRootInfo = provider( # resolves its own layers. "resolver": provider_field(Dependency | None), "selected": provider_field(list[Dependency]), + # The snapshot stores of the box's layers, lowest first. + "snapshots": provider_field(list[Artifact]), "transaction": provider_field(Artifact), }, ) @@ -132,10 +134,11 @@ def _install( arch: str, lower: list[Artifact], installed: Artifact | None, -) -> Artifact: - """Install `transaction` with `box`, which also verifies its packages, into a layer over `lower`. +) -> (Artifact, Artifact): + """Install `transaction` with `box`, which also verifies its packages, into a layer over the snapshots `lower`. - The layer leaves out the packages of `installed`. + The layer leaves out the packages of `installed`. Return the directory that the sandbox mounts and the + snapshot store of the layer. """ system = package_system[PackageSystemInfo] @@ -152,7 +155,7 @@ def _install( # Use the predecessor or bootstrapped root to install the box, or its layer over the parent. stage2 = _layer_dir(ctx.actions, "stage2") - work = _layer_dir(ctx.actions, "stage2.work") if lower else None + snapshot = _layer_dir(ctx.actions, "stage2.snapshot") ctx.actions.run( cmd_args( box_run( @@ -170,14 +173,15 @@ def _install( "langs": [], "lower": lower, "packages_dir": packages, + "snapshot": snapshot.as_output(), "target": stage2.as_output(), - "work": work.as_output() if work != None else None, + "work": None, }, ), ), category = "box", ) - return stage2 + return stage2, snapshot def _box_impl(ctx: AnalysisContext) -> list[Provider]: # A root box takes the repositories of its release rather than the release, because the release @@ -240,22 +244,24 @@ def _box_impl(ctx: AnalysisContext) -> list[Provider]: # See "Box layers" in docs/design/architecture.md. repository_labels = _repository_labels(repositories) lower = [] + snapshots = [] installed = None if parent != None and ctx.attrs.layered and _repository_labels(parent[_BoxRootInfo].selected) == repository_labels: lower = parent[BoxInfo].layers + snapshots = parent[_BoxRootInfo].snapshots installed = parent[_BoxRootInfo].transaction # With the resolver box as a dependency, the keyrings are anonymous targets. Every box that resolves # through the same resolver box shares them. A root box has no resolver box and verifies with its stage1 # root, which is no target. - stage2 = _install( + stage2, snapshot = _install( ctx, package_system, resolver_dep if resolver_dep != None else installer_box, repositories, transaction, ctx.attrs._arch, - lower, + snapshots, installed, ) info = BoxInfo( @@ -266,6 +272,7 @@ def _box_impl(ctx: AnalysisContext) -> list[Provider]: arch = ctx.attrs._arch, resolver = resolver_dep, selected = repositories, + snapshots = snapshots + [snapshot], transaction = transaction, ) sub_targets = { @@ -331,14 +338,14 @@ def _box_layer_impl(ctx: AnalysisContext) -> list[Provider]: resolver = root.resolver if root.resolver != None else parent names = _effective_packages(ctx.attrs.packages, parent, package_system) transaction = _resolve(ctx, package_system, resolver, _box_repositories(ctx, root.selected, package_system, None), names, root.arch) - layer = _install( + layer, _ = _install( ctx, package_system, resolver, root.selected, transaction, root.arch, - parent[BoxInfo].layers, + root.snapshots, root.transaction, ) return [DefaultInfo(default_output = layer)] diff --git a/docs/design/architecture.md b/docs/design/architecture.md index b90aa0f1..c0114afc 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -544,7 +544,7 @@ The host contract is intentionally small; its short list of requirements is docu ### Box layers -If a box and its `parent` come from the same repositories, the box is a delta layer over its parent's +If a box and its `parent` come from the same repositories, the box is a layer over its parent's layers, like with images. It enforces addition only, and fails if its lock would update or remove parent packages. One likely cause is an outdated committed lock, which needs a catalog refresh. The other is that resolving both package lists together picks other packages than the parent got, for example diff --git a/image/archive_test.py b/image/archive_test.py index 43fbd5dd..879849bc 100644 --- a/image/archive_test.py +++ b/image/archive_test.py @@ -58,7 +58,7 @@ def test_archives_keep_hardlinks_created_by_commands(self) -> None: # A command creates a hardlink to an inherited independent file. Upper-aware capture # must preserve the group in tar and cpio without linking the equal-content file a. child = self.scratch / "child" - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: tree = restored.path os.link(tree / "b", tree / "new-link") storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) diff --git a/image/finalize.py b/image/finalize.py index 2dd04f8c..168104c1 100644 --- a/image/finalize.py +++ b/image/finalize.py @@ -66,14 +66,16 @@ def _root( readonly: bool = True, strategy: storage.Strategy = "auto", ) -> Generator[storage.Root]: + snapshot = storage.Snapshot(spec["lower"]) + if base: + storage.delta(snapshot, base) + normalize_snapshot(snapshot) with ( storage.root( - spec["lower"], - base=base, + snapshot, # Authored tmpfiles rules write into the root. They fail on a read-only root. readonly=readonly and not spec["tmpfiles"], strategy=strategy, - prepare=normalize_snapshot, ) as restored, rootfs.rootfs("/buildroot", bind=restored.path, binds=binds) as tree, ): diff --git a/image/layer.py b/image/layer.py index dfac79b6..4ce192dd 100755 --- a/image/layer.py +++ b/image/layer.py @@ -293,7 +293,7 @@ def main(argv: list[str] | None = None) -> None: lower = spec["lower"] with ( - storage.root(lower) as restored, + storage.root(storage.Snapshot(lower)) as restored, tempfile.TemporaryDirectory(prefix="layer.", dir="/var/tmp") as scratch, ): tree = restored.path diff --git a/image/layer_test.py b/image/layer_test.py index 049b5c14..1aac5a29 100644 --- a/image/layer_test.py +++ b/image/layer_test.py @@ -111,7 +111,7 @@ def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: ) command.assert_called_once() - with storage.root([stored, self.out]) as restored: + with storage.root(storage.Snapshot([stored, self.out])) as restored: tree = restored.path self.assertEqual((tree / "copied").read_text(encoding="utf-8"), "copied") self.assertFalse((tree / "cache.pyc").exists()) @@ -141,7 +141,7 @@ def command(*args: object) -> None: with mock.patch.object(layer, "_run", side_effect=command): self._build([["run", ["command"], {}, False]], copied) - with storage.root([copied, self.out]) as restored: + with storage.root(storage.Snapshot([copied, self.out])) as restored: tree = restored.path self.assertEqual(stat.S_IMODE((tree / "file").stat().st_mode), 0o600) self.assertEqual((tree / "linked").stat().st_mtime_ns, 456) @@ -183,7 +183,7 @@ def command(*args: object) -> None: with mock.patch.object(layer, "_run", side_effect=command): self._build([["run", ["command"], {}, False]], parent) - with storage.root([parent, self.out]) as restored: + with storage.root(storage.Snapshot([parent, self.out])) as restored: tree = restored.path self.assertTrue((tree / "file").samefile(tree / "linked")) self.assertEqual((tree / "linked").read_text(), "changed") diff --git a/package/BUCK b/package/BUCK index 4825c08d..b3b268cf 100644 --- a/package/BUCK +++ b/package/BUCK @@ -29,7 +29,7 @@ python.library( python.library( name = "installer", srcs = ["installer.py"], - deps = ["//:specs", "//:util", "//rootfs:rootfs"], + deps = ["//:specs", "//:util", "//rootfs:rootfs", "//uapi16:storage"], visibility = ["PUBLIC"], ) diff --git a/package/installer.py b/package/installer.py index 7e02df5a..9bb9f859 100644 --- a/package/installer.py +++ b/package/installer.py @@ -15,12 +15,13 @@ from collections.abc import Callable, Generator from contextlib import contextmanager from pathlib import Path -from typing import TypedDict +from typing import NotRequired, TypedDict import specs from util import fail import rootfs +import storage # A fixed install path avoids embedding Buck hashes and supports scriptlet chroots. BUILDROOT = "/buildroot" @@ -66,6 +67,10 @@ class InstallSpec(TypedDict): installroot: str | None lower: list[str] work: str | None + # Only a box install sets `snapshot`. The install then restores the snapshot stores in `lower`, + # captures its own snapshot store at `snapshot` and writes the directory that the sandbox mounts to + # `target`. + snapshot: NotRequired[str | None] box_config: bool langs: list[str] docs: bool @@ -175,6 +180,10 @@ def run( if spec["target"] is None: fail(f"{prog}: one of target and installroot is required") + if (snapshot := spec.get("snapshot")) is not None: + _install_box(prog, install, packages_dir, spec, Path(snapshot), Path(spec["target"])) + return + # Bind sources must exist. target = Path(spec["target"]) target.mkdir(parents=True, exist_ok=True) @@ -208,3 +217,25 @@ def run( if layered and spec["box_config"] and (deleted := rootfs.deletions(target)): markers = ", ".join(map(str, deleted)) fail(f"{prog}: a box layer cannot delete from the layers below it, but has {markers}") + + +def _install_box( + prog: str, + install: Callable[[Path, Path, InstallSpec, bool], None], + packages_dir: Path, + spec: InstallSpec, + out: Path, + target: Path, +) -> None: + lowers = spec["lower"] + with storage.root(storage.Snapshot(lowers)) as restored: + with rootfs.rootfs(BUILDROOT, bind=restored.path, apivfs=True, strict_unmount=True): + install(packages_dir, Path(BUILDROOT), spec, bool(lowers)) + _normalize(Path(BUILDROOT), spec) + storage.capture(restored, out, lowers, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) + snapshot = storage.Snapshot([*lowers, out]) + # The sandbox stacks the directories of a box's layers and decodes no whiteouts. A file that a + # layer deletes would still show up from the layer below. + if deleted := storage.delta(snapshot, len(lowers)): + fail(f"{prog}: a box layer cannot delete from the layers below it, but deletes {', '.join(deleted)}") + storage.write_buck_tree(snapshot, target) diff --git a/package/installer_test.py b/package/installer_test.py index 0d9556f6..3b13474d 100644 --- a/package/installer_test.py +++ b/package/installer_test.py @@ -12,6 +12,7 @@ from unittest import mock import installer +import storage class TestPathRules(unittest.TestCase): @@ -40,6 +41,20 @@ def test_keeps_authselect_nsswitch_link(self) -> None: self.assertEqual(nsswitch.readlink(), Path("/etc/authselect/nsswitch.conf")) +def _snapshot(root: Path, name: str, files: dict[str, str]) -> Path: + """Capture a tree with `files` as the snapshot store `name` under `root`, with the epoch of the tests.""" + tree = root / f"{name}.tree" + # A box root contains the mount points of the API filesystems. Without them, the install would create + # them and a box layer would export them. + for path in ("etc", "dev", "proc", "run", "tmp", "var/tmp"): + (tree / path).mkdir(parents=True) + for path, content in files.items(): + (tree / path).write_text(content) + store = root / name + storage.capture(storage.Root(tree), store, epoch=0) + return store + + class TestRun(unittest.TestCase): def test_package_scripts_tolerate_unmapped_acl_groups(self) -> None: # The test box mounts /var/tmp as tmpfs, so a host mounted with noacl cannot hide this failure. @@ -87,16 +102,17 @@ def install( installer.run("install", install, []) def test_a_box_layer_may_add_but_not_delete(self) -> None: + # The sandbox mounts the directories of a box's layers without decoding deletions. Install a box + # layer over a lower snapshot that adds a file, and once more so that it also deletes a file of the + # lower snapshot. The first install must write only the added file, the second must fail. for deletes in (False, True): with ( self.subTest(deletes=deletes), tempfile.TemporaryDirectory(dir="/var/tmp") as scratch, - mock.patch.dict(os.environ), + mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "0"}), ): root = Path(scratch) - lower = root / "lower" - (lower / "etc").mkdir(parents=True) - (lower / "doomed").write_text("") + lower = _snapshot(root, "lower", {"doomed": ""}) (root / "packages").mkdir() (root / "packages" / "systemd.rpm").write_text("") spec = installer.InstallSpec( @@ -105,8 +121,9 @@ def test_a_box_layer_may_add_but_not_delete(self) -> None: target=str(root / "target"), installroot=None, lower=[str(lower)], - work=str(root / "work"), - box_config=True, + work=None, + snapshot=str(root / "snapshot"), + box_config=False, langs=[], docs=True, ) @@ -124,10 +141,11 @@ def install( with mock.patch.object(installer.specs, "parse", return_value=spec): if deletes: - with self.assertRaisesRegex(SystemExit, r"cannot delete .*\.wh\.doomed"): + with self.assertRaisesRegex(SystemExit, "cannot delete .*doomed"): installer.run("install", install, []) else: installer.run("install", install, []) + self.assertEqual([path.name for path in (root / "target").iterdir()], ["added"]) def test_an_empty_closure_leaves_the_layer_empty(self) -> None: # A layered install with no packages to install must not run the package manager. Buck still diff --git a/package/transaction.py b/package/transaction.py index 913230cd..f6e88564 100644 --- a/package/transaction.py +++ b/package/transaction.py @@ -182,7 +182,7 @@ def installed_root(lowers: Sequence[str]) -> Generator[Path]: # A snapshot store contains `uapi16.MANIFEST` at its top level. A box layer is a plain directory # tree, and no package installs a file of that name into the root. if (Path(lowers[-1]) / uapi16.MANIFEST).exists(): - with storage.root(lowers) as restored: + with storage.root(storage.Snapshot(lowers)) as restored: yield restored.path else: with rootfs.rootfs("/installroot", lowers=lowers) as tree: diff --git a/rootfs/rootfs.py b/rootfs/rootfs.py index 89356cfc..412e3dae 100644 --- a/rootfs/rootfs.py +++ b/rootfs/rootfs.py @@ -44,7 +44,7 @@ # OCI delta to native overlay translation. -def _escape(name: str) -> str: +def escape(name: str) -> str: """Escape a path component Buck cannot store, preserving injectivity.""" if "\\" not in name and not name.startswith(_ESC): return name @@ -81,7 +81,7 @@ def deletions(delta: Path) -> list[Path]: def _whiteout_marker(name: str) -> str: """Encode a possibly unstorable whiteout target.""" - return _escape(_WH + name) + return escape(_WH + name) def _whiteout(directory: Path, name: str) -> None: @@ -159,7 +159,7 @@ def _in_lower(lower: Path, rel: Path) -> bool: for part in rel.parts: if path.is_symlink() or not path.is_dir(): return False - path = path / _escape(part) + path = path / escape(part) if not os.path.lexists(path): return False return True @@ -192,7 +192,7 @@ def capture(tree: Path, lowers: Sequence[Path] | None = None) -> None: rel = p.relative_to(tree) if lowers is None or any(_in_lower(lower, rel) for lower in lowers): (p / _OPAQUE).write_bytes(b"") - escaped = _escape(p.name) + escaped = escape(p.name) if escaped != p.name: p.rename(p.parent / escaped) diff --git a/uapi16/BUCK b/uapi16/BUCK index 936460d2..ef9b1d60 100644 --- a/uapi16/BUCK +++ b/uapi16/BUCK @@ -15,7 +15,7 @@ python.library( python.library( name = "storage", srcs = ["storage.py"], - deps = [":uapi16", "//:util", "//box:isolation"], + deps = [":uapi16", "//:util", "//box:isolation", "//rootfs:rootfs"], visibility = ["tine//..."], ) diff --git a/uapi16/storage.py b/uapi16/storage.py index c9fc82ab..d227f340 100644 --- a/uapi16/storage.py +++ b/uapi16/storage.py @@ -7,7 +7,7 @@ import re import stat import tempfile -from collections.abc import Callable, Generator, Sequence +from collections.abc import Generator, Sequence from contextlib import ExitStack, contextmanager from dataclasses import dataclass, replace from pathlib import Path, PurePosixPath @@ -15,6 +15,7 @@ from util import ReflinkUnavailable, clone_file, fail, reflink +import rootfs import uapi16 from isolation import Bind, Overlay @@ -268,27 +269,42 @@ def _reflinks(snapshot: Snapshot, scratch: Path) -> bool: return True +def delta(snapshot: Snapshot, base: int) -> list[str]: + """Keep the objects that differ from the first `base` stores, and their parent directories. + + Return the names that the first `base` stores contain and the snapshot does not. + """ + inherited = _identities(Snapshot(snapshot.stores[:base]).objects[1:]) + current = _identities(snapshot.objects[1:]) + changed = {name for name, obj in current.items() if inherited.get(name) != obj} + needed = {str(parent) for name in changed for parent in Path(name).parents} + snapshot.objects = [snapshot.objects[0]] + [ + obj for obj in snapshot.objects[1:] if obj["name"] in changed or obj["name"] in needed + ] + return sorted(inherited.keys() - current.keys()) + + +def write_buck_tree(snapshot: Snapshot, out: Path) -> None: + """Write the snapshot as a directory that contains only what Buck stores.""" + out.mkdir(parents=True, exist_ok=True) + out.chmod(0o755) + for obj in snapshot.objects[1:]: + path = out.joinpath(*map(rootfs.escape, obj["name"].split("/"))) + kind = obj["type"] + if kind == "dir": + path.mkdir() + path.chmod(0o755) + elif kind == "reg": + clone_file(snapshot.content(obj), path) + path.chmod(0o755 if obj["mode"] & 0o111 else 0o644) + elif kind == "lnk": + path.symlink_to(uapi16.symlink_target(obj)) + else: + fail(f"storage: Buck cannot store the {kind} inode {obj['name']!r}") + + @contextmanager -def root( - stores: Sequence[str | Path], - *, - base: int = 0, - readonly: bool = False, - strategy: Strategy = "auto", - prepare: Callable[[Snapshot], None] | None = None, -) -> Generator[Root]: - snapshot = Snapshot(stores) - if base: - inherited = _identities(Snapshot(stores[:base]).objects[1:]) - changed = { - name for name, obj in _identities(snapshot.objects[1:]).items() if inherited.get(name) != obj - } - needed = {str(parent) for name in changed for parent in Path(name).parents} - snapshot.objects = [snapshot.objects[0]] + [ - obj for obj in snapshot.objects[1:] if obj["name"] in changed or obj["name"] in needed - ] - if prepare is not None: - prepare(snapshot) +def root(snapshot: Snapshot, *, readonly: bool = False, strategy: Strategy = "auto") -> Generator[Root]: # Overlay refuses a read-only mount with one lower layer and no data layer. An empty snapshot # has no blob directories to use as data layers. if not snapshot.stores: diff --git a/uapi16/storage_test.py b/uapi16/storage_test.py index 0beb1f15..88215599 100644 --- a/uapi16/storage_test.py +++ b/uapi16/storage_test.py @@ -57,8 +57,8 @@ def test_roundtrip_after_artifact_metadata_is_lost(self) -> None: materialized = self.scratch / "restored" storage.restore(storage.Snapshot([copied]), materialized) with ( - storage.root([copied], readonly=True) as readonly, - storage.root([copied]) as restored, + storage.root(storage.Snapshot([copied]), readonly=True) as readonly, + storage.root(storage.Snapshot([copied])) as restored, ): for tree in (materialized, readonly.path, restored.path): with self.subTest(tree=tree): @@ -95,7 +95,7 @@ def test_writes_and_metadata_leave_blobs_unchanged(self) -> None: storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) blob = next((self.store / "blobs").iterdir()) before = blob.read_bytes(), blob.stat() - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path self.assertTrue((root / "a").samefile(root / "linked")) self.assertFalse((root / "a").samefile(root / "b")) @@ -131,7 +131,7 @@ def test_readonly_reconstruction_never_copies_blob_data(self) -> None: os.utime(self.tree / "nested", ns=(999, 999)) storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) with mock.patch.object(storage, "clone_file", wraps=storage.clone_file) as copy: - with storage.root([self.store], readonly=True, strategy="overlay") as restored: + with storage.root(storage.Snapshot([self.store]), readonly=True, strategy="overlay") as restored: root = restored.path self.assertEqual(copy.call_count, 0) self.assertTrue((root / "a").samefile(root / "nested/linked")) @@ -154,7 +154,7 @@ def test_commands_can_create_new_hardlinks(self) -> None: self.files() storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store], strategy="overlay") as restored: + with storage.root(storage.Snapshot([self.store]), strategy="overlay") as restored: root = restored.path os.link(root / "b", root / "new-link") (root / "b").write_text("new group") @@ -165,7 +165,7 @@ def test_commands_can_create_new_hardlinks(self) -> None: entries = {entry.name: entry for entry in restored.entries()} self.assertEqual(entries["b"].inode, entries["new-link"].inode) storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child]) as restored: + with storage.root(storage.Snapshot([self.store, child])) as restored: root = restored.path self.assertTrue((root / "b").samefile(root / "new-link")) self.assertEqual((root / "new-link").read_text(), "new group") @@ -182,7 +182,9 @@ def test_extension_compares_hardlink_membership(self) -> None: os.link(self.tree / "a", self.tree / "a-linked") child = self.scratch / "child" storage.capture(storage.Root(self.tree), child, [self.store], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child], base=1, strategy="copy") as restored: + snapshot = storage.Snapshot([self.store, child]) + storage.delta(snapshot, 1) + with storage.root(snapshot, strategy="copy") as restored: self.assertEqual({p.name for p in restored.path.iterdir()}, {"a", "a-linked"}) self.assertTrue((restored.path / "a").samefile(restored.path / "a-linked")) @@ -192,7 +194,9 @@ def test_extension_compares_hardlink_membership(self) -> None: os.link(self.tree / "z", self.tree / "new-alias") alias = self.scratch / "alias" storage.capture(storage.Root(self.tree), alias, [self.store], epoch=CAPTURE_EPOCH) - with storage.root([self.store, alias], base=1, strategy="copy") as restored: + snapshot = storage.Snapshot([self.store, alias]) + storage.delta(snapshot, 1) + with storage.root(snapshot, strategy="copy") as restored: self.assertEqual({p.name for p in restored.path.iterdir()}, {"z", "z-linked", "new-alias"}) self.assertTrue((restored.path / "z").samefile(restored.path / "new-alias")) self.assertTrue((restored.path / "z-linked").samefile(restored.path / "new-alias")) @@ -203,7 +207,7 @@ def test_writable_reconstruction_copies_only_declared_groups(self) -> None: self.files() storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) with mock.patch.object(storage, "clone_file", wraps=storage.clone_file) as copy: - with storage.root([self.store], strategy="overlay") as restored: + with storage.root(storage.Snapshot([self.store]), strategy="overlay") as restored: root = restored.path assert restored.upper is not None self.assertEqual(copy.call_count, 1) @@ -228,7 +232,7 @@ def test_available_reflinks_use_private_inodes_without_overlay(self) -> None: self.subTest(readonly=readonly), mock.patch.object(storage, "reflink", side_effect=clone_file), mock.patch.object(storage, "Overlay", side_effect=AssertionError("unexpected overlay")), - storage.root([self.store], readonly=readonly) as restored, + storage.root(storage.Snapshot([self.store]), readonly=readonly) as restored, ): root = restored.path self.assertIsNone(restored.upper) @@ -262,7 +266,7 @@ def test_unavailable_reflinks_fall_back_to_overlay(self) -> None: with mock.patch.object( storage, "reflink", side_effect=ReflinkUnavailable(errno.EXDEV, "different filesystems") ) as reflink: - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path self.assertIsNotNone(restored.upper) self.assertEqual(reflink.call_count, 1) @@ -283,7 +287,7 @@ def test_metadata_failure_does_not_select_another_backend(self) -> None: mock.patch.object(storage, "Overlay", side_effect=AssertionError("unexpected overlay")), self.assertRaises(OSError) as error, ): - with storage.root([self.store]): + with storage.root(storage.Snapshot([self.store])): pass self.assertEqual(error.exception.errno, errno.EOPNOTSUPP) @@ -294,7 +298,7 @@ def test_independent_hardlink_groups_can_share_one_blob(self) -> None: os.link(self.tree / "b", self.tree / "second-link") storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) self.assertEqual(len(list((self.store / "blobs").iterdir())), 1) - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path self.assertTrue((root / "a").samefile(root / "linked")) self.assertTrue((root / "b").samefile(root / "second-link")) @@ -312,7 +316,7 @@ def test_reuse_and_metadata_only_changes(self) -> None: self.files() storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path (root / "b").chmod(0o600) os.utime(root / "b", ns=(999, 999)) @@ -321,7 +325,7 @@ def test_reuse_and_metadata_only_changes(self) -> None: self.assertNotEqual( (child / uapi16.MANIFEST).read_bytes(), (self.store / uapi16.MANIFEST).read_bytes() ) - with storage.root([self.store, child], readonly=True) as restored: + with storage.root(storage.Snapshot([self.store, child]), readonly=True) as restored: root = restored.path self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o600) self.assertEqual((root / "b").stat().st_mtime_ns, 999) @@ -335,7 +339,7 @@ def test_deletions_and_replacements(self) -> None: (self.tree / "file").write_text("file") storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path (root / "deleted").unlink() shutil.rmtree(root / "dir") @@ -346,7 +350,7 @@ def test_deletions_and_replacements(self) -> None: (root / "file").mkdir() (root / "file/child").write_text("child") storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child]) as restored: + with storage.root(storage.Snapshot([self.store, child])) as restored: root = restored.path self.assertFalse((root / "deleted").exists()) self.assertEqual([p.name for p in (root / "dir").iterdir()], ["new"]) @@ -355,7 +359,7 @@ def test_deletions_and_replacements(self) -> None: (root / "file").write_text("replacement") grandchild = self.scratch / "grandchild" storage.capture(restored, grandchild, [self.store, child], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child, grandchild]) as restored: + with storage.root(storage.Snapshot([self.store, child, grandchild])) as restored: root = restored.path self.assertEqual((root / "file").read_text(), "replacement") self.assertEqual({p.name for p in child.iterdir()}, {"blobs", uapi16.MANIFEST}) @@ -381,7 +385,7 @@ def test_capture_native_overlay_changes(self) -> None: (merged / "opaque/new").write_text("new") self.assertEqual(os.getxattr(upper / "opaque", "user.overlay.opaque"), b"y") storage.capture(storage.Root(merged, upper), child, [self.store], epoch=CAPTURE_EPOCH) - with storage.root([self.store, child]) as restored: + with storage.root(storage.Snapshot([self.store, child])) as restored: root = restored.path self.assertFalse((root / "deleted").exists()) self.assertEqual([path.name for path in (root / "opaque").iterdir()], ["new"]) @@ -394,13 +398,13 @@ def test_failed_commands_remove_the_temporary_root(self) -> None: storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) root: Path | None = None with self.assertRaisesRegex(RuntimeError, "command failed"): - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path (root / "a").write_text("changed") raise RuntimeError("command failed") assert root is not None self.assertFalse(root.exists()) - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path self.assertEqual((root / "a").read_text(), "identical") @@ -409,20 +413,20 @@ def test_child_stores_only_new_contents(self) -> None: self.files() storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) child = self.scratch / "child" - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path (root / "new").write_text("new contents") (root / "new-independent").write_text("new contents") storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) self.assertEqual(len(list((child / "blobs").iterdir())), 1) - with storage.root([self.store, child], readonly=True) as restored: + with storage.root(storage.Snapshot([self.store, child]), readonly=True) as restored: root = restored.path self.assertEqual((root / "a").read_text(), "identical") self.assertFalse((root / "new").samefile(root / "new-independent")) def test_readonly_empty_snapshot(self) -> None: # An image without parent stores still needs an empty, read-only root with its default mode. - with storage.root([], readonly=True) as restored: + with storage.root(storage.Snapshot([]), readonly=True) as restored: root = restored.path self.assertEqual(list(root.iterdir()), []) self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o755) @@ -437,7 +441,7 @@ def test_device_nodes(self) -> None: except PermissionError: self.skipTest("the sandbox does not permit device nodes") storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) - with storage.root([self.store]) as restored: + with storage.root(storage.Snapshot([self.store])) as restored: root = restored.path attributes = (root / "device").lstat() self.assertTrue(stat.S_ISCHR(attributes.st_mode)) @@ -450,12 +454,61 @@ def test_special_files(self) -> None: sock.bind(str(self.tree / "sock")) storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) for readonly in (False, True): - with self.subTest(readonly=readonly), storage.root([self.store], readonly=readonly) as restored: + with ( + self.subTest(readonly=readonly), + storage.root(storage.Snapshot([self.store]), readonly=readonly) as restored, + ): root = restored.path self.assertTrue(stat.S_ISFIFO((root / "fifo").lstat().st_mode)) self.assertTrue(stat.S_ISSOCK((root / "sock").lstat().st_mode)) self.assertEqual(stat.S_IMODE((root / "fifo").stat().st_mode), 0o600) + def test_buck_tree_contains_only_what_buck_stores(self) -> None: + # A box layer directory must contain only regular files, directories and symlinks with modes + # that Buck stores. Capture a tree with a hardlink, an extended attribute, a private directory + # and a name containing a backslash. The Buck tree must contain separate files with normalized + # modes, no extended attributes and the escaped name. + self.files() + os.setxattr(self.tree / "a", "user.example", b"value") + (self.tree / "private").mkdir(mode=0o700) + (self.tree / "back\\slash").write_text("escaped") + (self.tree / "link").symlink_to("a") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + out = self.scratch / "out" + storage.write_buck_tree(storage.Snapshot([self.store]), out) + self.assertEqual(stat.S_IMODE((out / "a").stat().st_mode), 0o644) + self.assertEqual(stat.S_IMODE((out / "b").stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((out / "private").stat().st_mode), 0o755) + self.assertFalse((out / "a").samefile(out / "linked")) + self.assertEqual((out / "linked").read_text(), "identical") + self.assertEqual(os.listxattr(out / "a"), []) + self.assertEqual((out / ".esc.back%5Cslash").read_text(), "escaped") + self.assertEqual(os.readlink(out / "link"), "a") + + def test_buck_tree_rejects_special_files(self) -> None: + # Buck cannot store a FIFO. Writing the Buck tree of a snapshot that contains one must fail. + os.mkfifo(self.tree / "fifo") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with self.assertRaises(SystemExit): + storage.write_buck_tree(storage.Snapshot([self.store]), self.scratch / "out") + + def test_delta_keeps_changes_and_returns_deletions(self) -> None: + # Capture a parent with three files, then change one, add one and delete one in a child. + # delta() must keep the changed and added files with their directory and return the deleted + # name. + (self.tree / "dir").mkdir() + for name in ("dir/kept", "dir/changed", "gone"): + (self.tree / name).write_text(name) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + (self.tree / "dir/changed").write_text("different") + (self.tree / "dir/new").write_text("new") + (self.tree / "gone").unlink() + child = self.scratch / "child" + storage.capture(storage.Root(self.tree), child, [self.store], epoch=CAPTURE_EPOCH) + snapshot = storage.Snapshot([self.store, child]) + self.assertEqual(storage.delta(snapshot, 1), ["gone"]) + self.assertEqual({obj["name"] for obj in snapshot.objects[1:]}, {"dir", "dir/changed", "dir/new"}) + if __name__ == "__main__": unittest.main() From 6f9dc15b5f7ebd6311f7cc9a5a94486c4be7d194 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 13 Nov 2026 14:38:38 +0100 Subject: [PATCH 7/7] package: Build buildroots from snapshots Buildroots and their BuildRequires deltas are still overlay uppers with whiteout and opaque markers encoded as files, while boxes and images use snapshots. Let's capture every install as a snapshot and restore it for plans and builds. rootfs then only needs bind mounts, and we can drop its marker translation. Every lower root of an install is now a restored snapshot, so the package installers no longer need to know whether they install over lower layers. Signed-off-by: Daan De Meyer --- box/build.bzl | 3 +- box/isolation.py | 11 +- box/isolation_test.py | 38 ++--- box/sandbox_test.py | 2 +- docs/design/architecture.md | 67 +++++---- image/layer.py | 4 +- image/layer_test.py | 2 +- package/BUCK | 4 +- package/build.bzl | 2 +- package/builder.py | 22 ++- package/buildroot.bzl | 2 +- package/install.bzl | 11 +- package/installer.py | 106 ++++--------- package/installer_test.py | 85 +++-------- package/transaction.py | 17 --- package_system/deb/BUCK | 2 + package_system/deb/install.py | 11 +- package_system/deb/plan.py | 3 +- package_system/pacman/BUCK | 3 +- package_system/pacman/install.py | 7 +- package_system/pacman/plan.py | 4 +- package_system/rpm/BUCK | 4 +- package_system/rpm/build_test.py | 9 ++ package_system/rpm/buildrequires.py | 2 +- package_system/rpm/install.py | 7 +- package_system/rpm/plan.py | 4 +- rootfs/rootfs.py | 223 ++-------------------------- rootfs/rootfs_test.py | 60 +++----- uapi16/storage.py | 2 +- uapi16/storage_test.py | 2 +- 30 files changed, 199 insertions(+), 520 deletions(-) diff --git a/box/build.bzl b/box/build.bzl index b990f87b..d4936cc3 100644 --- a/box/build.bzl +++ b/box/build.bzl @@ -66,7 +66,7 @@ def _repository_labels(repositories: list[Dependency]) -> list[str]: return sorted([str(repository.label.raw_target()) for repository in repositories]) def _layer_dir(actions: AnalysisActions, name: str) -> Artifact: - """Declare a directory output that holds a box root or the overlay work directory for one.""" + """Declare a directory output for a box root or the snapshot store of one.""" # A box root takes several gigabytes. Buck keeps every content-based version of an output until a # clean, so each rebuild that changes the root adds another full copy. A rebuild that produces the @@ -175,7 +175,6 @@ def _install( "packages_dir": packages, "snapshot": snapshot.as_output(), "target": stage2.as_output(), - "work": None, }, ), ), diff --git a/box/isolation.py b/box/isolation.py index ebeffc12..e43465b0 100644 --- a/box/isolation.py +++ b/box/isolation.py @@ -999,7 +999,7 @@ def attach(self, root: int = AT_FDCWD) -> None: class Overlay(_Mount): def __init__( self, - lowerdirs: tuple[StrPath, ...], + lowerdir: StrPath, # Without an upperdir and a workdir, the kernel mounts the overlay read-only. upperdir: StrPath | None, workdir: StrPath | None, @@ -1011,16 +1011,17 @@ def __init__( super().__init__(target, strict_unmount=strict_unmount) if (upperdir is None) != (workdir is None): raise ValueError("Overlay takes an upperdir and a workdir, or neither") - if upperdir is None and len(lowerdirs) + len(datadirs) < 2: - raise ValueError("a read-only Overlay needs at least two lowerdirs") - self.lowerdirs = tuple(os.fspath(path) for path in lowerdirs) + # overlayfs refuses a read-only mount of a single layer. + if upperdir is None and not datadirs: + raise ValueError("a read-only Overlay needs a datadir") + self.lowerdir = os.fspath(lowerdir) self.datadirs = tuple(os.fspath(path) for path in datadirs) self.upperdir = None if upperdir is None else os.fspath(upperdir) self.workdir = None if workdir is None else os.fspath(workdir) @override def _open(self) -> int: - return _overlay(self.lowerdirs, self.upperdir, self.workdir, datadirs=self.datadirs) + return _overlay((self.lowerdir,), self.upperdir, self.workdir, datadirs=self.datadirs) Filesystem = Bind | File | Devices | Tmpfs | Symlink | Overlay diff --git a/box/isolation_test.py b/box/isolation_test.py index e9a7b0df..4d443d44 100644 --- a/box/isolation_test.py +++ b/box/isolation_test.py @@ -337,19 +337,14 @@ def test_overlay_paths_may_contain_option_separators(self) -> None: root = Path(directory) # The mount option parser splits options at `,`. overlayfs splits lowerdir at `:` and treats `\` # as an escape. overlayfs gets its layers as file descriptors and doesn't parse their paths. - top, bottom, upper, work = (root / name for name in ("to:p", "bot,tom", "up,per", "wo\\rk")) - for path in (top, bottom, upper, work): + lower, upper, work = (root / name for name in ("lo:w,er", "up,per", "wo\\rk")) + for path in (lower, upper, work): path.mkdir() - (top / "original").write_text("original") - (bottom / "underneath").write_text("underneath") - (top / "shadowed").write_text("from the top") - (bottom / "shadowed").write_text("from the bottom") + (lower / "original").write_text("original") target = root / "target" - with isolation.Overlay((top, bottom), upper, work, target) as mounted: + with isolation.Overlay(lower, upper, work, target) as mounted: mounted.mount() self.assertEqual((target / "original").read_text(), "original") - self.assertEqual((target / "underneath").read_text(), "underneath") - self.assertEqual((target / "shadowed").read_text(), "from the top") (target / "generated").write_text("output") self.assertEqual((upper / "generated").read_text(), "output") @@ -381,7 +376,7 @@ def test_overlay_reads_data_layers_without_changing_blobs(self) -> None: before = (first / "shared").stat() target = root / "target" with isolation.Overlay( - (lower,), upper, work, target, datadirs=(first, second), strict_unmount=True + lower, upper, work, target, datadirs=(first, second), strict_unmount=True ) as mounted: mounted.mount() self.assertEqual({path.name for path in target.iterdir()}, {"a", "b", "c"}) @@ -405,25 +400,22 @@ def test_overlay_reads_data_layers_without_changing_blobs(self) -> None: def test_overlay_without_an_upper_is_read_only(self) -> None: with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: root = Path(directory) - top, bottom = root / "top", root / "bottom" - for path in (top, bottom): + lower, data = root / "lower", root / "data" + for path in (lower, data): path.mkdir() - (top / "shadowed").write_text("from the top") - (bottom / "shadowed").write_text("from the bottom") - (bottom / "underneath").write_text("underneath") + (lower / "underneath").write_text("underneath") target = root / "target" - with isolation.Overlay((top, bottom), None, None, target) as mounted: + with isolation.Overlay(lower, None, None, target, datadirs=(data,)) as mounted: mounted.mount() - self.assertEqual((target / "shadowed").read_text(), "from the top") self.assertEqual((target / "underneath").read_text(), "underneath") with self.assertRaises(OSError) as raised: (target / "generated").write_text("output") self.assertEqual(raised.exception.errno, errno.EROFS) - def test_overlay_needs_a_workdir_with_its_upper_and_two_lowers_without(self) -> None: - for lowers, upper, work in ((("lower",), "upper", None), (("lower",), None, None)): - with self.subTest(lowers=lowers, upper=upper, work=work), self.assertRaises(ValueError): - isolation.Overlay(lowers, upper, work, "target") + def test_overlay_needs_a_workdir_with_its_upper_and_a_datadir_without(self) -> None: + for upper, work in (("upper", None), (None, None)): + with self.subTest(upper=upper, work=work), self.assertRaises(ValueError): + isolation.Overlay("lower", upper, work, "target") def test_overlay_parent_keeps_its_mode_with_a_permissive_umask(self) -> None: with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: @@ -436,7 +428,7 @@ def test_overlay_parent_keeps_its_mode_with_a_permissive_umask(self) -> None: with ExitStack() as stack: stack.callback(os.umask, os.umask(0)) stack.enter_context(mock.patch.object(isolation, "_move_mount")) - with isolation.Overlay((lower,), upper, work, target) as overlay: + with isolation.Overlay(lower, upper, work, target) as overlay: overlay.attach() self.assertEqual(target.parent.stat().st_mode & 0o777, 0o755) @@ -527,7 +519,7 @@ def test_mounts_unwind_on_success_and_failure(self) -> None: isolation.Tmpfs(target), isolation.Devices(target), # Strict, so that unwinding covers both unmount flavours. - isolation.Overlay((lower,), upper, work, target, strict_unmount=True), + isolation.Overlay(lower, upper, work, target, strict_unmount=True), ] for mount in mounts: for fail in (False, True): diff --git a/box/sandbox_test.py b/box/sandbox_test.py index b117f999..2dfb590e 100644 --- a/box/sandbox_test.py +++ b/box/sandbox_test.py @@ -616,7 +616,7 @@ def test_a_start_leaves_the_identity_of_running_boxes_alone(self) -> None: upper, work = f"/var/tmp/{name}.upper", f"/var/tmp/{name}.work" os.mkdir(upper) os.mkdir(work) - with Overlay((lower,), upper, work, merged) as overlay: + with Overlay(lower, upper, work, merged) as overlay: overlay.attach() with open("/var/tmp/merged/sub/file") as handle: print(handle.read()) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index c0114afc..ee0e5fe1 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -50,7 +50,7 @@ tine//package/ package-system-neutral providers and installation flow tine//package_system/ one directory per package system: its repository, resolver, installer, extractor, and optional indexer and builder tine//box/ box bootstrap and sandbox command construction -tine//rootfs/ bind/overlay mounting and stored-delta translation +tine//rootfs/ bind mounts and Buck-storable names tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and drivers, and VM runners tine//uapi16/ UAPI.16 manifests and the snapshot stores built on them @@ -551,9 +551,12 @@ resolving both package lists together picks other packages than the parent got, `libcurl` for a box that needs it where the parent has `libcurl-minimal`; `layered = False` then installs the box on its own. -The sandbox merges the layers per top-level directory with a read-only overlay. Unlike `rootfs()`, it does -not decode stored whiteout markers, as that would walk every layer on every action, so the installer refuses -a box layer that deletes from the layers below. +Each box layer is a snapshot over the snapshots of its parent's layers, like an image layer. A child box +installs over its restored parent snapshot, so it sees the modes, hardlinks and extended attributes that the +package manager wrote. The installer also writes the objects that differ from the parent as a plain +directory with only what Buck stores: directories, symlinks and regular files with mode 0755 or 0644. The +sandbox merges these directories per top-level directory with a read-only overlay. A plain directory cannot +express a deletion, so the installer refuses a box layer that deletes from the layers below. If the parent has different repositories, the box only shares the parent's package list and is installed on its own. @@ -581,13 +584,14 @@ directory. Each chrooted `run` or `copy` mounts the project read-only under the the duration of that operation. Package scripts and remove globs cannot reach the project through that mount. apivfs covers `/run` with a tmpfs so the mount point is not captured into the image. -The sandbox only creates the execution environment. Drivers own their target-root layout through -`rootfs.rootfs()`: +The sandbox only creates the execution environment. Drivers restore their target root from snapshots and +bind it with `rootfs.rootfs()`: -- a fresh install or image layer binds an output directory at `/buildroot`; -- an incremental install or image layer mounts an ordered lower stack plus a persisted upper; -- a package build mounts its buildroot stack with an ephemeral upper and binds action scratch at `/build`; -- pack/disk operations merge a stack with an ephemeral upper so cleanup does not modify stored layers. +- an install or image layer restores its lower snapshots into a temporary root, binds it at `/buildroot` + and captures the result as a new snapshot; +- a package build restores its buildroot snapshots the same way and binds action scratch at `/build`; +- disk operations, and archives with authored tmpfiles rules, restore their snapshots into a temporary + root so cleanup does not modify stored layers. This division keeps one namespace boundary while letting each driver express the root it needs. Nesting a second sandbox inside a box would duplicate isolation, complicate mounts, and make remote execution @@ -605,10 +609,11 @@ actions never use relaxed mode. Native package installation has three phases shared by buildroots and images: 1. **Plan.** Run `PackageSystemInfo.plan` against each configured repository's materialized directory, - effective priority, and solver cache. Weak dependencies are disabled. Existing lower layers are mounted - read-only so installed packages can satisfy an incremental request. A request lists its packages in the - analysis, or in JSON files that an earlier action writes, such as the BuildRequires query of an rpm - package. A request with no packages installs nothing and leaves its layer empty. + effective priority, and solver cache. Weak dependencies are disabled. Existing lower snapshots are + restored into a temporary root so installed packages can satisfy an incremental request. A request lists + its packages in the analysis, or in JSON files that an earlier action writes, such as the BuildRequires + query of an rpm package. A request with no packages installs nothing, and its snapshot repeats the + manifest of the snapshot below it. 2. **Select.** Use the resulting transaction to select package files from repository pools, verified after download if the repository declares signing keys, named with the package system's declared suffix so its installer finds them. A local repository is materialized by @@ -620,9 +625,9 @@ Native package installation has three phases shared by buildroots and images: packages in it. Buildroots reject managers with local packages, because buildroot contents must come from the explicit, cycle-checked self-hosting locks. 3. **Install.** Run `PackageSystemInfo.install` over the exact package directory, whose file names carry - the package system's suffix. `install_packages()` owns a fresh root or incremental buildroot delta. An - image layer instead invokes the same installer against its already-mounted root so package and - filesystem operations have one output owner. + the package system's suffix. `install_packages()` writes a fresh root snapshot or an incremental + buildroot snapshot. An image layer instead invokes the same installer against its already-mounted root + so package and filesystem operations have one output owner. What a request looks like, rather than how one package system answers it, belongs to the neutral layer: `package/transaction.py` owns the transaction schema both planners write and Starlark reads back, and @@ -650,9 +655,9 @@ base. After installation, the installer parks the package database and scrubs the package-manager and ldconfig bookkeeping that would otherwise make identical roots differ; what parking takes is each package system's -own business. The action that owns the root then captures names and overlay metadata into Buck-storable -form. A fresh root receives the `uninitialized` machine-id marker systemd initializes on first boot; -incremental installs preserve any existing machine ID. +own business. The action that owns the root then captures it as a snapshot. A fresh root receives the +`uninitialized` machine-id marker systemd initializes on first boot; incremental installs preserve any +existing machine ID. `LocalPackageInfo` intentionally does not carry the producer's box. `local_repository` checks that its packages use one native package system and remains a box-independent declaration. An image or a buildroot @@ -722,13 +727,13 @@ An `rpm_package` action: read the contents of a spec, so an action stages the spec as the build does and runs `rpmbuild -bd` with the build's options in the base root and the delta. It writes the BuildRequires that are missing, and a second delta installs them. That delta is empty when the spec needs nothing more; -3. overlays the base and the deltas, stages its spec/sources in Buck action scratch, and runs `rpmbuild -ba`, - or `rpmbuild -bb --build-in-place` on a volatile overlay of a populated source tree. overlayfs shows the - directory under a mount rather than the mounted tree, so each directory mount below the source tree, - such as a `tine mount`, gets a volatile overlay of its own. The driver discovers directory mounts with - `listmount()` and `statmount()` using unique mount IDs. With `in_place_packaging`, a separate action - first fetches the dist-git commit that the tree pins, and the build takes its spec and Source/Patch files - from that commit. The project is read-only until the driver exits; +3. restores the base and the deltas into a temporary root, stages its spec/sources in Buck action scratch, + and runs `rpmbuild -ba`, or `rpmbuild -bb --build-in-place` on a volatile overlay of a populated source + tree. overlayfs shows the directory under a mount rather than the mounted tree, so each directory mount + below the source tree, such as a `tine mount`, gets a volatile overlay of its own. The driver discovers + directory mounts with `listmount()` and `statmount()` using unique mount IDs. With `in_place_packaging`, a + separate action first fetches the dist-git commit that the tree pins, and the build takes its spec and + Source/Patch files from that commit. The project is read-only until the driver exits; 4. freezes `%autorelease`, `_buildhost`, the dist tag, and the per-package source date epoch; 5. collects binary RPMs and the source RPM into one output directory, carrying its package-system identity in `LocalPackageInfo`; @@ -829,8 +834,6 @@ explicitly, because alpm rebases its system hook directory onto the target root administrator's, and the box's must not run against the image. Afterwards the driver replaces the wall-clock `%INSTALLDATE%` alpm records with the assembly epoch, and drops ldconfig's auxiliary cache, the sync databases and the transaction log. -It rewrites only the entries that actually change: alpm keeps one file per package, so rewriting an -entry a lower layer already parked would copy that whole database up into this layer's delta. The database stays at Arch's own `/var/lib/pacman`, since Arch has no usr-merged location for it. The package-database artifact is read from the assembled layer stack rather than from a terminal output, @@ -1157,9 +1160,9 @@ Parent release inspection reads the os-release file from the parent snapshot wit Temporary roots are removed when the operation exits, including on failure. There is no persistent materialized-root cache. -Boxes and other directory-layer callers of `rootfs` retain their direct mounting path and marker encoding: -`.wh.` represents a whiteout, `.wh..wh..opq` an opaque directory and `.esc.` an -unstorable component. Ordinary box startup does not parse image manifests or reconstruct image roots. +Box layers are snapshots too, but the sandbox mounts the plain directory that the installer writes from +each snapshot. In that directory, `.esc.` encodes a name that Buck cannot store. +Ordinary box startup does not parse image manifests or reconstruct image roots. Ownership remains uid/gid zero. Terminal outputs apply the image permission policy before authored tmpfiles rules and clamp timestamps to the assembly epoch. Permission normalization prepares the snapshot before diff --git a/image/layer.py b/image/layer.py index 4ce192dd..512ed515 100755 --- a/image/layer.py +++ b/image/layer.py @@ -238,10 +238,10 @@ def _install(install: LayerInstall, target: Path, scratch: Path) -> None: request = installer.InstallSpec( arch=install["arch"], packages_dir=str(Path(install["packages_dir"]).absolute()), - target=None, + snapshot=None, installroot=str(target), lower=[], - work=None, + target=None, box_config=False, langs=install["langs"], docs=install["docs"], diff --git a/image/layer_test.py b/image/layer_test.py index 1aac5a29..3dd7598b 100644 --- a/image/layer_test.py +++ b/image/layer_test.py @@ -69,7 +69,7 @@ def test_absolute_symlinks_resolve_inside_the_image(self) -> None: with ( chdir(self.project), - rootfs.rootfs(self.scratch / "target", bind=self.out, capture_bind=True) as target, + rootfs.rootfs(self.scratch / "target", bind=self.out) as target, ): layer._apply(["copy", "source", "/alias/file"], target) layer._apply(["os_release", {"ID": "test"}], target) diff --git a/package/BUCK b/package/BUCK index b3b268cf..8bd1f280 100644 --- a/package/BUCK +++ b/package/BUCK @@ -16,7 +16,7 @@ python.library( name = "builder", srcs = ["builder.py"], visibility = ["PUBLIC"], - deps = ["//:util", "//rootfs:rootfs"], + deps = ["//:util", "//rootfs:rootfs", "//uapi16:storage"], ) python.library( @@ -50,7 +50,7 @@ python.library( python.library( name = "transaction", srcs = ["transaction.py"], - deps = [":snapshotter", "//:util", "//rootfs:rootfs", "//uapi16:storage"], + deps = [":snapshotter", "//:util"], visibility = ["PUBLIC"], ) diff --git a/package/build.bzl b/package/build.bzl index 8408778c..15e93af9 100644 --- a/package/build.bzl +++ b/package/build.bzl @@ -74,7 +74,7 @@ def package_build( category: str, spec: dict[str, typing.Any], ) -> list[Provider]: - """Run the `build` driver of `package_system` on the layer stack `stack` that `base` starts. + """Run the `build` driver of `package_system` on the snapshot stores `stack` that `base` starts. `spec` contains the fields of the driver spec that the calling rule owns. `packages` names the output directory of the packages. A dev build keeps its build directory between runs if `keep_build_dir` is diff --git a/package/builder.py b/package/builder.py index b16320b9..4467600d 100644 --- a/package/builder.py +++ b/package/builder.py @@ -9,14 +9,15 @@ import shutil import tempfile -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext +from collections.abc import Callable, Generator +from contextlib import contextmanager, nullcontext from pathlib import Path from typing import TypedDict import util import rootfs +import storage # The source tree is mounted at topdir/CHECKOUT, and the persistent build directory at topdir/BUILD. CHECKOUT = "CHECKOUT" @@ -27,7 +28,7 @@ class Spec(TypedDict): # `build_dir` is a build directory that Buck keeps between runs, or None for a clean build. build_dir: str | None - # `lower` lists the layers of the buildroot, lowest first. + # `lower` lists the snapshot stores of the buildroot, lowest first. lower: list[str] out: str # `source_tree` is a prepared source tree to build in place, or None to build from the declared sources. @@ -76,8 +77,13 @@ def run(spec: Spec, build: Callable[[Path, Path, Path], int]) -> int: return rc -def buildroot(lower: list[str], topdir: Path) -> AbstractContextManager[Path]: - """Chroot into the buildroot with the layers `lower`, with topdir at `CHROOT_TOPDIR`.""" - return rootfs.rootfs( - "/buildroot", lowers=lower, binds=[(topdir, CHROOT_TOPDIR)], apivfs=True, chroot=True - ) +@contextmanager +def buildroot(lower: list[str], topdir: Path) -> Generator[Path]: + """Chroot into the buildroot that the snapshot stores `lower` restore, with topdir at `CHROOT_TOPDIR`.""" + with ( + storage.root(storage.Snapshot(lower)) as restored, + rootfs.rootfs( + "/buildroot", bind=restored.path, binds=[(topdir, CHROOT_TOPDIR)], apivfs=True, chroot=True + ) as root, + ): + yield root diff --git a/package/buildroot.bzl b/package/buildroot.bzl index c7132e6a..6a0ecfeb 100644 --- a/package/buildroot.bzl +++ b/package/buildroot.bzl @@ -86,7 +86,7 @@ def package_buildroot( build_requires: list[str], buildroot_packages: list[Dependency], ) -> list[Artifact]: - """Return the layer stack, bottom to top, that one package builds on. + """Return the snapshot stores, bottom to top, that one package builds on. The stack is the shared base root, plus a delta that installs `build_requires`. The solve of the delta prefers the packages of `buildroot_packages` over the repositories of the package manager. diff --git a/package/install.bzl b/package/install.bzl index 1dd655f4..b8089e39 100644 --- a/package/install.bzl +++ b/package/install.bzl @@ -21,7 +21,7 @@ load(":system.bzl", "PackageSystemInfo") _EXTRA_REPO_PRIORITY = 50 _RootInfo = provider( - doc = "The installed root tree carried out of an anonymous target.", + doc = "The snapshot store of an installed root, carried out of an anonymous target.", fields = {"root": provider_field(Artifact)}, ) @@ -117,8 +117,7 @@ def _install_actions( system = package_manager.package_system[PackageSystemInfo] closure = resolve_packages(ctx, package_manager_dep, box_dep, install, stack, extra_packages, identifier = identifier, install_files = install_files) prefix = identifier + "/" if identifier != None else "" - out = ctx.actions.declare_output(prefix + ("install.delta" if stack else "root"), dir = True) - work = ctx.actions.declare_output(prefix + "install.work", dir = True) if stack else None + out = ctx.actions.declare_output(prefix + ("install.snapshot" if stack else "root.snapshot"), dir = True) cmd = cmd_args( box.run(box = box_dep[box.Info], exe = system.install), specs.args( @@ -132,8 +131,8 @@ def _install_actions( "langs": [], "lower": stack, "packages_dir": closure, - "target": out.as_output(), - "work": work.as_output() if work != None else None, + "snapshot": out.as_output(), + "target": None, }, ), ) @@ -179,5 +178,5 @@ def install_packages( }, ).artifact("root") # A promise is resolved before the anon target is analysed, so its path kind is asserted here. - return ctx.actions.assert_short_path(ctx.actions.assert_has_content_based_path(root), short_path = "root") + return ctx.actions.assert_short_path(ctx.actions.assert_has_content_based_path(root), short_path = "root.snapshot") return _install_actions(ctx, package_manager, box_dep, install, stack, extra_packages, install_files, identifier) diff --git a/package/installer.py b/package/installer.py index 9bb9f859..bbcc4d98 100644 --- a/package/installer.py +++ b/package/installer.py @@ -4,10 +4,9 @@ """Run one native package installation against the root its spec names. Every package system's installer answers the same request: a directory holding the exact package -closure, and either a fresh root to fill, a delta to persist over a lower stack, or a root the -caller has already mounted. Which of those it is, and how the root is mounted and captured, is -the same work whichever system fills it, so it happens here and the driver is left with the -transaction itself. +closure, and either a snapshot to capture over lower snapshots, or a root the caller has already +mounted. Every package system restores and captures the root the same way, so this module does it and +the driver only runs the transaction. """ import os @@ -15,13 +14,14 @@ from collections.abc import Callable, Generator from contextlib import contextmanager from pathlib import Path -from typing import NotRequired, TypedDict +from typing import TypedDict import specs from util import fail import rootfs import storage +import uapi16 # A fixed install path avoids embedding Buck hashes and supports scriptlet chroots. BUILDROOT = "/buildroot" @@ -62,15 +62,13 @@ class InstallSpec(TypedDict): arch: str packages_dir: str - # Either an output root, bound at /buildroot to install into, or a root the caller mounted. - target: str | None + # The install captures the snapshot store `snapshot` over the snapshot stores in `lower`. An image layer + # sets `installroot` to the root that it mounted instead. + snapshot: str | None installroot: str | None lower: list[str] - work: str | None - # Only a box install sets `snapshot`. The install then restores the snapshot stores in `lower`, - # captures its own snapshot store at `snapshot` and writes the directory that the sandbox mounts to - # `target`. - snapshot: NotRequired[str | None] + # Only a box install sets `target`, to the directory that the sandbox mounts. + target: str | None box_config: bool langs: list[str] docs: bool @@ -142,13 +140,12 @@ def _normalize(installroot: Path, spec: InstallSpec) -> None: def run( prog: str, - install: Callable[[Path, Path, InstallSpec, bool], None], + install: Callable[[Path, Path, InstallSpec], None], argv: list[str] | None = None, ) -> None: """Mount the root this invocation names and hand it to `install`. - The callback receives the closure directory, the root to install into, the spec, and whether - it is layering over packages a lower stack already carries. + The callback receives the closure directory, the root to install into and the spec. """ spec = specs.parse(InstallSpec, prog, argv) os.environ.update( @@ -167,75 +164,38 @@ def run( # An installer resolves its own paths against the root, so give it no relative ones. packages_dir = Path(spec["packages_dir"]).absolute() - layered = bool(spec["lower"]) + lowers = spec["lower"] if spec["installroot"] is not None: - if spec["target"] is not None or layered or spec["work"] is not None: - fail(f"{prog}: installroot excludes target, lower, and work") + if spec["snapshot"] is not None or spec["target"] is not None or lowers: + fail(f"{prog}: installroot excludes snapshot, target and lower") installroot = Path(spec["installroot"]).absolute() - install(packages_dir, installroot, spec, layered) + install(packages_dir, installroot, spec) _normalize(installroot, spec) return - if spec["target"] is None: - fail(f"{prog}: one of target and installroot is required") + if spec["snapshot"] is None: + fail(f"{prog}: one of snapshot and installroot is required") + out = Path(spec["snapshot"]) - if (snapshot := spec.get("snapshot")) is not None: - _install_box(prog, install, packages_dir, spec, Path(snapshot), Path(spec["target"])) - return - - # Bind sources must exist. - target = Path(spec["target"]) - target.mkdir(parents=True, exist_ok=True) - - if layered: - if spec["work"] is None: - fail(f"{prog}: lower needs a work overlay directory") - # The closure is empty when the lower layers contain every package of the layer, for example in - # the buildroot of a package whose spec needs nothing beyond the base root, or in a box layer - # that adds nothing to its parent. The layer then stays empty, and the package manager doesn't - # run. Buck still expects the work directory as an output. - if not any(packages_dir.iterdir()): - Path(spec["work"]).mkdir(parents=True, exist_ok=True) - return - root = rootfs.rootfs( - BUILDROOT, - lowers=spec["lower"], - upperdir=target, - workdir=spec["work"], - apivfs=True, - ) + # The closure is empty when the lower snapshots contain every package of the layer, for example in the + # buildroot of a package whose spec needs nothing beyond the base root. The package manager then doesn't + # run, and the snapshot only repeats the manifest of the snapshot below it. + if lowers and not any(packages_dir.iterdir()): + (out / "blobs").mkdir(parents=True) + shutil.copyfile(Path(lowers[-1]) / uapi16.MANIFEST, out / uapi16.MANIFEST) else: - root = rootfs.rootfs(BUILDROOT, bind=target, capture_bind=True, apivfs=True) - - with root: - install(packages_dir, Path(BUILDROOT), spec, layered) - _normalize(Path(BUILDROOT), spec) + with storage.root(storage.Snapshot(lowers)) as restored: + with rootfs.rootfs(BUILDROOT, bind=restored.path, apivfs=True, strict_unmount=True): + install(packages_dir, Path(BUILDROOT), spec) + _normalize(Path(BUILDROOT), spec) + storage.capture(restored, out, lowers, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) - # The sandbox stacks a box's layers as Buck stores them, without turning markers back into - # overlayfs whiteouts. Deletions are not supported at box runtime right now. - if layered and spec["box_config"] and (deleted := rootfs.deletions(target)): - markers = ", ".join(map(str, deleted)) - fail(f"{prog}: a box layer cannot delete from the layers below it, but has {markers}") - - -def _install_box( - prog: str, - install: Callable[[Path, Path, InstallSpec, bool], None], - packages_dir: Path, - spec: InstallSpec, - out: Path, - target: Path, -) -> None: - lowers = spec["lower"] - with storage.root(storage.Snapshot(lowers)) as restored: - with rootfs.rootfs(BUILDROOT, bind=restored.path, apivfs=True, strict_unmount=True): - install(packages_dir, Path(BUILDROOT), spec, bool(lowers)) - _normalize(Path(BUILDROOT), spec) - storage.capture(restored, out, lowers, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) + if spec["target"] is None: + return snapshot = storage.Snapshot([*lowers, out]) # The sandbox stacks the directories of a box's layers and decodes no whiteouts. A file that a # layer deletes would still show up from the layer below. if deleted := storage.delta(snapshot, len(lowers)): fail(f"{prog}: a box layer cannot delete from the layers below it, but deletes {', '.join(deleted)}") - storage.write_buck_tree(snapshot, target) + storage.write_buck_tree(snapshot, Path(spec["target"])) diff --git a/package/installer_test.py b/package/installer_test.py index 3b13474d..c92b1f9f 100644 --- a/package/installer_test.py +++ b/package/installer_test.py @@ -7,7 +7,6 @@ import subprocess import tempfile import unittest -from contextlib import nullcontext from pathlib import Path from unittest import mock @@ -62,21 +61,21 @@ def test_package_scripts_tolerate_unmapped_acl_groups(self) -> None: with ( self.subTest(mode=mode), tempfile.TemporaryDirectory(dir="/var/tmp") as scratch, - mock.patch.dict(os.environ, {"SYSTEMD_IN_CHROOT": "0"}), + mock.patch.dict(os.environ, {"SYSTEMD_IN_CHROOT": "0", "SOURCE_DATE_EPOCH": "0"}), ): root = Path(scratch) - target, lower = root / "target", root / "lower" + target = root / "target" target.mkdir() - lower.mkdir() + lower = _snapshot(root, "lower", {}) if mode == "layered" else None (root / "packages").mkdir() (root / "packages" / "systemd.rpm").write_text("") spec = installer.InstallSpec( arch="x86_64", packages_dir=str(root / "packages"), - target=None if mode == "mounted" else str(target), + snapshot=None if mode == "mounted" else str(root / "snapshot"), installroot=str(target) if mode == "mounted" else None, - lower=[str(lower)] if mode == "layered" else [], - work=str(root / "work") if mode == "layered" else None, + lower=[str(lower)] if lower is not None else [], + target=None, box_config=False, langs=[], docs=True, @@ -86,7 +85,6 @@ def install( packages_dir: Path, installroot: Path, request: installer.InstallSpec, - layered: bool, ) -> None: (installroot / "journal").mkdir() # Debian's systemd postinst applies an ACL for adm (GID 4), which a namespace @@ -104,7 +102,7 @@ def install( def test_a_box_layer_may_add_but_not_delete(self) -> None: # The sandbox mounts the directories of a box's layers without decoding deletions. Install a box # layer over a lower snapshot that adds a file, and once more so that it also deletes a file of the - # lower snapshot. The first install must write only the added file, the second must fail. + # lower snapshot. The first install must export only the added file, the second must fail. for deletes in (False, True): with ( self.subTest(deletes=deletes), @@ -118,11 +116,10 @@ def test_a_box_layer_may_add_but_not_delete(self) -> None: spec = installer.InstallSpec( arch="x86_64", packages_dir=str(root / "packages"), - target=str(root / "target"), + snapshot=str(root / "snapshot"), installroot=None, lower=[str(lower)], - work=None, - snapshot=str(root / "snapshot"), + target=str(root / "target"), box_config=False, langs=[], docs=True, @@ -132,7 +129,6 @@ def install( packages_dir: Path, installroot: Path, request: installer.InstallSpec, - layered: bool, deletes: bool = deletes, ) -> None: (installroot / "added").write_text("") @@ -147,21 +143,21 @@ def install( installer.run("install", install, []) self.assertEqual([path.name for path in (root / "target").iterdir()], ["added"]) - def test_an_empty_closure_leaves_the_layer_empty(self) -> None: - # A layered install with no packages to install must not run the package manager. Buck still - # expects the work directory as an output. Install an empty package directory over one lower - # layer, and check that the layer is empty and that the work directory exists. + def test_an_empty_closure_repeats_the_lower_snapshot(self) -> None: + # A layered install with no packages to install must not run the package manager. Install an empty + # package directory over one lower snapshot, and check that the package manager doesn't run and + # that the new snapshot restores the same tree as the lower snapshot. with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: root = Path(scratch) - (root / "lower").mkdir() + lower = _snapshot(root, "lower", {"etc/os-release": "ID=test\n"}) (root / "packages").mkdir() spec = installer.InstallSpec( arch="x86_64", packages_dir=str(root / "packages"), - target=str(root / "target"), + snapshot=str(root / "snapshot"), installroot=None, - lower=[str(root / "lower")], - work=str(root / "work"), + lower=[str(lower)], + target=None, box_config=False, langs=[], docs=True, @@ -170,51 +166,10 @@ def test_an_empty_closure_leaves_the_layer_empty(self) -> None: with mock.patch.object(installer.specs, "parse", return_value=spec): installer.run("install", install, []) install.assert_not_called() - self.assertEqual(list((root / "target").iterdir()), []) - self.assertTrue((root / "work").is_dir()) - - def test_fresh_root_mount_owns_capture_after_an_install_failure(self) -> None: - with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: - root = Path(scratch) - target = root / "target" - spec = installer.InstallSpec( - arch="x86_64", - packages_dir=str(root / "packages"), - target=str(target), - installroot=None, - lower=[], - work=None, - box_config=False, - langs=[], - docs=True, - ) - - def fail( - packages_dir: Path, - installroot: Path, - request: installer.InstallSpec, - layered: bool, - ) -> None: - raise RuntimeError("install failed") - - with ( - mock.patch.dict(os.environ), - mock.patch.object(installer.specs, "parse", return_value=spec), - mock.patch.object( - installer.rootfs, - "rootfs", - return_value=nullcontext(), - ) as mounted, - self.assertRaisesRegex(RuntimeError, "install failed"), - ): - installer.run("install", fail, []) - - mounted.assert_called_once_with( - installer.BUILDROOT, - bind=target, - capture_bind=True, - apivfs=True, + self.assertEqual( + storage.Snapshot([lower, root / "snapshot"]).objects, storage.Snapshot([lower]).objects ) + self.assertEqual(list((root / "snapshot/blobs").iterdir()), []) if __name__ == "__main__": diff --git a/package/transaction.py b/package/transaction.py index f6e88564..ee6632a1 100644 --- a/package/transaction.py +++ b/package/transaction.py @@ -10,17 +10,12 @@ """ import json -from collections.abc import Generator, Sequence -from contextlib import contextmanager from pathlib import Path from typing import Literal, NamedTuple, NotRequired, TypedDict from util import fail, text_destination -import rootfs import snapshotter -import storage -import uapi16 from snapshotter import MetadataFile @@ -175,15 +170,3 @@ def write(path: Path, packages: list[TransactionPackage], repositories: list[Rep with text_destination(path) as output: json.dump(entries, output, indent=2) output.write("\n") - - -@contextmanager -def installed_root(lowers: Sequence[str]) -> Generator[Path]: - # A snapshot store contains `uapi16.MANIFEST` at its top level. A box layer is a plain directory - # tree, and no package installs a file of that name into the root. - if (Path(lowers[-1]) / uapi16.MANIFEST).exists(): - with storage.root(storage.Snapshot(lowers)) as restored: - yield restored.path - else: - with rootfs.rootfs("/installroot", lowers=lowers) as tree: - yield tree diff --git a/package_system/deb/BUCK b/package_system/deb/BUCK index 2bc7ece4..cc4b686c 100644 --- a/package_system/deb/BUCK +++ b/package_system/deb/BUCK @@ -46,6 +46,7 @@ python.binary( "//:util", "//package:href", "//package:transaction", + "//uapi16:storage", ], ) @@ -133,5 +134,6 @@ python.test( "//package:installer", "//package:snapshotter", "//package:transaction", + "//uapi16:storage", ], ) diff --git a/package_system/deb/install.py b/package_system/deb/install.py index 46263e58..f3e4646b 100755 --- a/package_system/deb/install.py +++ b/package_system/deb/install.py @@ -202,8 +202,8 @@ def install( # `plan.py` pins repositories above 1000, so it can select a version that is older # than the installed version. "--allow-downgrades", - # APT resolves the transaction again here. Refuse a removal as `plan.py` does: removing - # a package of a lower layer would write whiteouts into this layer. + # APT resolves the transaction again here. Refuse a removal as `plan.py` does, because a + # transaction cannot express a removal. "--no-remove", "install", *(str(package) for package in packages), @@ -237,7 +237,7 @@ def scrub(installroot: Path) -> None: update.unlink() -def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, layered: bool) -> None: +def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec) -> None: with installer.fresh_machine_id(installroot): install( packages_dir, @@ -246,10 +246,7 @@ def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, langs=spec["langs"], docs=spec["docs"], ) - # Scrub only after a transaction that installed a package. On an overlay, `unlink()` of a - # file of a lower layer writes a whiteout into this layer. - if any(path.suffix == ".deb" for path in packages_dir.iterdir()): - scrub(installroot) + scrub(installroot) def main(argv: list[str] | None = None) -> None: diff --git a/package_system/deb/plan.py b/package_system/deb/plan.py index aa76c968..9a1894b9 100755 --- a/package_system/deb/plan.py +++ b/package_system/deb/plan.py @@ -25,6 +25,7 @@ import util from util import MAGIC, decompressor +import storage import transaction from href import relative_href @@ -194,7 +195,7 @@ def solve( scratch = Path(stack.enter_context(TemporaryDirectory(prefix="deb-plan.", dir="/var/tmp"))) root = scratch / "root" if lower: - root = stack.enter_context(transaction.installed_root(lower)) + root = stack.enter_context(storage.root(storage.Snapshot(lower))).path status = root / "var/lib/dpkg/status" status.parent.mkdir(parents=True, exist_ok=True) status.touch(exist_ok=True) diff --git a/package_system/pacman/BUCK b/package_system/pacman/BUCK index dc649268..0933d49b 100644 --- a/package_system/pacman/BUCK +++ b/package_system/pacman/BUCK @@ -46,6 +46,7 @@ python.binary( "//:util", "//package:href", "//package:transaction", + "//uapi16:storage", ], ) @@ -99,7 +100,7 @@ python.test( name = "plan-test", box = "//tools:dev.box", srcs = ["alpm.py", "plan.py", "plan_test.py"], - deps = ["//:specs", "//:util", "//package:href", "//package:transaction"], + deps = ["//:specs", "//:util", "//package:href", "//package:transaction", "//uapi16:storage"], ) python.test( diff --git a/package_system/pacman/install.py b/package_system/pacman/install.py index ca2c6957..34b33f31 100755 --- a/package_system/pacman/install.py +++ b/package_system/pacman/install.py @@ -135,10 +135,7 @@ def parkdb(installroot: Path, epoch: int) -> None: f"{epoch}\n" if index and lines[index - 1].strip() == "%INSTALLDATE%" else line for index, line in enumerate(lines) ) - # Entries a lower layer already parked are unchanged, and writing one anyway would copy - # the whole inherited database up into this layer's delta. - if stamped != text: - desc.write_text(stamped, encoding="utf-8") + desc.write_text(stamped, encoding="utf-8") def scrub(installroot: Path) -> None: @@ -147,7 +144,7 @@ def scrub(installroot: Path) -> None: (installroot / "var/log/pacman.log").unlink(missing_ok=True) -def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, layered: bool) -> None: +def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec) -> None: with installer.fresh_machine_id(installroot), tempfile.TemporaryDirectory(prefix="pacman.") as scratch: install( packages_dir, diff --git a/package_system/pacman/plan.py b/package_system/pacman/plan.py index 5f5bb094..10fbc6d9 100755 --- a/package_system/pacman/plan.py +++ b/package_system/pacman/plan.py @@ -27,6 +27,7 @@ from util import fail import alpm +import storage import transaction from href import relative_href @@ -109,8 +110,7 @@ def main(argv: list[str] | None = None) -> None: with ExitStack() as stack: scratch = Path(stack.enter_context(TemporaryDirectory(prefix="plan."))) if spec["lower"]: - # An ephemeral upper keeps the lower stack unchanged while alpm writes into the root. - root = stack.enter_context(transaction.installed_root(spec["lower"])) + root = stack.enter_context(storage.root(storage.Snapshot(spec["lower"]))).path else: root = scratch / "root" root.mkdir() diff --git a/package_system/rpm/BUCK b/package_system/rpm/BUCK index f1df5f52..1abf20a1 100644 --- a/package_system/rpm/BUCK +++ b/package_system/rpm/BUCK @@ -35,7 +35,7 @@ python.binary( name = "plan", main = "plan.py", boxes = [_BOX], - deps = [":macros", "//:specs", "//:util", "//package:transaction"], + deps = [":macros", "//:specs", "//:util", "//package:transaction", "//uapi16:storage"], ) python.library(name = "rpmbuild", srcs = ["rpmbuild.py"], deps = ["//:util", "//package:builder"]) @@ -96,7 +96,7 @@ python.test( name = "plan-test", box = _BOX, srcs = ["createrepo.py", "plan.py", "plan_test.py"], - deps = [":macros", "//:specs", "//:util", "//package:transaction"], + deps = [":macros", "//:specs", "//:util", "//package:transaction", "//uapi16:storage"], ) python.test( diff --git a/package_system/rpm/build_test.py b/package_system/rpm/build_test.py index beb2bb5f..8e702152 100644 --- a/package_system/rpm/build_test.py +++ b/package_system/rpm/build_test.py @@ -33,6 +33,15 @@ def setUp(self) -> None: self.project.mkdir() self.auxiliary = self.scratch / "auxiliary" self.auxiliary.write_text("packaging input\n") + # The tests replace the buildroot mount, so the buildroot doesn't need a snapshot to restore. + self.enterContext(mock.patch.object(build.builder.storage, "Snapshot")) + self.enterContext( + mock.patch.object( + build.builder.storage, + "root", + return_value=contextlib.nullcontext(build.builder.storage.Root(Path("buildroot"))), + ) + ) def specification( self, diff --git a/package_system/rpm/buildrequires.py b/package_system/rpm/buildrequires.py index 45f67bd1..8c5a939a 100755 --- a/package_system/rpm/buildrequires.py +++ b/package_system/rpm/buildrequires.py @@ -28,7 +28,7 @@ class Spec(rpmbuild.Parse): - # `lower` lists the layers of the buildroot with declared BuildRequires installed, bottom first. + # `lower` lists the snapshot stores of the buildroot with declared BuildRequires installed, bottom first. lower: list[str] # `source_tree` is the source tree of an in-place build, or None. source_tree: str | None diff --git a/package_system/rpm/install.py b/package_system/rpm/install.py index 112e016e..8e6ae691 100755 --- a/package_system/rpm/install.py +++ b/package_system/rpm/install.py @@ -166,14 +166,13 @@ def install_into_root( scrub(installroot) -def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, layered: bool) -> None: +def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec) -> None: install_into_root( packages_dir, installroot, arch=spec["arch"], - # Installed packages satisfy dependencies for an incremental install; a root the caller - # mounted says so by already carrying an rpmdb. - system=layered or (installroot / DBPATH / "rpmdb.sqlite").exists(), + # Installed packages satisfy dependencies for an install over a root that contains an rpmdb. + system=(installroot / DBPATH / "rpmdb.sqlite").exists(), langs=spec["langs"], docs=spec["docs"], ) diff --git a/package_system/rpm/plan.py b/package_system/rpm/plan.py index bc303313..bbb80cd5 100755 --- a/package_system/rpm/plan.py +++ b/package_system/rpm/plan.py @@ -23,6 +23,7 @@ from util import fail import macros +import storage import transaction # A multilib package in a pinned-arch transaction indicates a bad solve. @@ -248,8 +249,7 @@ def main(argv: list[str] | None = None) -> None: with ExitStack() as stack: installroot = None if solve_spec["lower"]: - # An ephemeral upper keeps the lower stack unchanged. - installroot = stack.enter_context(transaction.installed_root(solve_spec["lower"])) + installroot = stack.enter_context(storage.root(storage.Snapshot(solve_spec["lower"]))).path tx = plan( repos, install, diff --git a/rootfs/rootfs.py b/rootfs/rootfs.py index 412e3dae..d3e11ebb 100644 --- a/rootfs/rootfs.py +++ b/rootfs/rootfs.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Mount target roots and translate stored OCI markers to native overlayfs state. +"""Mount target roots and capture output trees in a form Buck can store. -Buck cannot store device nodes, xattrs, or backslashes in paths, so deltas encode them as -regular marker files. Sidecar layers reconstruct markers on entry; persisted uppers are -captured back to the storage form after unmounting. +Buck cannot store backslashes in paths, so `capture()` escapes names that contain them. `readonly_project()` makes the project read-only while a build driver runs. `source_overlay()` gives a tool that writes into its sources, such as rpmbuild, a view of them that discards the writes. @@ -13,12 +11,10 @@ import errno import os -import shutil import stat import tempfile from collections.abc import Generator, Mapping, Sequence from contextlib import ExitStack, contextmanager, suppress -from enum import Enum, auto from pathlib import Path from isolation import ( @@ -35,13 +31,7 @@ chroot as _chroot, ) -_WH = ".wh." # `.wh.`: a whiteout hiding from lower layers -_OPAQUE = ".wh..wh..opq" # marks its parent dir opaque (hide everything below it) _ESC = ".esc." # `.esc.`: a name buck can't store, percent-escaped -_OPAQUE_XATTR = b"user.overlay.opaque" # an opaque dir's native form (userxattr namespace) - - -# OCI delta to native overlay translation. def escape(name: str) -> str: @@ -51,168 +41,35 @@ def escape(name: str) -> str: return _ESC + name.replace("%", "%25").replace("\\", "%5C") -def _unescape(name: str) -> str: - return name.removeprefix(_ESC).replace("%5C", "\\").replace("%25", "%") - - -class _Kind(Enum): - """What a stored delta entry means once decoded.""" - - NORMAL = auto() # a plain entry, carried through under its true name - WHITEOUT = auto() # deletes the payload name from lower layers - OPAQUE = auto() # marks its parent dir opaque - - -def _classify(name: str) -> tuple[_Kind, str]: - """Decode a stored name into its marker kind and payload.""" - true = _unescape(name) if name.startswith(_ESC) else name - if true == _OPAQUE: - return _Kind.OPAQUE, "" - if true.startswith(_WH): - return _Kind.WHITEOUT, true[len(_WH) :] - return _Kind.NORMAL, true - - -def deletions(delta: Path) -> list[Path]: - """The marker files of a stored delta that delete from the layers below it.""" - markers = (path for path in delta.rglob("*") if _classify(path.name)[0] is not _Kind.NORMAL) - return sorted(path.relative_to(delta) for path in markers) - - -def _whiteout_marker(name: str) -> str: - """Encode a possibly unstorable whiteout target.""" - return escape(_WH + name) - - -def _whiteout(directory: Path, name: str) -> None: - directory.mkdir(parents=True, exist_ok=True) - os.mknod(str(directory / name), stat.S_IFCHR | 0o600, os.makedev(0, 0)) - - -def _thaw(src: Path, dst: Path) -> None: - """Copy an escaped entry under its true name, recursively decoding markers.""" - if src.is_symlink() or not src.is_dir(): - shutil.copy2(src, dst, follow_symlinks=False) - return - dst.mkdir(mode=stat.S_IMODE(src.lstat().st_mode)) - for child in src.iterdir(): - kind, payload = _classify(child.name) - if kind is _Kind.OPAQUE: - os.setxattr(str(dst), _OPAQUE_XATTR, b"y") - elif kind is _Kind.WHITEOUT: - _whiteout(dst, payload) - else: - _thaw(child, dst / payload) - shutil.copystat(src, dst, follow_symlinks=False) - - -def _markers(delta: Path, dest: Path) -> tuple[Path | None, Path | None]: - """Express stored markers as sparse sidecar layers above and below `delta`.""" - above, below = dest / "above", dest / "below" - used_above = used_below = False - for p in delta.rglob("*"): - rel = p.parent.relative_to(delta) - if any(part.startswith(_ESC) for part in rel.parts): - continue # inside an escaped dir: thawed wholesale when its ancestor was hit - kind, payload = _classify(p.name) - if kind is _Kind.OPAQUE: - opaque = below / rel - opaque.mkdir(parents=True, exist_ok=True) - os.setxattr(str(opaque), _OPAQUE_XATTR, b"y") - _whiteout(above / rel, p.name) # hide the marker file itself - used_above = used_below = True - elif kind is _Kind.WHITEOUT: - _whiteout(above / rel, payload) # whiteout the deleted target - _whiteout(above / rel, p.name) # hide the literal marker file (`.wh.` or `.esc.`) - used_above = True - elif p.name.startswith(_ESC): # an escaped normal entry - (above / rel).mkdir(parents=True, exist_ok=True) - _thaw(p, above / rel / payload) # materialize it under its true name - _whiteout(above / rel, p.name) # hide the `.esc.` entry itself - used_above = True - return (above if used_above else None, below if used_below else None) - - -def _reconstruct(lowers: list[Path], scratch: Path) -> list[Path]: - """Frame stored deltas as an overlayfs lowerdir list in top-to-bottom order.""" - components: list[Path] = [] - for i, lo in enumerate(reversed(lowers)): - above, below = _markers(lo, scratch / f"m{i}") - if above is not None: - components.append(above) - components.append(lo) - if below is not None: - components.append(below) - return components - - -def _is_opaque(d: Path) -> bool: - try: - return os.getxattr(str(d), _OPAQUE_XATTR) == b"y" - except OSError: - return False - - -def _in_lower(lower: Path, rel: Path) -> bool: - """Return whether the stored layer `lower` has an entry at `rel`, without following its symlinks.""" - path = lower - for part in rel.parts: - if path.is_symlink() or not path.is_dir(): - return False - path = path / escape(part) - if not os.path.lexists(path): - return False - return True - - -def capture(tree: Path, lowers: Sequence[Path] | None = None) -> None: - """Rewrite native overlay state and unstorable names into regular marker files. - - `lowers` are the stored layers below `tree`, if `tree` is an overlay upper directory. - """ +def capture(tree: Path) -> None: + """Make every directory of `tree` writable for its owner and escape the names Buck cannot store.""" # Buck must be able to delete and rename through package-supplied read-only directories. for p in tree.rglob("*"): st = p.lstat() if stat.S_ISDIR(st.st_mode) and stat.S_IMODE(st.st_mode) & 0o700 != 0o700: p.chmod(stat.S_IMODE(st.st_mode) | 0o700) for p in sorted(tree.rglob("*"), key=lambda q: len(q.parts), reverse=True): - st = p.lstat() - if stat.S_ISCHR(st.st_mode) and st.st_rdev == 0: - p.unlink() - (p.parent / _whiteout_marker(p.name)).write_bytes(b"") - continue - if stat.S_ISDIR(st.st_mode) and _is_opaque(p): - os.removexattr(str(p), _OPAQUE_XATTR) - # overlayfs marks a directory of the upper layer opaque when the directory is renamed - # into a directory that a lower layer also contains. dpkg unpacks every new directory - # under a temporary name and renames it into place, so a Debian layer gets an opaque mark - # on every new directory. - # The mark hides nothing when no lower layer has an entry at the path, so capture() writes - # no marker for it. - rel = p.relative_to(tree) - if lowers is None or any(_in_lower(lower, rel) for lower in lowers): - (p / _OPAQUE).write_bytes(b"") escaped = escape(p.name) if escaped != p.name: p.rename(p.parent / escaped) @contextmanager -def capture_on_exit(tree: str | Path, lowers: Sequence[Path] | None = None) -> Generator[Path]: +def capture_on_exit(tree: str | Path) -> Generator[Path]: """Capture a tree after use without hiding an error from the body.""" tree = Path(tree) try: yield tree except BaseException as error: try: - capture(tree, lowers) + capture(tree) except BaseException as capture_error: error.add_note( f"rootfs capture of {tree} also failed: {capture_error.__class__.__name__}: {capture_error}" ) raise else: - capture(tree, lowers) + capture(tree) # Root setup. @@ -220,11 +77,7 @@ def capture_on_exit(tree: str | Path, lowers: Sequence[Path] | None = None) -> G @contextmanager def source_overlay(source: Path, target: Path) -> Generator[Path]: - """Mount `source` at `target` as an overlay, and discard every write when the context exits. - - `rootfs()` decodes the `.wh.` and `.esc.` marker files of a stored image layer. This function - does not, because a checkout is not an image layer and can contain such names. - """ + """Mount `source` at `target` as an overlay, and discard every write when the context exits.""" # An overlay shows the directory that a mount in its lower directory covers, not the mounted tree. # A `tine mount` covers a directory of the project, so directory submounts need overlays of their own. # statmount() returns mount points with their symlinks resolved, so `source` needs them resolved too. @@ -245,7 +98,7 @@ def source_overlay(source: Path, target: Path) -> Generator[Path]: upper.chmod(stat.S_IMODE(attributes.st_mode)) os.utime(upper, ns=(attributes.st_atime_ns, attributes.st_mtime_ns)) try: - stack.enter_context(Overlay((lower,), upper, work, merged)).mount() + stack.enter_context(Overlay(lower, upper, work, merged)).mount() except OSError as error: if error.errno == errno.EINVAL: error.add_note( @@ -341,68 +194,16 @@ def chroot(target: str | Path) -> Generator[Path]: def rootfs( target: str | Path, *, - bind: str | Path | None = None, - capture_bind: bool = False, - lowers: Sequence[str | Path] | None = None, - upperdir: str | Path | None = None, - workdir: str | Path | None = None, + bind: str | Path, apivfs: bool = False, binds: Sequence[tuple[str | Path, str | Path]] | None = None, chroot: bool = False, strict_unmount: bool = False, ) -> Generator[Path]: - """Mount a bind or overlay root, optionally chrooting and persisting its output. - - Persistent overlay uppers and bind sources with `capture_bind=True` are captured after all - mounts have been torn down. - """ - if capture_bind and bind is None: - raise ValueError("rootfs(capture_bind=True) needs bind=") + """Bind `bind` at `target` with optional API filesystems, extra binds and a chroot.""" target = Path(target) with ExitStack() as stack: - if lowers is not None: - # _in_lower() treats a symlink as a missing directory. Without resolve(), capture() would find - # no entry in a layer that the caller passes as a symlink. - resolved = [Path(lo).resolve() for lo in lowers] - scratch = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="rootfs."))) - components = _reconstruct(resolved, scratch) - if not components: - empty = scratch / "empty" - empty.mkdir() - components.append(empty) - if upperdir is not None: - if workdir is None: - raise ValueError("rootfs(upperdir=...) needs a matching workdir=") - upper, work = Path(upperdir), Path(workdir) - else: - # A writable ephemeral upper also permits a single-component stack. - upper, work = scratch / "upper", scratch / "work" - upper.mkdir(parents=True, exist_ok=True) - work.mkdir(parents=True, exist_ok=True) - if upperdir is not None: - # Enter before registering the unmount so capture runs after it. - stack.enter_context(capture_on_exit(upper, resolved)) - # capture() reads and rewrites the upperdir after the unmount, so a captured upper needs a - # strict unmount. - mounted: Overlay | Bind = stack.enter_context( - Overlay( - tuple(components), - upper, - work, - target, - strict_unmount=upperdir is not None or strict_unmount, - ) - ) - elif bind is not None: - if capture_bind: - # Enter before registering the unmount so capture runs after it. - stack.enter_context(capture_on_exit(bind)) - # capture() reads the bind source after the unmount, like the upper of an overlay. - mounted = stack.enter_context( - Bind(Path(bind), target, strict_unmount=capture_bind or strict_unmount) - ) - else: - raise ValueError("rootfs needs bind= or lowers=") + mounted = stack.enter_context(Bind(Path(bind), target, strict_unmount=strict_unmount)) mounted.mount() # An image layer can contain an absolute symlink at a mount point. The mount points resolve inside # the root, so that the symlink doesn't place a mount outside of the root. diff --git a/rootfs/rootfs_test.py b/rootfs/rootfs_test.py index 651a23ed..474d61c3 100644 --- a/rootfs/rootfs_test.py +++ b/rootfs/rootfs_test.py @@ -41,8 +41,6 @@ def setUp(self) -> None: (self.source / "escaping-dangling-link").symlink_to(self.scratch / "absent") (self.source / "empty").mkdir() (self.source / "readonly").mkdir(mode=0o555) - for name in (".wh.original", ".wh..wh..opq", ".esc.literal"): - (self.source / name).write_text(name, encoding="utf-8") # overlayfs takes the mode and the times of the mount root from the upper directory. The # setgid bit and the old timestamp make the test fail if the root has the mode and the # times of a fresh upper directory. @@ -59,7 +57,7 @@ def test_an_invalid_overlay_has_a_possible_locked_mount_hint(self) -> None: pass self.assertIn("Locked mounts below the source can cause EINVAL", "".join(raised.exception.__notes__)) - def test_preserves_the_tree_without_decoding_image_markers(self) -> None: + def test_preserves_the_tree(self) -> None: with chdir(self.project), rootfs.source_overlay(Path("input"), self.scratch / "build/src") as tree: original = {path.relative_to(self.source) for path in self.source.rglob("*")} self.assertEqual({path.relative_to(tree) for path in tree.rglob("*")}, original) @@ -185,7 +183,7 @@ def test_is_a_tmpfs_outside_a_run_action(self) -> None: mock.patch.dict(os.environ), ): os.environ.pop("BUCK_SCRATCH_PATH", None) - with rootfs.rootfs(scratch / "target", lowers=[lower], apivfs=True) as tree: + with rootfs.rootfs(scratch / "target", bind=lower, apivfs=True) as tree: (tree / "var/tmp/staged").write_text("staged", encoding="utf-8") self.assertTrue((tree / "var/tmp").is_mount()) self.assertEqual(list(scratch.glob("var-tmp.*")), []) @@ -204,7 +202,7 @@ def test_is_backed_where_tmpdir_points(self) -> None: with ( mock.patch.dict(os.environ, environment), mock.patch.object(tempfile, "tempdir", None), - rootfs.rootfs(scratch / "target", lowers=[lower], apivfs=True) as tree, + rootfs.rootfs(scratch / "target", bind=lower, apivfs=True) as tree, ): (tree / "var/tmp/staged").write_text("staged", encoding="utf-8") backing = next(scratch.glob("var-tmp.*")) @@ -227,13 +225,15 @@ def test_an_absolute_symlink_in_a_layer_resolves_inside_the_root(self) -> None: self.subTest(apivfs=apivfs), rootfs.rootfs( scratch / "target", - lowers=[lower], + bind=lower, binds=[] if apivfs else [(scratch, "/run/project")], apivfs=apivfs, ) as tree, ): + # The bind of `scratch` is on the same filesystem as the root, so is_mount() can't detect + # it. `scratch` contains `lower`, and the root doesn't. if not apivfs: - self.assertTrue((tree / outside.relative_to("/") / "project").is_mount()) + self.assertTrue((tree / outside.relative_to("/") / "project/lower").is_dir()) self.assertFalse(outside.is_mount()) self.assertFalse((outside / "project").exists()) @@ -350,7 +350,7 @@ def test_captures_after_success(self) -> None: with rootfs.capture_on_exit(output): pass - capture.assert_called_once_with(output, None) + capture.assert_called_once_with(output) def test_capture_failure_does_not_hide_body_failure(self) -> None: output = Path("output") @@ -376,46 +376,20 @@ def test_capture_failure_after_success_is_reported(self) -> None: class TestCapture(unittest.TestCase): - def test_keeps_an_opaque_mark_only_where_a_lower_layer_has_the_directory(self) -> None: - with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: - lower, upper = Path(directory) / "lower", Path(directory) / "upper" - (lower / "usr/share/doc/old").mkdir(parents=True) - (lower / "usr/share/doc/old/file").write_text("old") - for name in ("old", "new"): - (upper / "usr/share/doc" / name).mkdir(parents=True, exist_ok=True) - os.setxattr(str(upper / "usr/share/doc" / name), b"user.overlay.opaque", b"y") - - rootfs.capture(upper, [lower]) - - self.assertEqual(rootfs.deletions(upper), [Path("usr/share/doc/old/.wh..wh..opq")]) - self.assertRaises(OSError, os.getxattr, str(upper / "usr/share/doc/new"), b"user.overlay.opaque") - - def test_keeps_every_opaque_mark_without_lower_layers(self) -> None: + def test_makes_directories_writable_and_escapes_backslashes(self) -> None: + # Buck can't remove a tree through a directory without owner write access, and it can't store a + # name with a backslash. Capture a tree with a read-only directory that contains such a name, and + # check that the directory becomes writable and the name is escaped. with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: tree = Path(directory) - (tree / "new").mkdir() - os.setxattr(str(tree / "new"), b"user.overlay.opaque", b"y") + (tree / "readonly").mkdir() + (tree / "readonly/back\\slash").write_text("kept") + (tree / "readonly").chmod(0o555) rootfs.capture(tree) - self.assertEqual(rootfs.deletions(tree), [Path("new/.wh..wh..opq")]) - - -class TestDeletions(unittest.TestCase): - def test_lists_whiteouts_and_opaque_markers_only(self) -> None: - with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: - delta = Path(directory) - (delta / "etc/opaque").mkdir(parents=True) - (delta / "etc/added").write_text("added") - (delta / "etc/.wh.removed").write_bytes(b"") - (delta / "etc/opaque/.wh..wh..opq").write_bytes(b"") - (delta / ".esc..wh.esc%5Caped").write_bytes(b"") - (delta / ".esc.back%5Cslash").write_text("kept") - - self.assertEqual( - rootfs.deletions(delta), - [Path(".esc..wh.esc%5Caped"), Path("etc/.wh.removed"), Path("etc/opaque/.wh..wh..opq")], - ) + self.assertEqual((tree / "readonly").stat().st_mode & 0o777, 0o755) + self.assertEqual([path.name for path in (tree / "readonly").iterdir()], [".esc.back%5Cslash"]) if __name__ == "__main__": diff --git a/uapi16/storage.py b/uapi16/storage.py index d227f340..fa1a14d0 100644 --- a/uapi16/storage.py +++ b/uapi16/storage.py @@ -329,6 +329,6 @@ def root(snapshot: Snapshot, *, readonly: bool = False, strategy: Strategy = "au if work is not None: work.mkdir() datadirs = tuple(store / "blobs" for store in snapshot.stores) - with Overlay((lower,), upper, work, tree, datadirs=datadirs) as mounted: + with Overlay(lower, upper, work, tree, datadirs=datadirs) as mounted: mounted.mount() yield Root(tree, upper) diff --git a/uapi16/storage_test.py b/uapi16/storage_test.py index 88215599..b15fe36c 100644 --- a/uapi16/storage_test.py +++ b/uapi16/storage_test.py @@ -377,7 +377,7 @@ def test_capture_native_overlay_changes(self) -> None: upper.mkdir() work.mkdir() child = self.scratch / "child" - with Overlay((lower,), upper, work, merged) as overlay: + with Overlay(lower, upper, work, merged) as overlay: overlay.mount() (merged / "deleted").unlink() shutil.rmtree(merged / "opaque")