image: Store filesystem snapshots in manifests

README.md

@@ -7,7 +7,7 @@ SPDX-License-Identifier: MPL-2.0
 
 tine composes operating-system images out of packages and pinned third-party repositories with
 [Buck2](https://buck2.build/). Every build action runs in an unprivileged sandbox without network access.
-Filesystem images are stacks of overlay deltas. tine's primary output targets are [unified kernel
+Each image layer describes a complete filesystem snapshot. tine's primary output targets are [unified kernel
 images](https://uapi-group.org/specifications/specs/unified_kernel_image/) and dm-verity protected GPT
 disks in various formats, and also additional artifacts like
 [SBOM](https://en.wikipedia.org/wiki/Software_supply_chain).
@@ -25,6 +25,7 @@ The host needs only:
  * `git`
  * `/bin/sh`, which runs the fetch of a Rust project's git dependencies
  * `/dev/kvm` to run VMs for testing; without it, you can still build everything
+ * Read access to `/proc/sys/kernel/overflowuid` for directory exports
 
 tine uses the host's `python` and `git` to bootstrap itself and to read the project's git state, such as the
 version an image derives from it. A build also runs the host's `git` to fetch git checkouts, and the host's

archive/BUCK

@@ -9,18 +9,19 @@ python.library(
     name = "cpio",
     srcs = ["cpio.py"],
     visibility = ["tine//..."],
-    deps = ["//:util"],
+    deps = ["//:util", "//uapi16:uapi16"],
 )
 
 python.library(
     name = "tar",
     srcs = ["tar.py"],
     visibility = ["tine//..."],
+    deps = ["//uapi16:uapi16"],
 )
 
 python.test(
     name = "test",
     box = "//tools:dev.box",
     srcs = ["cpio.py", "cpio_test.py", "tar.py", "tar_test.py"],
-    deps = ["//:util"],
+    deps = ["//:util", "//uapi16:uapi16"],
 )

archive/cpio.py

@@ -10,12 +10,15 @@
 import mmap
 import os
 import stat
-from collections.abc import Iterable, Iterator
+from collections import Counter
+from collections.abc import Callable, Iterable, Iterator, Sequence
 from pathlib import Path, PurePath, PurePosixPath
 from typing import NamedTuple, Self
 
 import util
 
+import uapi16
+
 # What an archive is read from, spelled concretely because the reader slices it: an mmap and a
 # `bytes` both hand back `bytes`, where the `Buffer` protocol promises no indexing at all.
 type Source = mmap.mmap | bytes
@@ -151,7 +154,7 @@ def _extract(source: Source, dest: Path, src_fd: int, start: int) -> int:
         elif fmt == 0o120000:  # symlink (last-writer-wins across multi-archive extracts)
             target.parent.mkdir(parents=True, exist_ok=True)
             target.unlink(missing_ok=True)
-            target.symlink_to(e.target.decode())
+            target.symlink_to(os.fsdecode(e.target))
         elif fmt == 0o100000:  # regular file
             _put_regular(target, e, src_fd)
         else:  # device/fifo/socket
@@ -163,10 +166,13 @@ def _extract(source: Source, dest: Path, src_fd: int, start: int) -> int:
                 _hardlink(link, target)
                 count += 1
     for links in pending.values():  # a fully-empty hardlink set has no content carrier
-        for target, mode in links:
-            target.parent.mkdir(parents=True, exist_ok=True)
-            target.write_bytes(b"")
-            target.chmod(mode & 0o7777)
+        first, mode = links[0]
+        first.parent.mkdir(parents=True, exist_ok=True)
+        first.write_bytes(b"")
+        first.chmod(mode & 0o7777)
+        count += 1
+        for target, _ in links[1:]:
+            _hardlink(target, first)
             count += 1
     return count
 
@@ -197,6 +203,7 @@ def __init__(self, path: Path, epoch: int, *, block_align: bool = True) -> None:
         self._epoch = epoch
         self._pos = 0
         self._ino = 0
+        self._links: dict[int, tuple[int, int]] = {}
         # Alignment pads the name field, and the kernel's early cpio reader (lib/earlycpio.c) refuses
         # an entry whose name field exceeds its 18-byte name buffer, so an archive it reads before
         # unpacking anything, microcode, is written without.
@@ -221,8 +228,22 @@ def _pad(self, align: int) -> None:
         if pad := (-self._pos) % align:
             self._write(b"\x00" * pad)
 
-    def _header(self, name: str, mode: int, filesize: int, *, mtime: int, block_align: bool) -> None:
+    def _header(
+        self,
+        name: str,
+        mode: int,
+        filesize: int,
+        *,
+        mtime: int,
+        block_align: bool,
+        ino: int | None = None,
+        nlink: int = 1,
+        major: int = 0,
+        minor: int = 0,
+    ) -> None:
         self._ino += 1
+        if name == TRAILER:
+            name = "./" + name
         raw = name.encode()
         namesize = len(raw) + 1  # name + its NUL terminator
         if block_align:
@@ -231,21 +252,75 @@ def _header(self, name: str, mode: int, filesize: int, *, mtime: int, block_alig
             padded = _roundup(self._pos + _HEADER + namesize, _BLOCK) - (self._pos + _HEADER)
             if padded <= _PATH_MAX:
                 namesize = padded
-        fields = (self._ino, mode, 0, 0, 1, min(mtime, self._epoch), filesize, 0, 0, 0, 0, namesize, 0)
+        fields = (
+            self._ino if ino is None else ino,
+            mode,
+            0,
+            0,
+            nlink,
+            min(mtime, self._epoch),
+            filesize,
+            0,
+            0,
+            major,
+            minor,
+            namesize,
+            0,
+        )
         self._write(MAGIC + b"".join(b"%08x" % v for v in fields))
         self._write(raw + b"\x00" * (namesize - len(raw)))
         self._pad(4)
 
+    def _link(self, link: tuple[int, int] | None) -> tuple[int | None, int, int]:
+        if link is None:
+            return None, 1, 1
+        token, nlink = link
+        ino, remaining = self._links.get(token, (self._ino + 1, nlink))
+        self._links[token] = (ino, remaining - 1)
+        return ino, nlink, remaining
+
     def add_dir(self, name: str, *, mode: int, mtime: int) -> None:
         self._header(name, stat.S_IFDIR | (mode & 0o7777), 0, mtime=mtime, block_align=False)
 
-    def add_symlink(self, name: str, target: str, *, mtime: int) -> None:
-        data = target.encode()
-        self._header(name, stat.S_IFLNK | 0o777, len(data), mtime=mtime, block_align=False)
+    def add_symlink(
+        self, name: str, target: str, *, mtime: int, link: tuple[int, int] | None = None
+    ) -> None:
+        ino, nlink, _ = self._link(link)
+        data = os.fsencode(target)
+        self._header(
+            name, stat.S_IFLNK | 0o777, len(data), mtime=mtime, block_align=False, ino=ino, nlink=nlink
+        )
         self._write(data)
         self._pad(4)
 
-    def add_file(self, name: str, src: Path, *, mode: int, mtime: int) -> None:
+    def add_node(
+        self,
+        name: str,
+        *,
+        mode: int,
+        mtime: int,
+        major: int = 0,
+        minor: int = 0,
+        link: tuple[int, int] | None = None,
+    ) -> None:
+        ino, nlink, _ = self._link(link)
+        self._header(
+            name, mode, 0, mtime=mtime, block_align=False, major=major, minor=minor, ino=ino, nlink=nlink
+        )
+
+    def add_file(
+        self,
+        name: str,
+        src: Path,
+        *,
+        mode: int,
+        mtime: int,
+        link: tuple[int, int] | None = None,
+    ) -> None:
+        ino, nlink, remaining = self._link(link)
+        if remaining > 1:
+            self._header(name, stat.S_IFREG | mode, 0, mtime=mtime, block_align=False, ino=ino, nlink=nlink)
+            return
         fd = os.open(src, os.O_RDONLY)
         try:
             size = os.fstat(fd).st_size
@@ -255,6 +330,8 @@ def add_file(self, name: str, src: Path, *, mode: int, mtime: int) -> None:
                 size,
                 mtime=mtime,
                 block_align=self._block_align and size >= _BLOCK,
+                ino=ino,
+                nlink=nlink,
             )
             util.copy_range(self._fd, self._pos, fd, 0, size)
             self._pos += size
@@ -274,31 +351,27 @@ def close(self) -> None:
         self._fd = -1
 
 
-def _add(w: Writer, tree: Path, path: Path) -> bool:
+def _add(w: Writer, tree: Path, path: Path, link: tuple[int, int] | None = None) -> bool:
     """Add one path under `tree`; false for a type an initramfs has no use for."""
     st = path.lstat()
     name = str(path.relative_to(tree))
     mode = stat.S_IMODE(st.st_mode)
-    mtime = int(st.st_mtime)
+    mtime = st.st_mtime_ns // 10**9
     if stat.S_ISDIR(st.st_mode):
         w.add_dir(name, mode=mode, mtime=mtime)
     elif stat.S_ISLNK(st.st_mode):
         w.add_symlink(name, os.readlink(path), mtime=mtime)
     elif stat.S_ISREG(st.st_mode):
-        w.add_file(name, path, mode=mode, mtime=mtime)
+        w.add_file(name, path, mode=mode, mtime=mtime, link=link)
     else:
         return False  # devices/fifos/sockets: unneeded in an initramfs (systemd mounts /dev)
     return True
 
 
-def pack_tree(tree: Path, out: Path, epoch: int, *, block_align: bool = True) -> int:
-    """Pack a whole tree into `out`."""
+def pack_tree(tree: Path, out: Path, epoch: int) -> int:
+    """Pack a whole tree into `out` without block alignment, which the kernel's early cpio reader needs."""
     tree = Path(tree)
-    count = 0
-    with Writer(out, epoch, block_align=block_align) as w:
-        for path in sorted(tree.rglob("*")):
-            count += _add(w, tree, path)
-    return count
+    return _pack_paths(tree, out, epoch, sorted(tree.rglob("*")), block_align=False)
 
 
 def pack_paths(tree: Path, out: Path, epoch: int, paths: Iterable[str | PurePath]) -> int:
@@ -307,8 +380,54 @@ def pack_paths(tree: Path, out: Path, epoch: int, paths: Iterable[str | PurePath
     Sorting is what puts a directory before its contents, which the kernel's unpacker needs.
     """
     tree = Path(tree)
+    return _pack_paths(
+        tree, out, epoch, [tree / rel for rel in sorted({PurePosixPath(path) for path in paths})]
+    )
+
+
+def _pack_paths(
+    tree: Path, out: Path, epoch: int, paths: Sequence[Path], *, block_align: bool = True
+) -> int:
+    inodes = {path: path.lstat() for path in paths}
+    counts = Counter((st.st_dev, st.st_ino) for st in inodes.values() if stat.S_ISREG(st.st_mode))
+    tokens: dict[tuple[int, int], int] = {}
     count = 0
-    with Writer(out, epoch) as w:
-        for rel in sorted({PurePosixPath(path) for path in paths}):
-            count += _add(w, tree, tree / rel)
+    with Writer(out, epoch, block_align=block_align) as w:
+        for path, st in inodes.items():
+            inode = st.st_dev, st.st_ino
+            link = None
+            if stat.S_ISREG(st.st_mode) and counts[inode] > 1:
+                link = (tokens.setdefault(inode, len(tokens) + 1), counts[inode])
+            count += _add(w, tree, path, link)
     return count
+
+
+def pack_manifest(
+    objects: Sequence[uapi16.Object],
+    content: Callable[[uapi16.Object], Path],
+    out: Path,
+    epoch: int,
+) -> None:
+    counts = Counter(obj.get("inodeToken") for obj in objects if obj.get("inodeToken"))
+    kinds = {"fifo": stat.S_IFIFO, "chr": stat.S_IFCHR, "blk": stat.S_IFBLK, "sock": stat.S_IFSOCK}
+    with Writer(out, epoch) as w:
+        for obj in objects:
+            kind = obj["type"]
+            name, mode, mtime = obj.get("name", "."), obj.get("mode", 0o755), obj["mTimeNSec"] // 10**9
+            token = obj.get("inodeToken")
+            link = (token, counts[token]) if token else None
+            if kind == "reg":
+                w.add_file(name, content(obj), mode=mode, mtime=mtime, link=link)
+            elif kind == "dir":
+                w.add_dir(name, mode=mode, mtime=mtime)
+            elif kind == "lnk":
+                w.add_symlink(name, uapi16.symlink_target(obj), mtime=mtime, link=link)
+            else:
+                w.add_node(
+                    name,
+                    mode=kinds[kind] | mode,
+                    mtime=mtime,
+                    major=obj.get("major", 0),
+                    minor=obj.get("minor", 0),
+                    link=link,
+                )

archive/cpio_test.py

@@ -28,6 +28,19 @@ def payload(data: bytes) -> bytes:
 
 
 class TestUnpack(unittest.TestCase):
+    def test_symlink_targets_need_not_be_utf8(self) -> None:
+        # A symlink target is an arbitrary byte string even when its filename is valid UTF-8.
+        # Packing and extracting the target must preserve its bytes without a Unicode error.
+        with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch:
+            root = Path(scratch)
+            archive = root / "image.cpio"
+            target = b"target-\xff"
+            with cpio.Writer(archive, 0) as writer:
+                writer.add_symlink("link", os.fsdecode(target), mtime=0)
+            with archive.open("rb") as stream:
+                cpio.unpack(stream.fileno(), root / "unpacked")
+            self.assertEqual(os.fsencode(os.readlink(root / "unpacked/link")), target)
+
     def test_extracts_an_ordinary_hardlink_set(self) -> None:
         with tempfile.TemporaryDirectory() as scratch:
             root = Path(scratch)
@@ -133,5 +146,58 @@ def test_an_unaligned_archive_keeps_the_stock_name_field(self) -> None:
                     self.assertEqual(int(header[6 + 11 * 8 : 6 + 12 * 8], 16), namesize)
 
 
+class TestPackPaths(unittest.TestCase):
+    def test_a_literal_trailer_name_does_not_end_the_archive(self) -> None:
+        # A regular file named TRAILER!!! must be escaped with a relative prefix so the reader
+        # extracts it and the later file instead of treating it as the archive terminator.
+        with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch:
+            root = Path(scratch)
+            tree = root / "tree"
+            tree.mkdir()
+            (tree / "TRAILER!!!").write_text("ordinary file")
+            (tree / "z-last").write_text("later file")
+            out = root / "image.cpio"
+            cpio.pack_tree(tree, out, 0)
+            self.assertEqual([obj.name for obj in cpio.read(out.read_bytes())], ["./TRAILER!!!", "z-last"])
+            with out.open("rb") as stream:
+                cpio.unpack(stream.fileno(), root / "unpacked")
+            self.assertEqual((root / "unpacked/TRAILER!!!").read_text(), "ordinary file")
+            self.assertEqual((root / "unpacked/z-last").read_text(), "later file")
+
+    def test_hardlinks_depend_on_selected_paths(self) -> None:
+        # A full selection preserves explicit links without merging equal files. Selecting one
+        # member of a group must emit a self-contained file with nlink=1 and its own payload.
+        with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch:
+            root = Path(scratch)
+            tree = root / "tree"
+            tree.mkdir()
+            (tree / "a").write_text("same")
+            os.link(tree / "a", tree / "b")
+            (tree / "c").write_text("same")
+            out = root / "image.cpio"
+            cpio.pack_tree(tree, out, 0)
+            objects = {obj.name: obj for obj in cpio.read(out.read_bytes())}
+            self.assertEqual(objects["a"].ino, objects["b"].ino)
+            self.assertEqual(objects["a"].nlink, 2)
+            self.assertNotEqual(objects["a"].ino, objects["c"].ino)
+            cpio.pack_paths(tree, out, 0, ["a"])
+            (obj,) = cpio.read(out.read_bytes())
+            self.assertEqual(obj.nlink, 1)
+            self.assertEqual(obj.size, 4)
+
+    def test_timestamp_truncation_does_not_round_up(self) -> None:
+        # A nanosecond below a second boundary must truncate to the preceding second, even when
+        # the floating-point stat timestamp rounds up to the boundary.
+        with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch:
+            root = Path(scratch)
+            (root / "file").write_text("contents")
+            cutoff = 1000000000
+            os.utime(root / "file", ns=(0, cutoff * 10**9 - 1))
+            out = root / "image.cpio"
+            cpio.pack_paths(root, out, cutoff, ["file"])
+            header = out.read_bytes()[: cpio._HEADER]
+            self.assertEqual(int(header[6 + 5 * 8 : 6 + 6 * 8], 16), cutoff - 1)
+
+
 if __name__ == "__main__":
     unittest.main()