diff --git a/docs/design/architecture.md b/docs/design/architecture.md index fb6680e6..c7a4e71c 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -915,7 +915,8 @@ Both pinned files keep the path the mirror serves them under, such as `dists/tes `dists/testing/main/binary-amd64/Packages.xz`. That layout is what the signed `InRelease` is held to: - It has to say it is the suite its directory names, by `Suite` or `Codename`. An archive signs every - suite with the same keys, so the signature alone would accept the `InRelease` of another suite. + suite with the same keys, so the signature alone would accept the `InRelease` of another suite. The + only exception is a flat repository, where there is only one `InRelease` to verify anyway. - It has to state the index at the path the index sits at. Every index of a suite is signed, so matching the bytes alone would accept the index of another component. diff --git a/package_system/deb/deb822.py b/package_system/deb/deb822.py index 51e4b898..82a5b848 100644 --- a/package_system/deb/deb822.py +++ b/package_system/deb/deb822.py @@ -33,16 +33,26 @@ def package_index(directory: Path, what: str) -> Path: The index is at the path that the mirror serves it under. A mirror serves one index in several compressions, but a materialized repository must hold exactly one file. """ - served = f"{SUITES}/*/{index_directory('*', '*')}" - found = [path for name in INDEX_NAMES for path in directory.glob(f"{served}/{name}")] + if (directory / SUITES).is_dir(): + served = f"{SUITES}/*/{index_directory('*', '*')}/" + found = [path for name in INDEX_NAMES for path in directory.glob(f"{served}/{name}")] + else: + found = [path for name in INDEX_NAMES for path in directory.glob(name)] if len(found) != 1: util.fail(f"{what}: expected exactly one Packages stream in {directory}, found {len(found)}") return found[0] def suite_directory(index: Path) -> Path: - """Return the suite directory of an index that `package_index()` found.""" - return index.parents[2] + """Return the suite directory of an index that `package_index()` found. + + For a flat repository, returns the repo root. + """ + if index.parents[3].name == SUITES: + return index.parents[2] + else: + # If there is no "dists", assume the repository is flat. + return index.parents[0] def stanzas(source: Iterable[str]) -> Iterator[dict[str, str]]: diff --git a/package_system/deb/snapshot.py b/package_system/deb/snapshot.py index 3eb3e27c..4689484e 100644 --- a/package_system/deb/snapshot.py +++ b/package_system/deb/snapshot.py @@ -19,7 +19,6 @@ """ import io -import sys import tempfile from collections.abc import Mapping from pathlib import Path @@ -43,7 +42,7 @@ def read_release(rid: str, base: str, suite: str) -> tuple[MetadataFile, dict[st url = f"{base}/{release.INRELEASE}" with io.BytesIO() as raw: sha256, size = snapshotter.download(rid, release.INRELEASE, url, raw) - out = f"{deb822.SUITES}/{suite}/{release.INRELEASE}" + out = f"{deb822.SUITES}/{suite}/{release.INRELEASE}" if suite else release.INRELEASE signed = MetadataFile(out=out, url=url, sha256=sha256, size=size) with io.BytesIO() as raw: snapshotter.download(rid, release.RELEASE, f"{base}/{release.RELEASE}", raw) @@ -81,16 +80,16 @@ def package_index( index. The URL uses the checksum path whenever the Release offers it. """ stated = release.stated(rid, stanza) - directory = deb822.index_directory(component, arch) - name = next((name for name in deb822.INDEX_NAMES if f"{directory}/{name}" in stated), None) + directory = f"{deb822.index_directory(component, arch)}/" if suite else "" + name = next((name for name in deb822.INDEX_NAMES if f"{directory}{name}" in stated), None) if name is None: - util.fail(f"{rid}: Release states no {directory}/Packages index") - sha256, size = stated[f"{directory}/{name}"] + util.fail(f"{rid}: Release states no {directory}Packages index") + sha256, size = stated[f"{directory}{name}"] if size <= 0: - util.fail(f"{rid}: Release states an empty {directory}/{name}") + util.fail(f"{rid}: Release states an empty {directory}{name}") by_hash = stanza.get("acquire-by-hash") == "yes" - path = f"{directory}/by-hash/SHA256/{sha256}" if by_hash else f"{directory}/{name}" - out = f"{deb822.SUITES}/{suite}/{directory}/{name}" + path = f"{directory}by-hash/SHA256/{sha256}" if by_hash else f"{directory}{name}" + out = f"{deb822.SUITES}/{suite}/{directory}{name}" if suite else name return MetadataFile(out=out, url=f"{base}/{path}", sha256=sha256, size=size) @@ -107,19 +106,32 @@ def inventory(rid: str, index: Path) -> dict[str, PackageEntry]: return packages +def _repository_base(rid: str, baseurl: str, suite: str) -> tuple[str, str]: + if suite.endswith("/"): + # A suite ending in '/' is a path to a flat repo, not a suite. + suite_rstripped = suite.rstrip("/") + if suite_rstripped: + relative_href(rid, "suite", suite_rstripped) + return (f"{baseurl}/{suite_rstripped}", "") + else: + return (baseurl, "") + else: + # A standard repository stores the suite in `dists/`, so the name must be a single + # path component. + if "/" in relative_href(rid, "suite", suite): + util.fail(f"{rid}: a suite in a directory below another is not supported: {suite!r}") + return (f"{baseurl}/{deb822.SUITES}/{suite}", suite) + + def snapshot_repository(spec: Spec) -> Mapping[str, object]: """Pin the index of one component of a suite and the packages that the index describes.""" - rid, suite, component, arch = spec["id"], spec["suite"], spec["component"], spec["arch"] - print(f"{rid}: snapshotting {suite}/{component}/binary-{arch}…", file=sys.stderr) - # A materialized repository stores the suite in `dists/`, so the name must be a single - # path component. - if "/" in relative_href(rid, "suite", suite): - util.fail(f"{rid}: a suite in a directory below another is not supported: {suite!r}") - base = f"{spec['baseurl'].rstrip('/')}/{deb822.SUITES}/{suite}" + rid, component, arch = spec["id"], spec["component"], spec["arch"] + base, suite = _repository_base(rid, spec["baseurl"].rstrip("/"), spec["suite"]) signed, stanza = read_release(rid, base, suite) - release.named(rid, stanza, suite) - _indexed_together(rid, stanza, _offered(rid, stanza, "Architectures", arch)) - _offered(rid, stanza, "Components", component) + if suite: + release.named(rid, stanza, suite) + _indexed_together(rid, stanza, _offered(rid, stanza, "Architectures", arch)) + _offered(rid, stanza, "Components", component) stream = package_index(rid, stanza, base, suite, component, arch) with tempfile.TemporaryDirectory() as scratch: diff --git a/package_system/deb/snapshot_test.py b/package_system/deb/snapshot_test.py index 443edbe6..a0a501fb 100644 --- a/package_system/deb/snapshot_test.py +++ b/package_system/deb/snapshot_test.py @@ -13,7 +13,8 @@ import snapshot -BASE = "https://snapshot.invalid/archive/debian/20260701T000000Z/dists/trixie" +BASEURL = "https://snapshot.invalid/archive/debian/20260701T000000Z" +BASE = f"{BASEURL}/dists/trixie" XZ = "3ab4e811cf4f3e5a335d382c58cc19d85f1abe7a4ef4689160ca1f637fa0e9b3" PLAIN = "be70297f6ea499e8ef0bd93906719298c795bc3dc4ce72560f13e6c9836bfedb" DEB = "03571d298ed24c615641d54dec4eb023ef5710a2b8cb9304acb66e372cab9c46" @@ -33,6 +34,17 @@ def release(*, by_hash: str = "yes", entries: str | None = None) -> dict[str, st } +def release_flat(*, by_hash: str = "yes", entries: str | None = None) -> dict[str, str]: + stated = entries + if stated is None: + stated = f"\n{PLAIN} 65651288 Packages\n{XZ} 9672648 Packages.xz" + return { + "architectures": "all amd64 arm64", + "acquire-by-hash": by_hash, + "sha256": stated, + } + + def packages_record(*, name: str = "bash", digest: str = DEB, filename: str | None = None) -> str: location = filename or f"pool/main/b/{name}/{name}_5.2.37-2_amd64.deb" return ( @@ -82,6 +94,30 @@ def test_rejects_a_malformed_or_escaping_release_entry(self) -> None: with self.subTest(entries=entries), self.assertRaises(SystemExit): snapshot.package_index("main", release(entries=entries), BASE, "trixie", "main", "amd64") + def test_tolerates_flat_release_with_no_suite(self) -> None: + stream = snapshot.package_index("main", release_flat(), BASE, "", "", "amd64") + self.assertEqual(stream["out"], "Packages.xz") + self.assertEqual(stream["sha256"], XZ) + self.assertEqual(stream["size"], 9672648) + self.assertEqual(stream["url"], f"{BASE}/by-hash/SHA256/{XZ}") + + +class TestRepositoryBase(unittest.TestCase): + def test_standard_repo_preserves_suite(self) -> None: + base, suite = snapshot._repository_base("main", BASEURL, "trixie") + self.assertEqual(base, f"{BASEURL}/dists/trixie") + self.assertEqual(suite, "trixie") + + def test_flat_repo_removes_suite(self) -> None: + base, suite = snapshot._repository_base("main", BASEURL, "not/really/a/suite/") + self.assertEqual(base, f"{BASEURL}/not/really/a/suite") + self.assertEqual(suite, "") + + def test_flat_repo_tolerates_slash_only(self) -> None: + base, suite = snapshot._repository_base("main", BASEURL, "/") + self.assertEqual(base, BASEURL) + self.assertEqual(suite, "") + class TestIndexedTogether(unittest.TestCase): def test_refuses_a_suite_that_indexes_architecture_all_apart(self) -> None: diff --git a/package_system/deb/verify.py b/package_system/deb/verify.py index 39b7098d..41ae8b4a 100755 --- a/package_system/deb/verify.py +++ b/package_system/deb/verify.py @@ -148,7 +148,8 @@ def vouched_packages( # sqv refuses to overwrite a file, so each generation gets its own output file. message = authenticated(keys, suite / release.INRELEASE, scratch / generation.name, at) stanza = release.stanza(release.INRELEASE, message) - release.named(release.INRELEASE, stanza, suite.name) + if suite.parents[0].name == deb822.SUITES: + release.named(release.INRELEASE, stanza, suite.name) unexpired(release.INRELEASE, stanza, at) vouched(release.stated(release.INRELEASE, stanza), suite, index) return {package.sha256: package.size for package in deb822.packages(index, release.INRELEASE)} diff --git a/package_system/deb/verify_test.py b/package_system/deb/verify_test.py index 91755f42..c0aea8b6 100644 --- a/package_system/deb/verify_test.py +++ b/package_system/deb/verify_test.py @@ -141,6 +141,30 @@ def repository( (directory / SUITE / release.INRELEASE).write_bytes(signed) return directory + def flat_repository( + self, + *records: str, + signers: tuple[str, ...] | None = None, + date: str = DATED, + valid_until: str | None = None, + signed_at: str = SIGNED, + ) -> Path: + """Write a pinned repository like repository(), but with a flat layout.""" + directory = Path(tempfile.mkdtemp(dir=self._scratch.name)) / "repo" + directory.mkdir() + index = "".join(records).encode() + (directory / "Packages").write_bytes(index) + expires = f"Valid-Until: {valid_until}\n" if valid_until is not None else "" + text = ( + f"Origin: tine\nComponent: deb\nDate: {date}\n{expires}" + "Architectures: amd64\nSHA256:\n" + f" {hashlib.sha256(index).hexdigest()} {len(index)} Packages\n" + ) + signers = signers if signers is not None else (self.archive.key,) + signed = self.archive.clearsign(text, *signers, at=signed_at) if signers else text.encode() + (directory / release.INRELEASE).write_bytes(signed) + return directory + def retain(self, repository: Path, older: Path, pinned_at: str | None = None) -> None: """Move the repository `older` into `repository` as a retained generation.""" retained = repository / snapshotter.RETAINED @@ -208,8 +232,10 @@ def test_refuses_a_time_it_cannot_read(self) -> None: def test_publishes_a_package_the_signed_release_vouches_for(self) -> None: package, record = self.package("good") - out = self.verify(self.declared(self.archive.key), self.repository(record), package) - self.assertEqual((out / f"pkg--{package.name}").read_bytes(), package.read_bytes()) + for repository in [self.repository(record), self.flat_repository(record)]: + with self.subTest(repository=repository): + out = self.verify(self.declared(self.archive.key), repository, package) + self.assertEqual((out / f"pkg--{package.name}").read_bytes(), package.read_bytes()) def test_one_declared_key_among_the_signers_suffices(self) -> None: # The archive signs with its current key and with the previous key, so a repository can