diff --git a/src/agent_comms/build_artifact.py b/src/agent_comms/build_artifact.py new file mode 100644 index 0000000000..b5c9d72bc9 --- /dev/null +++ b/src/agent_comms/build_artifact.py @@ -0,0 +1,46 @@ +"""Stable file-content observations shared by interpreter and wheel build inputs.""" + +from __future__ import annotations + +import hashlib +from io import BufferedReader +from pathlib import Path +from typing import Annotated + +from pydantic import BaseModel, ConfigDict, field_validator + +from agent_comms.nominal.field_codec import Sha256Text +from agent_comms.private_path import FileRevision + + +class BuildArtifact(BaseModel): + """A native file identity and its bytes, captured through the same open handle.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + path: Path + revision: FileRevision + sha256: Annotated[str, Sha256Text] + + @field_validator("sha256") + @classmethod + def valid_digest(cls, value: str) -> str: + return Sha256Text.decode(value) + + @classmethod + def capture(cls, path: Path) -> BuildArtifact: + path = path.resolve(strict=True) + with path.open("rb") as stream: + return cls.from_stream(stream) + + @classmethod + def from_stream(cls, stream: BufferedReader) -> BuildArtifact: + """Capture the already-open file, including a platform-qualified native image.""" + path = Path(stream.name).resolve(strict=True) + stream.seek(0) + revision = FileRevision.from_file(stream) + digest = hashlib.file_digest(stream, "sha256").hexdigest() + if revision != FileRevision.from_file(stream): + raise ValueError(f"Build artifact changed while reading: {path}") + if FileRevision.capture(path) != revision: + raise ValueError(f"Build artifact was replaced while reading: {path}") + return cls(path=path, revision=revision, sha256=digest) diff --git a/src/agent_comms/build_identity.py b/src/agent_comms/build_identity.py index 5c86b7c150..c0aae98668 100644 --- a/src/agent_comms/build_identity.py +++ b/src/agent_comms/build_identity.py @@ -1,16 +1,17 @@ -"""Installed repository commits, shared by delivery proofs and close checks.""" +"""Installed repository and interpreter identity, shared by proofs and close checks.""" from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from collections.abc import Iterable from typing import TYPE_CHECKING, Annotated, ClassVar from agent_comms.nominal.declared_family import DeclaredFamily -from agent_comms.nominal.field_codec import GitCommitText +from agent_comms.nominal.field_codec import GitCommitText, WireOmitDefault from agent_comms.installer.runtime_resolution import RuntimePackage, RuntimeResolution +from agent_comms.installer.probes.interpreter import InterpreterBuild, RecordedInterpreterBuild from agent_comms.launch_runtime import LaunchRuntime from agent_comms.repo import Repo from agent_comms.repository_name import RepositoryName @@ -21,7 +22,7 @@ class ProofBuildMismatchError(ValueError): - """A retained proof does not identify the currently installed repositories.""" + """A retained proof does not identify the installed repositories and interpreter.""" class DeliveredCommitNotInstalledError(ValueError): @@ -137,7 +138,7 @@ def installation(self, commits: tuple[str, ...], directory: Path) -> CommitInsta @dataclass(frozen=True) class UnrecordedBuildIdentity(BuildIdentity): def recorded(self) -> RecordedBuildIdentity: - return RecordedBuildIdentity.installed() + return RecordedBuildIdentity.installed().recorded() def installation(self, commits: tuple[str, ...], directory: Path) -> CommitInstallation: installed = RecordedBuildIdentity.installed() @@ -148,28 +149,43 @@ def installation(self, commits: tuple[str, ...], directory: Path) -> CommitInsta @dataclass(frozen=True) class RecordedBuildIdentity(BuildIdentity, InstalledBuild): - """Declared runtime source revisions, each identified by its repository.""" + """Repository revisions and the interpreter observation owned by their runtime.""" commits: tuple[RepositoryCommit, ...] + interpreter: Annotated[ + RecordedInterpreterBuild | InterpreterBuild, WireOmitDefault() + ] = field(default_factory=InterpreterBuild) def __str__(self) -> str: - return ", ".join( - f"{record.repository.to_wire()} {record.commit}" for record in self.commits - ) + return ", ".join(( + *(f"{record.repository.to_wire()} {record.commit}" for record in self.commits), + str(self.interpreter), + )) def recorded(self) -> RecordedBuildIdentity: + try: + self.interpreter.require_recorded() + except ValueError as error: + raise ProofBuildMismatchError( + f"Proof build {self} has no interpreter observation; run and record a new proof." + ) from error return self @classmethod def installed(cls) -> RecordedBuildIdentity: - runtime = LaunchRuntime.installed() + return cls.at(LaunchRuntime.installed()) + + @classmethod + def at(cls, runtime: LaunchRuntime) -> RecordedBuildIdentity: + """Read the selected build, never borrow the current pointer for a candidate.""" runtime.require_available() resolution = RuntimeResolution.at(runtime.prefix) return cls( tuple( RepositoryCommit(member.repository, GitCommitText.decode(member.commit(resolution))) for member in RuntimePackage.members_with(RuntimePackage) - ) + ), + interpreter=resolution.interpreter, ) def recorded_in(self, repos: Iterable[Repo]) -> tuple[tuple[Repo, RepositoryCommit], ...]: @@ -201,7 +217,8 @@ def contains(self, commit: str, directory: Path) -> bool: return Repo.descends(directory, commit, recorded.commit) def installation(self, commits: tuple[str, ...], directory: Path) -> CommitInstallation: - installed = self.installed() + self.recorded() + installed = self.installed().recorded() if self != installed: raise ProofBuildMismatchError( f"Proof build {self} differs from installed {installed}; run the proof again." diff --git a/src/agent_comms/child_process.py b/src/agent_comms/child_process.py index 3c5fcd61ca..3f8afdbb9c 100644 --- a/src/agent_comms/child_process.py +++ b/src/agent_comms/child_process.py @@ -37,9 +37,10 @@ from contextvars import copy_context from dataclasses import dataclass from functools import cache, cached_property, partial +from io import BufferedReader from pathlib import Path from tempfile import TemporaryDirectory -from typing import IO, TYPE_CHECKING, Annotated, Any, ClassVar, NoReturn, Self +from typing import IO, TYPE_CHECKING, Annotated, Any, BinaryIO, ClassVar, NoReturn, Self from .nominal.declared_family import DeclaredFamily from .errors import RelationViolationError @@ -529,6 +530,28 @@ def sync_directory(self, path: Path) -> None: finally: os.close(descriptor) + class NativeImage(ctypes.Structure): + """The POSIX loader's native symbol-image ABI.""" + + _fields_ = ( + ("filename", ctypes.c_char_p), + ("base", ctypes.c_void_p), + ("symbol", ctypes.c_char_p), + ("address", ctypes.c_void_p), + ) + + @contextmanager + def native_image(self, address: ctypes.c_void_p) -> Generator[BufferedReader]: + """Open the native file providing a symbol; the caller consumes this handle.""" + lookup = ctypes.CDLL(None).dladdr + lookup.argtypes = (ctypes.c_void_p, ctypes.POINTER(self.NativeImage)) + lookup.restype = ctypes.c_int + image = self.NativeImage() + if not lookup(address, ctypes.byref(image)) or not image.filename: + raise ValueError("The loader did not identify the native symbol image") + with Path(os.fsdecode(image.filename)).resolve(strict=True).open("rb") as stream: + yield stream + @abstractmethod def require_watched_procfs(self, refusal: Exception) -> None: ... @@ -657,6 +680,30 @@ class LinuxProcessRecord: class LinuxPlatform(PidfdHandles, Platform): + @contextmanager + def native_image(self, address: ctypes.c_void_p) -> Generator[BufferedReader]: + # glibc's main-image filename aliases argv[0]; setproctitle changes it. + # Resolve the actual symbol mapping instead, including shared libpython. + # ctypes represents C null as None; kernel mapping addresses are integers. + pointer = address.value or 0 + if pointer == 0: + raise ValueError("A null address has no native image") + with Path("/proc/self/maps").open() as mappings: + for line in mappings: + extent, _permissions, _offset, device, inode, *_name = line.split(maxsplit=5) + first, stop = (int(value, 16) for value in extent.split("-")) + if first <= pointer < stop: + link = Path("/proc/self/map_files") / f"{first:x}-{stop:x}" + with link.resolve(strict=True).open("rb") as stream: + observed = os.fstat(stream.fileno()) + major, minor = (int(value, 16) for value in device.split(":")) + # Decode kernel metadata once at this native boundary. + if (observed.st_dev, observed.st_ino) != (os.makedev(major, minor), int(inode)): + raise ValueError("The native image file no longer matches its mapping") + yield stream + return + raise ValueError("The kernel did not identify the native symbol image") + def require_watched_procfs(self, refusal: Exception) -> None: return None diff --git a/src/agent_comms/cli_verbs/prepare_runtime.py b/src/agent_comms/cli_verbs/prepare_runtime.py new file mode 100644 index 0000000000..82fefd1bbe --- /dev/null +++ b/src/agent_comms/cli_verbs/prepare_runtime.py @@ -0,0 +1,54 @@ +"""Prepare a functional candidate through the existing runtime/preparation owners.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated + +from agent_comms.cli_command import CliCommand +from agent_comms.cli_result import CliResult +from agent_comms.comms import Comms +from agent_comms.installer.commands import InstallCommands +from agent_comms.installer.runtime_stack import WheelRuntime +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.nominal.field_codec import Argument, FieldCodec, Normalize, PathText +from agent_comms.shell_effects import BuildStorageEffect, WriteStorageEffect +from agent_comms.storage_projection import StorageProjection +from agent_comms.tool_caller import ThreadCaller + + +@dataclass(frozen=True) +class PreparedRuntimeResult(CliResult): + prefix: Annotated[Path, PathText] + + +@dataclass(frozen=True, kw_only=True) +class PrepareRuntimeCliCommand(CliCommand, declared_name="prepare-runtime"): + help = ("Prepare a NEW owned functional target from an explicit interpreter, " + "recorded dependency runtime and exact local wheels; never publish it") + caller: Annotated[ThreadCaller, Normalize(FieldCodec.encode), + Argument(flags=("--from",), options={"type": ThreadCaller})] + into: Annotated[Path, PathText, Argument(flags=("--into",))] + interpreter: Annotated[Path, PathText, Argument(flags=("--python",))] + source_runtime: Annotated[Path, PathText, Argument(flags=("--source-runtime",))] + checkout: Annotated[Path, PathText, Argument(flags=("--checkout",))] + wheels: Annotated[tuple[Annotated[Path, PathText], ...], + Argument(flags=("--wheels",), options={"nargs": "+"})] + + def apply(self, ctx: Comms) -> CliResult: + thread = ctx.registry.require(self.caller.own_thread()) + prefix = self.into.expanduser().absolute() + checkout = self.checkout.expanduser().resolve() + projection = StorageProjection(ctx.registry.storage_rule, thread, ctx) + projection.check((BuildStorageEffect(), WriteStorageEffect(prefix), + WriteStorageEffect(checkout))) + with InstallCommands() as commands: + target = WheelRuntime( + checkout=checkout, commands=commands, + runtime=LaunchRuntime.at_prefix(prefix), + interpreter=LaunchRuntime.from_interpreter(self.interpreter.expanduser().absolute()), + source_runtime=LaunchRuntime.at_prefix(self.source_runtime.expanduser().resolve()), + wheels=tuple(path.expanduser().resolve(strict=True) for path in self.wheels), + ) + target.prepare() + return PreparedRuntimeResult(prefix) diff --git a/src/agent_comms/installer/probes/interpreter.py b/src/agent_comms/installer/probes/interpreter.py new file mode 100644 index 0000000000..bedc64a414 --- /dev/null +++ b/src/agent_comms/installer/probes/interpreter.py @@ -0,0 +1,70 @@ +"""The selected interpreter's observed native build, not its repository ancestry.""" + +from __future__ import annotations + +import ctypes +from pathlib import Path +import sys +import sysconfig + +from pydantic import BaseModel, ConfigDict, Field + +from agent_comms.installer.interpreter_operation import RuntimeProbe, TargetProbe +from agent_comms.build_artifact import BuildArtifact +from agent_comms.child_process import Platform + + +class InterpreterBuild(BaseModel): + """An existing build record without an interpreter observation is unqualified.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + def __str__(self) -> str: + return "unrecorded interpreter" + + def matches(self, actual: RecordedInterpreterBuild) -> bool: + """Absent evidence cannot make a cached runtime eligible for reuse.""" + return False + + def require_recorded(self) -> RecordedInterpreterBuild: + raise ValueError("The runtime has no recorded interpreter build") + + +class RecordedInterpreterBuild(InterpreterBuild): + """Native artifact identity, with the invoked environment kept separately.""" + + invoked: Path + version: str = Field(min_length=1) + abi: str = Field(min_length=1) + stdlib: Path + artifacts: tuple[BuildArtifact, ...] = Field(min_length=1) + + def __str__(self) -> str: + images = ", ".join(f"{artifact.path} sha256={artifact.sha256}" + for artifact in self.artifacts) + return f"interpreter {self.invoked} [{self.abi}]: {images}" + + def matches(self, actual: RecordedInterpreterBuild) -> bool: + return self == actual + + def require_recorded(self) -> RecordedInterpreterBuild: + return self + + +class InterpreterBuildProbe(TargetProbe, RuntimeProbe[RecordedInterpreterBuild]): + def run(self) -> RecordedInterpreterBuild: + abi = sysconfig.get_config_var("SOABI") + if not isinstance(abi, str) or not abi: + raise ValueError("The selected interpreter does not declare its native ABI") + address = ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p) + with Platform.current().native_image(address) as stream: + native = BuildArtifact.from_stream(stream) + executable = Path(sys.executable).resolve(strict=True) + images = {native.path: native} + if executable not in images: + images[executable] = BuildArtifact.capture(executable) + return RecordedInterpreterBuild( + invoked=Path(sys.executable), version=sys.version, abi=abi, + stdlib=Path(sysconfig.get_path("stdlib")).resolve(strict=True), + artifacts=tuple(images[path] for path in dict.fromkeys((executable, native.path))), + ) diff --git a/src/agent_comms/installer/probes/packages.py b/src/agent_comms/installer/probes/packages.py index f924c841d8..888a48434b 100644 --- a/src/agent_comms/installer/probes/packages.py +++ b/src/agent_comms/installer/probes/packages.py @@ -2,20 +2,23 @@ from __future__ import annotations -import json -import sys +import hashlib from dataclasses import dataclass -from importlib.metadata import distributions -from pathlib import Path -from typing import Self - -from pydantic import BaseModel, JsonValue, RootModel +from importlib.metadata import Distribution, distributions +from pathlib import Path, PurePosixPath +from urllib.parse import unquote, urlsplit +from zipfile import ZipExtFile, ZipFile +from typing import ClassVar, Self + +from packaging.utils import canonicalize_name +from pydantic import BaseModel, Field, JsonValue, RootModel from pydantic.json_schema import DEFAULT_REF_TEMPLATE -import agent_comms -from agent_comms.nominal.field_codec import WireValue +from agent_comms.build_artifact import BuildArtifact +from agent_comms.nominal.field_codec import Sha256Text, WireValue +from agent_comms.private_path import FileRevision +from agent_comms.launch_runtime import LaunchRuntime from agent_comms.installer.interpreter_operation import RuntimeProbe, TargetProbe -from agent_comms.package_stamp import STAMP from agent_comms.runtime_preparation import RuntimePreparation @@ -42,22 +45,111 @@ class GitPin(PackagePin): @dataclass(frozen=True) class ArchivePin(PackagePin): url: str + hashes: tuple[tuple[str, str], ...] = () class CommitRecord(BaseModel): commit_id: str -class SourceRecord(BaseModel): +class WheelSource: + """Only an archive source can prove installed bytes against a supplied wheel.""" + + def verified_wheel( + self, wheel: Path, distribution: Distribution, runtime: LaunchRuntime + ) -> ArchiveRecord: + raise ValueError("A supplied wheel requires an archive installation origin") + + def wheel_inputs(self) -> tuple[Path, ...]: + return () + + +class SourceRecord(BaseModel, WheelSource): url: str def revision(self) -> str: raise ValueError("The installed package does not record a Git revision") +class ArchiveInfo(BaseModel): + """PEP610's algorithm-to-digest map; an empty map supplies no content proof.""" + + hashes: dict[str, str] = Field(default_factory=dict) + + def verified(self, artifact: BuildArtifact) -> Self: + """A missing PEP610 hash supplies no constraint; an existing one must match.""" + if self.hashes.get("sha256", artifact.sha256) != artifact.sha256: + raise ValueError(f"Archive content differs from its recorded hash: {artifact.path}") + verified = self.model_copy() + verified.hashes = {**self.hashes, "sha256": artifact.sha256} + return verified + class ArchiveRecord(SourceRecord): + """PEP610 archive metadata and its local-wheel URL boundary.""" + + archive_info: ArchiveInfo = Field(default_factory=ArchiveInfo) + file_scheme: ClassVar[str] = "file" + local_hosts: ClassVar[tuple[str, ...]] = ("", "localhost") + wheel_suffix: ClassVar[str] = ".whl" + def pin(self, version: str) -> PackagePin: - return ArchivePin(version, self.url) + return ArchivePin(version, self.url, tuple(sorted(self.archive_info.hashes.items()))) + + def wheel_inputs(self) -> tuple[Path, ...]: + if "sha256" not in self.archive_info.hashes: + return () + Sha256Text.decode(self.archive_info.hashes["sha256"]) + source = urlsplit(self.url) + # A remote archive retains its PEP610 hash without fetching its URL. + if (source.scheme != self.file_scheme + or Path(source.path).suffix != self.wheel_suffix): + return () + if source.netloc not in self.local_hosts: + raise ValueError("Runtime wheel input names a foreign file host") + wheel = Path(unquote(source.path)) + if not wheel.is_absolute(): + raise ValueError("Verified runtime archive is not an absolute wheel input") + return (wheel,) + + def verified_wheel( + self, wheel: Path, distribution: Distribution, runtime: LaunchRuntime + ) -> ArchiveRecord: + artifact = BuildArtifact.capture(wheel) + if self.url != artifact.path.as_uri(): + raise ValueError(f"Installed archive origin differs from supplied wheel: {wheel}") + with ZipFile(artifact.path) as archive: + entries = archive.infolist() + names = tuple(entry.filename for entry in entries) + records = tuple(name for name in names if name.endswith(".dist-info/RECORD")) + if len(set(names)) != len(names) or len(records) != 1: + raise ValueError("A runtime wheel requires unique entries and one RECORD") + (record,) = records + for entry in entries: + relative = PurePosixPath(entry.filename) + if (relative.is_absolute() or ".." in relative.parts + or relative.as_posix() != entry.filename.rstrip("/")): + raise ValueError(f"Unsafe wheel member: {entry.filename}") + if relative.parts[0].endswith(".data"): + raise ValueError("Exact runtime inputs require root-layout wheels") + if entry.is_dir() or entry.filename == record: + continue + location = distribution.locate_file(entry.filename) + if not isinstance(location, Path): + raise ValueError(f"Installed wheel member is not a filesystem path: {location}") + installed = location.resolve(strict=True) + if not installed.is_relative_to(runtime.prefix): + raise ValueError(f"Installed wheel member escapes its runtime: {installed}") + with archive.open(entry) as source: + if not isinstance(source, ZipExtFile): + raise ValueError("Wheel member is not a readable ZIP entry") + expected = hashlib.file_digest(source, "sha256").hexdigest() + if BuildArtifact.capture(installed).sha256 != expected: + raise ValueError(f"Installed bytes differ from supplied wheel: {installed}") + if FileRevision.capture(artifact.path) != artifact.revision: + raise ValueError(f"Supplied wheel changed during verification: {wheel}") + verified = self.model_copy() + verified.archive_info = self.archive_info.verified(artifact) + return verified class GitRecord(SourceRecord): @@ -70,7 +162,7 @@ def pin(self, version: str) -> PackagePin: return GitPin(version, self.vcs_info.commit_id) -class RegistryRecord(RootModel[None]): +class RegistryRecord(RootModel[None], WheelSource): """null is the build record's declared registry-package representation.""" def revision(self) -> str: @@ -101,38 +193,20 @@ def schema( return cls.model_json_schema(by_alias=by_alias, ref_template=ref_template) -class DistributionVersionsProbe(TargetProbe, RuntimeProbe[dict[str, str]]): - def run(self) -> dict[str, str]: - from packaging.utils import canonicalize_name +class DistributionProbe(TargetProbe): + """The selected interpreter's distribution inventory, rejecting duplicate identities.""" - return {canonicalize_name(dist.metadata["Name"]): dist.version for dist in distributions()} + @staticmethod + def distributions() -> dict[str, Distribution]: + installed = tuple(distributions()) + selected: dict[str, Distribution] = { + canonicalize_name(dist.name): dist for dist in installed + } + if len(selected) != len(installed): + raise ValueError("The runtime contains duplicate distribution identities") + return selected -@dataclass(frozen=True) -class InstalledPackagesProbe(TargetProbe, RuntimeProbe[dict[str, PackageRecord]]): - core: str - - def run(self) -> dict[str, PackageRecord]: - from packaging.utils import canonicalize_name - - if not Path(agent_comms.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()): - raise ValueError("Core is editable, not a built runtime package") - actual = STAMP.commit() - if actual != self.core: - raise ValueError( - "The wheel stamp does not name the selected source: " - f"installed {actual}, requested {self.core}" - ) - return { - canonicalize_name(dist.name): PackageRecord.from_wire( - { - "version": dist.version, - "source": ( - json.loads(source) - if (source := dist.read_text("direct_url.json")) - else None - ), - } - ) - for dist in distributions() - } +class DistributionVersionsProbe(DistributionProbe, RuntimeProbe[dict[str, str]]): + def run(self) -> dict[str, str]: + return {name: dist.version for name, dist in self.distributions().items()} diff --git a/src/agent_comms/installer/probes/resolution.py b/src/agent_comms/installer/probes/resolution.py new file mode 100644 index 0000000000..6afb2d982e --- /dev/null +++ b/src/agent_comms/installer/probes/resolution.py @@ -0,0 +1,85 @@ +"""One target-owned observation of package inputs and the interpreter running them.""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from pathlib import Path + +from packaging.version import Version + +import agent_comms +from agent_comms.installer.interpreter_operation import RuntimeProbe, TargetProbe +from agent_comms.installer.probes.interpreter import InterpreterBuildProbe +from agent_comms.installer.probes.packages import DistributionProbe, PackageRecord +from agent_comms.installer.runtime_resolution import ProjectMetadata, RuntimeResolution +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.package_stamp import STAMP + + +@dataclass(frozen=True) +class InstalledPackagesProbe(DistributionProbe, RuntimeProbe[dict[str, PackageRecord]]): + core: str + + def wheel_inputs(self) -> tuple[Path, ...]: + """A built donor re-observes its own inputs without changing the request wire.""" + return RuntimeResolution.at(LaunchRuntime.running().prefix).wheel_inputs() + + def run(self) -> dict[str, PackageRecord]: + if not Path(agent_comms.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()): + raise ValueError("Core is editable, not a built runtime package") + actual = STAMP.commit() + if actual != self.core: + raise ValueError( + "The wheel stamp does not name the selected source: " + f"installed {actual}, requested {self.core}" + ) + selected = self.distributions() + records = { + name: PackageRecord.from_wire( + { + "version": dist.version, + "source": ( + json.loads(source) + if (source := dist.read_text("direct_url.json")) + else None + ), + } + ) + for name, dist in selected.items() + } + for wheel in self.wheel_inputs(): + metadata = ProjectMetadata.from_wheel(wheel) + record = records[metadata.name] + if Version(record.version) != metadata.version: + raise ValueError(f"Installed version differs from supplied wheel: {wheel}") + records[metadata.name] = record.model_copy(update={ + "source": record.source.verified_wheel( + wheel, selected[metadata.name], LaunchRuntime.running() + ) + }) + return records + + +@dataclass(frozen=True) +class WheelPackagesProbe(InstalledPackagesProbe): + """Unrecorded targets supply inputs before their first resolution can be saved.""" + + wheels: tuple[Path, ...] + + def wheel_inputs(self) -> tuple[Path, ...]: + return self.wheels + + +@dataclass(frozen=True) +class RuntimeResolutionProbe(TargetProbe, RuntimeProbe[RuntimeResolution]): + core: str + wheels: tuple[Path, ...] = () + + def run(self) -> RuntimeResolution: + return RuntimeResolution( + core_commit=self.core, + packages=WheelPackagesProbe(self.core, self.wheels).run(), + interpreter=InterpreterBuildProbe().run(), + ) diff --git a/src/agent_comms/installer/runtime_resolution.py b/src/agent_comms/installer/runtime_resolution.py index f0f3ed9bb3..00034c1b9c 100644 --- a/src/agent_comms/installer/runtime_resolution.py +++ b/src/agent_comms/installer/runtime_resolution.py @@ -31,10 +31,14 @@ from agent_comms.nominal.field_codec import FieldCodec, GitCommitText, boundary_decoder from agent_comms.installer.commands import InstallCommands from agent_comms.installer.interpreter_operation import RuntimeProbe +from agent_comms.installer.probes.interpreter import ( + InterpreterBuild, + InterpreterBuildProbe, + RecordedInterpreterBuild, +) from agent_comms.installer.probes.packages import ( DistributionVersionsProbe, GitPin, - InstalledPackagesProbe, PackagePin, PackageRecord, PreparePackageProbe, @@ -636,6 +640,11 @@ class RuntimePackage(DeclaredFamily, affix="Package"): def commit(cls, resolution: RuntimeResolution) -> str: return resolution.packages[cls.distribution].source.revision() + @classmethod + def pin(cls, resolution: RuntimeResolution) -> PackagePin: + """Package origin owns its pin; an archive is not a repository revision.""" + return resolution.packages[cls.distribution].pin() + @classmethod def preparations(cls) -> tuple[RuntimeProbe[None], ...]: """Only packages declaring requirements add target-runtime operations.""" @@ -654,6 +663,11 @@ class CorePackage(RuntimePackage, ConsoleScriptPackage): def commit(cls, resolution: RuntimeResolution) -> str: return resolution.core_commit + @classmethod + def pin(cls, resolution: RuntimeResolution) -> PackagePin: + """Core's built wheel carries its source stamp separately from its URL.""" + return GitPin(resolution.packages[cls.distribution].version, cls.commit(resolution)) + class ToadPackage(RuntimePackage, ConsoleScriptPackage): distribution = "batrachian-toad" @@ -842,15 +856,19 @@ class UnbuiltRuntime: def matches(self, lock: Path, stack: RuntimeStack, core: str) -> bool: return False - def prepare(self, stack: RuntimeStack) -> None: + def prepare(self, stack: RuntimeStack, core: str) -> None: stack.sync() + def wheel_inputs(self) -> tuple[Path, ...]: + """Only the explicit-input stack can supply wheels before first capture.""" + return () + def complete(self, stack: RuntimeStack, core: str) -> None: """Link the native package, record the resolution and prove the client imports; a failure leaves no prefix, so the next build syncs it afresh.""" try: stack.install_native() - RuntimeResolution.capture(stack, core).save(self.prefix) + stack.observe_resolution(core).save(self.prefix) stack.imports() except BaseException: shutil.rmtree(self.prefix) @@ -860,6 +878,16 @@ def complete(self, stack: RuntimeStack, core: str) -> None: class RuntimeResolution(BaseModel): core_commit: str packages: dict[str, PackageRecord] + interpreter: RecordedInterpreterBuild | InterpreterBuild = Field( + default_factory=InterpreterBuild + ) + + def require_runtime(self, actual: RuntimeResolution) -> None: + """Require recorded interpreter evidence on both sides before comparing inputs.""" + self.interpreter.require_recorded() + actual.interpreter.require_recorded() + if self != actual: + raise UpgradeRefused("Observed runtime differs from its recorded build inputs") @classmethod def record_path(cls) -> Path: @@ -879,19 +907,24 @@ def recorded(cls, prefix: Path) -> RuntimeResolution | UnbuiltRuntime: return UnbuiltRuntime(prefix) return cls.at(prefix) - def prepare(self, stack: RuntimeStack) -> None: - """A recorded runtime is already synced; ``complete`` checks it still matches.""" - - def complete(self, stack: RuntimeStack, core: str) -> None: + def prepare(self, stack: RuntimeStack, core: str) -> None: + """Production-cache reuse must still satisfy the newly resolved source lock.""" if not self.matches(stack.lock, stack, core): raise UpgradeRefused("Existing built runtime differs from its resolved inputs") + + def complete(self, stack: RuntimeStack, core: str) -> None: + """Completion proves recorded inputs, not eligibility for a different build.""" + self.require_runtime(stack.observe_resolution(core)) stack.stage_native() stack.prepare_runtime() - @classmethod - def capture(cls, stack: RuntimeStack, core: str) -> RuntimeResolution: - packages = stack.probe(InstalledPackagesProbe(core)) - return cls(core_commit=core, packages=packages) + def wheel_inputs(self) -> tuple[Path, ...]: + """Only content-qualified archive records request another wheel observation.""" + return tuple( + wheel + for package in self.packages.values() + for wheel in package.source.wheel_inputs() + ) def save(self, prefix: Path) -> None: path = prefix / self.record_path().name @@ -902,14 +935,13 @@ def pins(self) -> dict[str, PackagePin]: result: dict[str, PackagePin] = { canonicalize_name(name): record.pin() for name, record in self.packages.items() } - # Wheel URLs locate build output, not the source revisions recorded by this owner. for member in RuntimePackage.members_with(RuntimePackage): - result[member.distribution] = GitPin( - self.packages[member.distribution].version, member.commit(self) - ) + result[member.distribution] = member.pin(self) return result def matches(self, lock: Path, stack: RuntimeStack, core: str) -> bool: + if not self.interpreter.matches(stack.probe(InterpreterBuildProbe())): + return False resolved = ResolvedLock.read(lock) print(f"install: real lock decodes {len(resolved.package)} packages", file=sys.stderr) # sync --locked has already selected the platform, groups and extras. Read its diff --git a/src/agent_comms/installer/runtime_stack.py b/src/agent_comms/installer/runtime_stack.py index 5a7ba7f77b..6044d99633 100644 --- a/src/agent_comms/installer/runtime_stack.py +++ b/src/agent_comms/installer/runtime_stack.py @@ -13,6 +13,7 @@ from pathlib import Path from agent_comms.child_process import ResidentManager +from agent_comms.build_artifact import BuildArtifact from agent_comms.installer.carry import STEPS from agent_comms.installer.commands import InstallCommands from agent_comms.installer.declared_uv import DeclaredUv @@ -26,8 +27,11 @@ from agent_comms.installer.practice import RuntimeHome from agent_comms.installer.probes.imports import ClientImportsProbe from agent_comms.installer.probes.native import PrepareNativeProbe, PrepareProducerDigestsProbe +from agent_comms.installer.probes.resolution import InstalledPackagesProbe, RuntimeResolutionProbe from agent_comms.installer.refusals import UpgradeRefused -from agent_comms.installer.runtime_resolution import ResolvedLock, RuntimePackage, RuntimeResolution +from agent_comms.installer.runtime_resolution import ( + ConsoleScriptPackage, ProjectMetadata, ResolvedLock, RuntimePackage, RuntimeResolution, +) from agent_comms.launch_runtime import LaunchRuntime, RestartEnvironment from agent_comms.native_build_inputs import NativeBuildInputs from agent_comms.native_package import NativePackageStore, pinned_package_for @@ -114,7 +118,7 @@ def build(checkout: Checkout, commands: InstallCommands, home: Path = HOME) -> T built = source.extended( TargetRuntime, runtime=LaunchRuntime.at_prefix(prefix), program=TargetProgram(checkout.path / STEPS.parent)) - RuntimeResolution.recorded(prefix).prepare(built) + RuntimeResolution.recorded(prefix).prepare(built, core) return built def sync(self) -> None: @@ -131,8 +135,11 @@ def sync(self) -> None: raise def complete(self) -> None: - """Finish the synced runtime from inside it: its native package, recorded resolution - and import check, or the check that a recorded one still matches its inputs.""" + """Finish inside this runtime, or re-observe its recorded packages and interpreter. + + Production-cache selection checks the source lock in build/prepare; completion + also verifies exact explicit-wheel inputs without claiming their Git ancestry. + """ with BuildPhase().running(self.commands.clock): core = self.commands.git(self.checkout, "rev-parse", "HEAD").strip() RuntimeResolution.recorded(self.runtime.prefix).complete(self, core) @@ -159,9 +166,16 @@ def probe[Answer](self, probe: RuntimeProbe[Answer]) -> Answer: env=self.environment(*self.program.import_roots()), program=self.program) + def observe_resolution(self, core: str) -> RuntimeResolution: + return self.probe(RuntimeResolutionProbe(core, self.wheel_inputs())) + def imports(self) -> None: self.probe(ClientImportsProbe()) + def wheel_inputs(self) -> tuple[Path, ...]: + """Re-observe retained inputs from the record; unbuilt lock targets have none.""" + return RuntimeResolution.recorded(self.runtime.prefix).wheel_inputs() + def stage_native(self) -> tuple[Path, Path]: """Prepare a missing artifact once; the checkout holds only its store link.""" manifest = self.checkout / "stack/pi-native.sha256" @@ -208,3 +222,70 @@ class TargetRuntime(RuntimeStack): """A runtime built from the target tree: its worker declares the tree's carry steps.""" program: TargetProgram = field(kw_only=True) + + +@dataclass(frozen=True, kw_only=True) +class WheelRuntime(RuntimeStack): + """A fresh functional target: explicit Python, retained dependencies, exact wheels. + + This never enters the production Core/lock reuse cache. Completion, native + preparation, package capture, imports and worker retirement stay on RuntimeStack. + Local-wheel origins remain archive origins, not invented repository revisions. + """ + + interpreter: LaunchRuntime + source_runtime: LaunchRuntime + wheels: tuple[Path, ...] + + def wheel_inputs(self) -> tuple[Path, ...]: + """The new runtime retains its own exact inputs; borrowed originals stay read-only.""" + return tuple(self.runtime.prefix / "inputs" / wheel.name for wheel in self.wheels) + + def sync(self) -> None: + named_prefix = self.runtime.interpreter.parent.parent + if named_prefix.exists() or named_prefix.is_symlink(): + raise UpgradeRefused(f"Preserve existing candidate: {named_prefix}") + prefix = self.runtime.prefix + self.interpreter.require_available() + self.source_runtime.require_available() + metadata = tuple(ProjectMetadata.from_wheel(wheel) for wheel in self.wheels) + selected = {project.name for project in metadata} + registered = {member.distribution for member in RuntimePackage.members_with(RuntimePackage)} + required = {member.distribution for member in RuntimePackage.members_with(ConsoleScriptPackage)} + if len(selected) != len(metadata) or not required <= selected <= registered: + raise UpgradeRefused("Provide one exact wheel for each console-script package; " + "only declared runtime packages may be replaced") + declared = DeclaredUv.of(self.checkout / "pyproject.toml", self.uv) + if not declared.accepts(self.commands.run(self.uv, "--version")): + raise UpgradeRefused("Candidate preparation requires Core's existing declared uv") + source = RuntimeResolution.at(self.source_runtime.prefix) + donor = RuntimeStack(self.checkout, self.commands, self.home, runtime=self.source_runtime) + actual = donor.probe(InstalledPackagesProbe(source.core_commit)) + if actual != source.packages: + raise UpgradeRefused("Dependency source differs from its recorded packages") + artifacts = tuple(BuildArtifact.capture(wheel) for wheel in self.wheels) + self.commands.run(self.uv, "--offline", "venv", "--python", + self.interpreter.interpreter, prefix) + if self.runtime.interpreter.resolve() != self.interpreter.interpreter.resolve(): + raise UpgradeRefused("Prepared target did not select the requested interpreter") + (prefix / "inputs").mkdir(mode=0o700) + for artifact, destination in zip(artifacts, self.wheel_inputs(), strict=True): + shutil.copyfile(artifact.path, destination) + destination.chmod(0o400) + if BuildArtifact.capture(destination).sha256 != artifact.sha256: + raise UpgradeRefused(f"Retained wheel differs from its input: {artifact.path}") + shutil.copytree(self.source_runtime.purelib, self.runtime.purelib, + dirs_exist_ok=True, symlinks=True) + # Prefix data is copied intact; package preparation validates its native assets. + shutil.copytree(self.source_runtime.prefix / "share", prefix / "share", symlinks=True) + self.commands.run(self.uv, "--offline", "pip", "install", "--no-deps", "--reinstall", + "--python", self.python, *self.wheel_inputs(), env=self.environment()) + self.commands.run(self.uv, "--offline", "pip", "check", "--python", self.python, + env=self.environment()) + + def prepare(self) -> RuntimeResolution: + """Prepare once; a retained path is never silently rebound or reused.""" + with BuildPhase().running(self.commands.clock): + self.sync() + self.complete() + return RuntimeResolution.at(self.runtime.prefix) diff --git a/src/agent_comms/launch_runtime.py b/src/agent_comms/launch_runtime.py index c5fbf35524..d9b72d9bb7 100644 --- a/src/agent_comms/launch_runtime.py +++ b/src/agent_comms/launch_runtime.py @@ -112,11 +112,15 @@ def at_prefix(cls, prefix: Path) -> Self: def prefix(self) -> Path: return self.interpreter.parent.parent.resolve() + @property + def purelib(self) -> Path: + """The selected runtime's package directory, including before installation.""" + return Path(sysconfig.get_path("purelib", vars={"base": str(self.prefix)})) + @property def package_root(self) -> Path: """The Core package installed in this runtime's purelib directory.""" - purelib = Path(sysconfig.get_path("purelib", vars={"base": str(self.prefix)})) - root = purelib / "agent_comms" + root = self.purelib / "agent_comms" initializer = root / "__init__.py" if not initializer.is_file(): raise FileNotFoundError(f"Installed Core package is missing: {initializer}") diff --git a/src/agent_comms/nominal/field_codec.py b/src/agent_comms/nominal/field_codec.py index b26d15e5b9..8b16f245ec 100644 --- a/src/agent_comms/nominal/field_codec.py +++ b/src/agent_comms/nominal/field_codec.py @@ -37,6 +37,7 @@ get_type_hints, overload, ) +from urllib.parse import quote from uuid import uuid4 from weakref import WeakKeyDictionary @@ -1123,6 +1124,10 @@ def represented_class(cls, annotation: object) -> type[ValueDecoder[object]]: ) return next(iter(cls._registered(annotation)), scalar) + def value_schema(self) -> dict[str, JsonValue]: + """The compiled decoder owns target-dependent schema as well as conversion.""" + return self.schema() + @classmethod def with_representation( cls, annotation: object, target: object, args: tuple[object, ...], @@ -1459,7 +1464,7 @@ def schema_for(cls, annotation: object) -> dict[str, JsonValue]: return FieldCodec._family_schema(annotation) if isinstance(annotation, type) and is_dataclass(annotation): return FieldCodec.record_schema(annotation) - return ValueDecoder.represented_class(annotation).schema() + return compile_decoder(annotation).value_schema() class AcceptsNull[T](FieldRepresentation[T]): @@ -3797,8 +3802,20 @@ def _structural_decoder(annotation: object) -> ValueDecoder[object]: class _PydanticDecoder(ValueDecoder[object]): """An external model owns its validation and JSON representation.""" + origins = (BaseModel,) target: type[BaseModel] + def value_schema(self) -> dict[str, JsonValue]: + # A model's root-relative references belong to that model's resource, + # even when its schema is nested inside a tool field or union. + identity = quote(f"{self.target.__module__}.{self.target.__qualname__}", safe="") + return { + "$id": f"urn:python-schema:{identity}", + **FieldCodec.decode(dict[str, JsonValue], self.target.model_json_schema( + by_alias=True, mode="validation", + )), + } + @boundary_decoder def decode(self, data: object) -> BaseModel: with MalformedRecord.refusing(f"Invalid {self.target.__name__}"): @@ -3820,8 +3837,6 @@ class _MemberDecoders(dict[object, ValueDecoder[object]]): def __missing__(self, target: object) -> ValueDecoder[object]: if isinstance(target, type) and is_dataclass(target): decoder: ValueDecoder[object] = _RecordDecoder(target) - elif isinstance(target, type) and issubclass(target, BaseModel): - decoder = _PydanticDecoder(target) else: raise UnsupportedFieldType(f"Unsupported JSON type: {target}") self[target] = decoder diff --git a/src/agent_comms/team/order.py b/src/agent_comms/team/order.py index 8086a90616..ee4ebf5f03 100644 --- a/src/agent_comms/team/order.py +++ b/src/agent_comms/team/order.py @@ -1499,7 +1499,9 @@ class InstallVerb(OrderVerb): clears = (InstallDue,) def run(self, ctx: Team) -> None: - for due in InstallDue.due(ctx.comms, ProgressRead()): + pulls = ProgressRead() + InstallDue.prepare(ctx.comms, ctx.commander, HeldOrders.read(ctx.comms), pulls) + for due in InstallDue.due(ctx.comms, pulls): print(f"{due.repo.to_wire()} main {due.main[:12]}: {due.launch(ctx.comms)}") diff --git a/tests/conftest.py b/tests/conftest.py index 97c178be38..b28a093a5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,6 +58,7 @@ from agent_comms.team.order import Team from agent_comms.repo import Repo from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.installer.probes.interpreter import InterpreterBuildProbe, RecordedInterpreterBuild from source_programs import SourcePrograms from source_revision_fixture import SourceRepository from agent_comms.threads import Thread @@ -440,7 +441,19 @@ def configure(bus: Comms, *check_repositories: Repo) -> Team: @pytest.fixture -def installed_runtime(durable_home: Path, tmp_path: Path) -> LaunchRuntime: +def interpreter_build() -> RecordedInterpreterBuild: + """The native implementation tests execute under, also symlinked by fixture runtimes. + + This observes the real current interpreter; it does not claim a process ran + inside the synthetic installation's package environment. + """ + return InterpreterBuildProbe().run() + + +@pytest.fixture +def installed_runtime( + durable_home: Path, tmp_path: Path, interpreter_build: RecordedInterpreterBuild, +) -> LaunchRuntime: """A temporary installation record, executable and pointer, never the real home.""" from agent_comms.installer.probes.packages import CommitRecord, GitRecord, PackageRecord from agent_comms.installer.runtime_resolution import RuntimePackage, RuntimeResolution @@ -457,7 +470,7 @@ def installed_runtime(durable_home: Path, tmp_path: Path) -> LaunchRuntime: ) return record_installed_runtime(tmp_path / 'installed-runtime', RuntimeResolution( core_commit=repository.git(repository.checkout, 'rev-parse', 'HEAD').strip(), - packages=packages, + packages=packages, interpreter=interpreter_build, )) diff --git a/tests/test_field_codec.py b/tests/test_field_codec.py index c195100bca..e841c97f20 100644 --- a/tests/test_field_codec.py +++ b/tests/test_field_codec.py @@ -6,7 +6,7 @@ from typing import Annotated, ClassVar, Self import pytest -from pydantic import JsonValue +from pydantic import BaseModel, JsonValue from typing_extensions import TypeForm from agent_comms.nominal.declared_family import DeclaredFamily, UnknownFamilyMember @@ -104,6 +104,28 @@ class NoSchema: FieldCodec.value_schema(NoSchema) +def test_external_model_schema_and_conversion_use_the_same_decoder() -> None: + class Child(BaseModel): + count: int + + class Model(BaseModel): + child: Child + + @dataclass + class Envelope: + value: Model + + value = Envelope(Model(child=Child(count=3))) + assert FieldCodec.decode(Envelope, FieldCodec.encode(value)) == value + with pytest.raises(MalformedRecord): + FieldCodec.decode(Envelope, {"value": {"child": {"count": "wrong"}}}) + schema = FieldCodec.value_schema(Model) + assert schema["$id"] + assert {key: item for key, item in schema.items() if key != "$id"} == Model.model_json_schema() + assert FieldCodec.record_schema(Envelope)["properties"]["value"] == schema + assert compile_decoder(Model).target is Model + + def test_transient_plan_excludes_fields_before_decoder_or_value_access() -> None: class Opaque: pass diff --git a/tests/test_install_worker_holds.py b/tests/test_install_worker_holds.py index bfee19f4c4..781103edc1 100644 --- a/tests/test_install_worker_holds.py +++ b/tests/test_install_worker_holds.py @@ -25,7 +25,8 @@ ) from agent_comms.installer.owner_starts import OwnerStart, RefusedOwner, StartedOwner from agent_comms.installer.probes.commands import CliCommandProbe, CliOutput, CliRefusedError -from agent_comms.installer.probes.packages import DistributionVersionsProbe, InstalledPackagesProbe +from agent_comms.installer.probes.packages import DistributionVersionsProbe +from agent_comms.installer.probes.resolution import InstalledPackagesProbe from agent_comms.installer.refusals import InterpreterWorkerDied, UpgradeRefused from agent_comms.installer.upgrade import Stack from agent_comms.launch_root import LaunchRoot diff --git a/tests/test_proof_build.py b/tests/test_proof_build.py index eec84deff8..1cc57150da 100644 --- a/tests/test_proof_build.py +++ b/tests/test_proof_build.py @@ -14,6 +14,7 @@ ProofBuildMismatchError, RecordedBuildIdentity, UninstalledRepositoryError, + UnrecordedBuildIdentity, ) from agent_comms.check_root import CheckRoot from agent_comms.team.order import Team @@ -22,6 +23,8 @@ from agent_comms.nominal.field_codec import FieldCodec, GoalId from agent_comms.goal_actions import CreateProblemGoalAction from agent_comms.goals import CountCheckResult, OrderCheck +from agent_comms.held_orders import HeldOrders +from agent_comms.installer.probes.interpreter import RecordedInterpreterBuild from agent_comms.installer.probes.packages import ArchiveRecord, CommitRecord, GitRecord, PackageRecord from agent_comms.installer.runtime_resolution import CorePackage, RuntimePackage, RuntimeResolution from agent_comms.installer.logs import InstallLogs @@ -68,7 +71,7 @@ class ProofBuildCheckout: @pytest.fixture def proof_build_checkouts( - tmp_path: Path, durable_home: Path + tmp_path: Path, durable_home: Path, interpreter_build: RecordedInterpreterBuild, ) -> tuple[ProofBuildCheckout, ...]: """Own each package's distinct commit graph and the installed manifest; durable_home is the HOME the installation records under.""" @@ -112,7 +115,9 @@ def proof_build_checkouts( version="1", source=ArchiveRecord(url="file:///owned/core-wheel.whl") ) record_installed_runtime( - tmp_path / "runtime", RuntimeResolution(core_commit=core.installed, packages=packages)) + tmp_path / "runtime", RuntimeResolution( + core_commit=core.installed, packages=packages, interpreter=interpreter_build, + )) return tuple(checkouts) @@ -163,7 +168,7 @@ def test_merge_in_head_cannot_close_until_the_recorded_build_contains_it( runtime = LaunchRuntime.installed() resolution = RuntimeResolution.at(runtime.prefix) - RuntimeResolution(core_commit=graph.newer, packages=resolution.packages).save(runtime.prefix) + resolution.model_copy(update={"core_commit": graph.newer}).save(runtime.prefix) accepted = run.after_merge(todo, merged) assert accepted.state is after_install # P553: bf328ceb5 sat in the installed build under "Merged, waiting on install". @@ -191,9 +196,50 @@ def test_runtime_package_family_owns_installed_commits(proof_build_checkouts): ) assert recorded.repository == package.repository assert recorded.commit == graph.installed + assert installed.interpreter == RuntimeResolution.at(LaunchRuntime.installed().prefix).interpreter assert FieldCodec.decode(RecordedBuildIdentity, FieldCodec.encode(installed)) == installed +def test_same_commits_do_not_reuse_a_proof_from_another_interpreter(proof_build_checkouts): + original = RecordedBuildIdentity.installed().recorded() + runtime = LaunchRuntime.installed() + resolution = RuntimeResolution.at(runtime.prefix) + interpreter = resolution.interpreter.require_recorded() + changed_image = interpreter.artifacts[0].model_copy(update={"sha256": "0" * 64}) + changed = interpreter.model_copy(update={ + "artifacts": (changed_image, *interpreter.artifacts[1:]), + }) + resolution.model_copy(update={"interpreter": changed}).save(runtime.prefix) + current = RecordedBuildIdentity.installed().recorded() + assert original.commits == current.commits + assert original != current + assert FieldCodec.decode(RecordedBuildIdentity, FieldCodec.encode(current)) == current + graph = proof_build_checkouts[0] + with pytest.raises(ProofBuildMismatchError, match="differs from installed"): + original.installation((), graph.repository.checkout) + # Ancestry still answers its own question; it is not an interpreter proof. + assert original.contains(graph.installed, graph.repository.checkout) + assert current.contains(graph.installed, graph.repository.checkout) + + +def test_missing_interpreter_evidence_is_readable_but_cannot_certify_a_proof(proof_build_checkouts): + installed = RecordedBuildIdentity.installed().recorded() + unobserved = RecordedBuildIdentity(installed.commits) + retained = FieldCodec.decode(RecordedBuildIdentity, FieldCodec.encode(unobserved)) + assert retained.commits == installed.commits + with pytest.raises(ProofBuildMismatchError, match="no interpreter observation"): + retained.recorded() + with pytest.raises(ProofBuildMismatchError, match="no interpreter observation"): + retained.installation((), proof_build_checkouts[0].repository.checkout) + runtime = LaunchRuntime.installed() + resolution = RuntimeResolution.at(runtime.prefix) + resolution.model_copy(update={"interpreter": retained.interpreter}).save(runtime.prefix) + with pytest.raises(ProofBuildMismatchError, match="no interpreter observation"): + UnrecordedBuildIdentity().recorded() + with pytest.raises(ProofBuildMismatchError, match="no interpreter observation"): + installed.installation((), proof_build_checkouts[0].repository.checkout) + + @pytest.mark.parametrize("package", RuntimePackage.members_with(RuntimePackage)) def test_containment_selects_the_checkouts_installed_repository(package, proof_build_checkouts): installed = RecordedBuildIdentity.installed() @@ -249,18 +295,22 @@ def test_installed_proof_closes_and_older_build_is_refused_on_copied_bus( checked, contained, missing = graph.installed, graph.installed, graph.newer github_cli.configure(graph.repository.github) for number, commit in ((1, checked), (2, checked), (3, contained), (4, missing)): - github_cli.put(graph.repository.github, MergedListedPullRequest( + row = MergedListedPullRequest( number, f'fix/{number}', commit, updated='2026-10-25T12:00:00Z', base=WorkBranches.main, reviewable_times=('2026-10-25T12:00:00Z',), merge=PullRequestCommit(commit), - merged='2026-10-25T12:00:00Z')) + merged='2026-10-25T12:00:00Z') + graph.repository.git(checkout, 'push', 'origin', + f'{row.head}:refs/pull/{row.number}/head') + github_cli.put(graph.repository.github, row) caller = ThreadCaller("lead") + holder = ThreadCaller("worker") works = "Open the installed application and confirm the copied goal keeps its source location." artifact = tmp_path / "proof.txt" artifact.write_text(works) source.registry.storage_rule = replace(source.registry.storage_rule, scratch_base=tmp_path) with CheckRoot(source, ("lead", "worker"), tmp_path / "proof" / "copied").open() as copied: request = copied.goals.todos.add_request( - copied.registry.require("lead").incarnation, "Record the copied application proof", + holder.identity(copied), "Record the copied application proof", ).id cases = ( ("P1", OperationDelivery(proof_build=installed), None), @@ -276,7 +326,7 @@ def test_installed_proof_closes_and_older_build_is_refused_on_copied_bus( DeliveredCommitNotInstalledError, ), ) - for problem, delivery, expected_error in cases: + for number, (problem, delivery, expected_error) in enumerate(cases, 1): build = delivery.proof_build order = PendingProblemOrder( scope=FIX_SCOPE, @@ -288,7 +338,9 @@ def test_installed_proof_closes_and_older_build_is_refused_on_copied_bus( report="Report the actual close result.", run=ApplicationOrderRun(works), ) - caller.invoke( + # The holder accepts each scenario; this tests proof binding, not + # permission for a peer to replace the preceding active goal. + holder.invoke( CommsTodoCreateTool( owner="worker", order=CreateProblemGoalAction( @@ -296,7 +348,7 @@ def test_installed_proof_closes_and_older_build_is_refused_on_copied_bus( order=order, request=request, review_at=RecordedEventTime(4102444800), - commander="lead", assigner=caller.identity(copied), + commander="lead", assigner=holder.identity(copied), repo=graph.repository.github, text="Bind proof to its build", ), @@ -370,7 +422,9 @@ def test_install_due_compares_fetched_main_to_installed_build( Team.on_root("lead", registered, bus.root) github_cli.configure(graph.repository.github) - due = InstallDue.due(bus, ProgressRead()) + pulls = ProgressRead() + InstallDue.prepare(bus, "lead", HeldOrders.read(bus), pulls) + due = InstallDue.due(bus, pulls) assert Repo.git(checkout, "rev-parse", "HEAD") == before assert Repo.git(checkout, "rev-parse", "origin/main").strip() == ( @@ -389,8 +443,10 @@ def test_install_due_compares_fetched_main_to_installed_build( # An install advances the canonical manifest, not a clock or bus schedule. runtime = LaunchRuntime.installed().prefix resolution = RuntimeResolution.at(runtime) - RuntimeResolution(core_commit=graph.newer, packages=resolution.packages).save(runtime) - after_install = InstallDue.due(bus, ProgressRead()) + resolution.model_copy(update={"core_commit": graph.newer}).save(runtime) + pulls = ProgressRead() + InstallDue.prepare(bus, "lead", HeldOrders.read(bus), pulls) + after_install = InstallDue.due(bus, pulls) assert after_install == () assert Repo.git(checkout, "rev-parse", "HEAD") == before configured = bus.registry.snapshot().commander_project.require().repos @@ -431,7 +487,9 @@ def test_a_one_minute_old_merge_is_due_and_names_its_request_without_launching( merged = int(Repo.git(checkout, "log", "-1", "--format=%ct", merge)) monkeypatch.setattr(time, "time", lambda: merged + 60) - (due,) = InstallDue.due(bus, ProgressRead()) + pulls = ProgressRead() + InstallDue.prepare(bus, "lead", HeldOrders.read(bus), pulls) + (due,) = InstallDue.due(bus, pulls) assert due.main == merge assert "#41 Install what merged" in due.line() and "#42 fix/next" in due.line() assert "order install" in due.line().rsplit(" -> ", 1)[1] diff --git a/tests/test_runtime_resolution.py b/tests/test_runtime_resolution.py index 1eb23cc990..3900729dec 100644 --- a/tests/test_runtime_resolution.py +++ b/tests/test_runtime_resolution.py @@ -1,14 +1,30 @@ +import ctypes +import hashlib +import json import os +import sys import tomllib +from dataclasses import dataclass, fields +from importlib.metadata import PathDistribution from pathlib import Path +from zipfile import ZipFile import pytest +from agent_comms.build_artifact import BuildArtifact +from agent_comms.child_process import Platform from agent_comms.nominal.field_codec import FieldCodec from agent_comms.installer.refusals import UpgradeRefused from agent_comms.installer.commands import InstallCommands -from agent_comms.installer.probes.packages import InstalledPackagesProbe -from agent_comms.installer.runtime_resolution import ResolvedLock, RuntimeResolution +from agent_comms.installer.probes.interpreter import InterpreterBuildProbe +from agent_comms.installer.probes.packages import ( + ArchivePin, ArchiveRecord, GitPin, PackageRecord, +) +from agent_comms.installer.probes.resolution import InstalledPackagesProbe +from agent_comms.installer.runtime_resolution import ( + CorePackage, ResolvedLock, RuntimeResolution, ToadPackage, +) +from agent_comms.installer.runtime_stack import RuntimeStack from agent_comms.launch_runtime import LaunchRuntime @@ -22,6 +38,172 @@ def test_package_probe_entry_uses_one_declaration_before_refusing_build_identity ) +@dataclass(frozen=True) +class InstalledWheel: + runtime: LaunchRuntime + wheel: Path + distribution: PathDistribution + source: ArchiveRecord + +@pytest.fixture +def installed_wheel(tmp_path: Path) -> InstalledWheel: + prefix = tmp_path / "runtime" + purelib = LaunchRuntime.at_prefix(prefix).purelib + metadata = "sample-1.0.dist-info" + contents = { + "sample/__init__.py": b"value = 1\n", + f"{metadata}/METADATA": b"Metadata-Version: 2.4\nName: sample\nVersion: 1.0\n", + f"{metadata}/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + f"{metadata}/RECORD": b"installer replaces this record\n", + } + wheel = tmp_path / "sample-1.0-py3-none-any.whl" + with ZipFile(wheel, "w") as archive: + for name, data in contents.items(): + archive.writestr(name, data) + target = purelib / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + source = ArchiveRecord(url=wheel.as_uri()) + (purelib / metadata / "direct_url.json").write_text(json.dumps({ + "url": source.url, "archive_info": {}, + })) + return InstalledWheel(LaunchRuntime.at_prefix(prefix), wheel, + PathDistribution(purelib / metadata), source) + + +def test_verified_archive_preserves_content_without_inventing_git(installed_wheel: InstalledWheel): + original = installed_wheel.distribution.read_text("direct_url.json") + verified = installed_wheel.source.verified_wheel( + installed_wheel.wheel, installed_wheel.distribution, installed_wheel.runtime + ) + assert verified.archive_info.hashes == { + "sha256": hashlib.sha256(installed_wheel.wheel.read_bytes()).hexdigest() + } + assert verified.wheel_inputs() == (installed_wheel.wheel,) + assert installed_wheel.distribution.read_text("direct_url.json") == original + with pytest.raises(ValueError, match="Git revision"): + verified.revision() + record = RuntimeResolution.model_validate({ + "core_commit": "unqualified-fixture", + "packages": {"sample": {"version": "1.0", "source": verified.model_dump()}}, + }) + assert RuntimeResolution.model_validate_json(record.model_dump_json()) == record + + +@pytest.mark.parametrize("changed", [False, True]) +def test_donor_wheel_selection_uses_its_recorded_prefix( + installed_wheel: InstalledWheel, monkeypatch: pytest.MonkeyPatch, changed: bool +): + """Model the invoked prefix; this does not launch or qualify a donor worker.""" + original = installed_wheel.distribution.read_text("direct_url.json") + verified = installed_wheel.source.verified_wheel( + installed_wheel.wheel, installed_wheel.distribution, installed_wheel.runtime + ) + recorded = RuntimeResolution.model_validate({ + "core_commit": "unqualified-fixture", + "packages": {"sample": {"version": "1.0", "source": verified.model_dump()}}, + }) + recorded.save(installed_wheel.runtime.prefix) + # sys.executable is the foreign launch input; no production method is patched. + monkeypatch.setattr(sys, "executable", str(installed_wheel.runtime.interpreter)) + probe = InstalledPackagesProbe(recorded.core_commit) + assert tuple(field.name for field in fields(probe)) == ("core",) + assert probe.wheel_inputs() == recorded.wheel_inputs() == (installed_wheel.wheel,) + assert original is not None + raw = ArchiveRecord.model_validate_json(original) + if changed: + installed_wheel.distribution.locate_file("sample/__init__.py").write_bytes(b"changed\n") + with pytest.raises(ValueError, match="Installed bytes differ"): + raw.verified_wheel(probe.wheel_inputs()[0], installed_wheel.distribution, + installed_wheel.runtime) + else: + assert raw.verified_wheel(probe.wheel_inputs()[0], installed_wheel.distribution, + installed_wheel.runtime) == verified + assert installed_wheel.distribution.read_text("direct_url.json") == original + + +def test_runtime_stack_reobserves_its_resolution_wheel_inputs(installed_wheel: InstalledWheel): + with InstallCommands() as commands: + stack = RuntimeStack(Path(__file__).resolve().parents[1], commands, + runtime=installed_wheel.runtime) + assert stack.wheel_inputs() == () + verified = installed_wheel.source.verified_wheel( + installed_wheel.wheel, installed_wheel.distribution, installed_wheel.runtime + ) + recorded = RuntimeResolution( + core_commit="unqualified-fixture", + packages={"sample": PackageRecord(version="1.0", source=verified)}, + ) + recorded.save(installed_wheel.runtime.prefix) + assert stack.wheel_inputs() == (installed_wheel.wheel,) + + +def test_changed_installed_member_refuses_archive_proof(installed_wheel: InstalledWheel): + installed_wheel.distribution.locate_file("sample/__init__.py").write_bytes(b"value = 2\n") + with pytest.raises(ValueError, match="Installed bytes differ"): + installed_wheel.source.verified_wheel( + installed_wheel.wheel, installed_wheel.distribution, installed_wheel.runtime + ) + + +def test_root_wheel_domain_does_not_silently_skip_data_members(installed_wheel: InstalledWheel): + with ZipFile(installed_wheel.wheel, "a") as archive: + archive.writestr("sample-1.0.data/scripts/other", b"unverified\n") + with pytest.raises(ValueError, match="root-layout"): + installed_wheel.source.verified_wheel( + installed_wheel.wheel, installed_wheel.distribution, installed_wheel.runtime + ) + + +def test_archive_proof_binds_both_origin_and_runtime(installed_wheel: InstalledWheel, tmp_path: Path): + wrong_origin = ArchiveRecord(url=(tmp_path / "different.whl").as_uri()) + with pytest.raises(ValueError, match="origin differs"): + wrong_origin.verified_wheel(installed_wheel.wheel, installed_wheel.distribution, + installed_wheel.runtime) + with pytest.raises(ValueError, match="escapes its runtime"): + installed_wheel.source.verified_wheel( + installed_wheel.wheel, installed_wheel.distribution, + LaunchRuntime.at_prefix(tmp_path / "unrelated"), + ) + + +def test_native_image_artifact_uses_its_open_handle(): + address = ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p) + with Platform.current().native_image(address) as stream: + observed = BuildArtifact.from_stream(stream) + assert observed == BuildArtifact.capture(observed.path) + assert not stream.closed + assert stream.closed + + +def test_unrecorded_interpreter_is_not_a_runtime_cache_match(tmp_path: Path): + recorded = manifest() + observed = InterpreterBuildProbe().run() + assert not recorded.interpreter.matches(observed) + assert observed.matches(observed) + changed = observed.model_copy(update={"abi": observed.abi + "-different"}) + assert not observed.matches(changed) + with InstallCommands() as commands: + stack = RuntimeStack(Path(__file__).resolve().parents[1], commands, + home=tmp_path, runtime=LaunchRuntime.running()) + assert not recorded.matches(stack.lock, stack, recorded.core_commit) + + +def test_runtime_identity_refuses_missing_and_changed_interpreter(): + with pytest.raises(ValueError, match="no recorded interpreter"): + manifest().require_runtime(manifest()) + observed = InterpreterBuildProbe().run() + first = manifest().model_copy(update={"interpreter": observed}) + assert RuntimeResolution.model_validate_json(first.model_dump_json()) == first + first.require_runtime(first) + changed_image = observed.artifacts[0].model_copy(update={"sha256": "0" * 64}) + second = first.model_copy(update={"interpreter": observed.model_copy(update={ + "artifacts": (changed_image, *observed.artifacts[1:]), + })}) + with pytest.raises(UpgradeRefused, match="recorded build inputs"): + first.require_runtime(second) + + TOAD_COMMIT = "a" * 40 TEXTUAL_COMMIT = "b" * 40 @@ -97,6 +279,28 @@ def test_recorded_sources_match_selected_lock_not_wheel_locations(): assert lock().pins({**selected, "unselected": "5"}, "core") != built.pins() +@pytest.mark.parametrize("digest", ["0" * 64, "1" * 64]) +def test_explicit_archive_keeps_its_pin_and_refuses_repository_ancestry(digest: str): + built = manifest() + archive = ArchiveRecord.model_validate({ + "url": "file:///owned/inputs/batrachian_toad-2-py3-none-any.whl", + "archive_info": {"hashes": {"sha256": digest}}, + }) + built = built.model_copy(update={"packages": { + **built.packages, + ToadPackage.distribution: PackageRecord(version="2", source=archive), + }}) + pins = built.pins() + assert pins[CorePackage.distribution] == GitPin("1", "core") + assert pins[ToadPackage.distribution] == ArchivePin( + "2", archive.url, (("sha256", digest),) + ) + selected = {name: record.version for name, record in built.packages.items()} + assert pins != lock().pins(selected, built.core_commit) + with pytest.raises(ValueError, match="Git revision"): + ToadPackage.commit(built) + + def test_missing_and_ambiguous_lock_evidence_refuses(): with pytest.raises(ValueError): lock().pins({"missing": "1"}, "core") diff --git a/tools/checker_stack.py b/tools/checker_stack.py index 4b7666332d..4390a194a9 100644 --- a/tools/checker_stack.py +++ b/tools/checker_stack.py @@ -244,7 +244,7 @@ def prepare(self) -> RuntimeResolution: ) self.commands.run(self.uv, "pip", "check", "--python", self.python) commit = Repo.git(self.core, "rev-parse", "HEAD").strip() - resolution = RuntimeResolution.capture(self, commit) + resolution = self.observe_resolution(commit) self.require_resolution(resolution) resolution.save(self.runtime.prefix) digest = hashlib.sha256(self.lock.read_bytes()).hexdigest()