From 19d96f223f80ed0af61bdc20494e870bc1459247 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:23:17 -0400 Subject: [PATCH 1/6] ci: publish eval releases through release app --- .github/workflows/release.yml | 31 +++++++++++++++++++++++++++++-- tests/test_release_contract.py | 10 ++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac839e5..e7383b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -277,7 +277,7 @@ jobs: runs-on: ubuntu-latest environment: pypi permissions: - contents: write + contents: read steps: - name: Checkout the exact release tag uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -285,9 +285,36 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Create the release App token + id: release-app + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ vars.OPENADAPT_RELEASE_APP_ID }} + private-key: ${{ secrets.OPENADAPT_RELEASE_APP_PRIVATE_KEY }} + owner: OpenAdaptAI + repositories: openadapt-evals + permission-contents: write + + - name: Verify the release App identity + env: + APP_SLUG: ${{ steps.release-app.outputs.app-slug }} + APP_TOKEN: ${{ steps.release-app.outputs.token }} + EXPECTED_APP_ID: ${{ vars.OPENADAPT_RELEASE_APP_ID }} + run: | + set -euo pipefail + test "$APP_SLUG" = 'openadapt-release' + test -n "$APP_TOKEN" + case "$EXPECTED_APP_ID" in + ''|*[!0-9]*) + echo '::error::OPENADAPT_RELEASE_APP_ID must be a positive integer.' + exit 1 + ;; + esac + test "$EXPECTED_APP_ID" -gt 0 + - name: Publish the exact GitHub release env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ steps.release-app.outputs.token }} RELEASE_TAG: ${{ github.ref_name }} run: | set -euo pipefail diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 463aa59..12be996 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -79,6 +79,16 @@ def test_release_configuration_is_fail_closed() -> None: assert "secrets.OPENADAPT_RELEASE_APP_PRIVATE_KEY" in workflow assert "permission-contents: write" in workflow assert "permission-pull-requests: write" not in workflow + publish_github_job = workflow.split(" publish-github-release:", 1)[1] + assert "environment: pypi" in publish_github_job + assert "permissions:\n contents: read" in publish_github_job + assert "id: release-app" in publish_github_job + assert "owner: OpenAdaptAI" in publish_github_job + assert "repositories: openadapt-evals" in publish_github_job + assert "permission-contents: write" in publish_github_job + assert "test \"$APP_SLUG\" = 'openadapt-release'" in publish_github_job + assert "GH_TOKEN: ${{ steps.release-app.outputs.token }}" in publish_github_job + assert "GH_TOKEN: ${{ github.token }}" not in publish_github_job assert re.search(r"(?m)^ workflow_dispatch:\s*$", workflow) assert re.search(r"(?m)^ version:\s*$", workflow) assert re.search(r"(?m)^ source_commit:\s*$", workflow) From 26a74a1cdf7a42722e8ce997da29d9be0204e4d6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:32:34 -0400 Subject: [PATCH 2/6] ci: verify immutable eval release recovery --- .github/workflows/release.yml | 26 ++- scripts/verify_pypi_release.py | 271 ++++++++++++++++++++++++++++++ tests/test_release_contract.py | 15 ++ tests/test_verify_pypi_release.py | 170 +++++++++++++++++++ 4 files changed, 476 insertions(+), 6 deletions(-) create mode 100644 scripts/verify_pypi_release.py create mode 100644 tests/test_verify_pypi_release.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7383b2..2507253 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -264,6 +264,19 @@ jobs: print-hash: true skip-existing: true + - name: Verify immutable PyPI publication bytes + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + version="${RELEASE_TAG#v}" + test "$RELEASE_TAG" = "v${version}" + python scripts/verify_pypi_release.py \ + --directory dist \ + --version "$version" \ + --wait-seconds 300 \ + --poll-seconds 10 + publish-github-release: needs: - authorize-release-tag @@ -321,12 +334,6 @@ jobs: test "$(git cat-file -t "$GITHUB_REF")" = 'tag' test "$(git rev-parse "${GITHUB_REF}^{commit}")" = "$GITHUB_SHA" if gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - state=$(gh release view "$RELEASE_TAG" \ - --repo "$GITHUB_REPOSITORY" \ - --json isDraft,isPrerelease,tagName) - test "$(jq -r '.tagName' <<<"$state")" = "$RELEASE_TAG" - test "$(jq -r '.isDraft' <<<"$state")" = 'false' - test "$(jq -r '.isPrerelease' <<<"$state")" = 'false' echo 'The exact GitHub release already exists.' else gh release create "$RELEASE_TAG" \ @@ -335,3 +342,10 @@ jobs: --generate-notes \ --title "$RELEASE_TAG" fi + state=$(gh release view "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --json author,isDraft,isPrerelease,tagName) + test "$(jq -r '.tagName' <<<"$state")" = "$RELEASE_TAG" + test "$(jq -r '.isDraft' <<<"$state")" = 'false' + test "$(jq -r '.isPrerelease' <<<"$state")" = 'false' + test "$(jq -r '.author.login' <<<"$state")" = 'openadapt-release[bot]' diff --git a/scripts/verify_pypi_release.py b/scripts/verify_pypi_release.py new file mode 100644 index 0000000..68c081f --- /dev/null +++ b/scripts/verify_pypi_release.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Verify one immutable PyPI release against the exact local build.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +import urllib.error +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlsplit + +PYPI_PROJECT = "openadapt-evals" +STABLE_VERSION = re.compile(r"^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +MAX_METADATA_BYTES = 4 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 256 * 1024 * 1024 + + +class ReleaseVerificationError(RuntimeError): + """The public release does not equal the reviewed local build.""" + + +class PublicationPending(ReleaseVerificationError): + """The exact immutable PyPI publication is not visible yet.""" + + +@dataclass(frozen=True) +class Artifact: + """One local immutable distribution file.""" + + name: str + package_type: str + body: bytes + sha256: str + + @property + def size(self) -> int: + return len(self.body) + + +Fetch = Callable[[str, int], bytes] + + +def _canonical_project(value: str) -> str: + return re.sub(r"[-_.]+", "-", value).lower() + + +def _fetch_url(url: str, limit: int) -> bytes: + request = urllib.request.Request( + url, + headers={"User-Agent": "openadapt-evals-release-verifier/1"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + body = response.read(limit + 1) + except urllib.error.HTTPError as exc: + if exc.code == 404: + raise PublicationPending(f"PyPI has not published {url}") from exc + raise ReleaseVerificationError(f"PyPI request failed with HTTP {exc.code}: {url}") from exc + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise PublicationPending(f"PyPI request is not ready: {url}: {exc}") from exc + if len(body) > limit: + raise ReleaseVerificationError(f"PyPI response exceeds the size limit: {url}") + return body + + +def _local_artifacts(directory: Path) -> dict[str, Artifact]: + if not directory.is_dir() or directory.is_symlink(): + raise ReleaseVerificationError(f"distribution directory is missing or invalid: {directory}") + + artifacts: dict[str, Artifact] = {} + for path in sorted(directory.iterdir()): + if not path.is_file() or path.is_symlink(): + raise ReleaseVerificationError(f"unexpected local distribution: {path.name}") + if path.name.endswith(".whl"): + package_type = "bdist_wheel" + elif path.name.endswith(".tar.gz"): + package_type = "sdist" + else: + raise ReleaseVerificationError(f"unexpected local distribution: {path.name}") + body = path.read_bytes() + if not body: + raise ReleaseVerificationError(f"local distribution is empty: {path.name}") + if len(body) > MAX_ARTIFACT_BYTES: + raise ReleaseVerificationError( + f"local distribution exceeds the size limit: {path.name}" + ) + artifacts[path.name] = Artifact( + name=path.name, + package_type=package_type, + body=body, + sha256=hashlib.sha256(body).hexdigest(), + ) + + if len(artifacts) != 2 or {artifact.package_type for artifact in artifacts.values()} != { + "bdist_wheel", + "sdist", + }: + raise ReleaseVerificationError("local release must contain exactly one wheel and one sdist") + return artifacts + + +def _metadata_object(body: bytes) -> dict[str, Any]: + try: + value = json.loads(body) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise ReleaseVerificationError("PyPI returned invalid JSON metadata") from exc + if not isinstance(value, dict): + raise ReleaseVerificationError("PyPI metadata is not an object") + return value + + +def _published_files(document: dict[str, Any]) -> dict[str, dict[str, Any]]: + urls = document.get("urls") + if not isinstance(urls, list): + raise ReleaseVerificationError("PyPI metadata has no release file inventory") + + published: dict[str, dict[str, Any]] = {} + for entry in urls: + if not isinstance(entry, dict): + raise ReleaseVerificationError("PyPI release file metadata is invalid") + filename = entry.get("filename") + if ( + not isinstance(filename, str) + or not filename + or Path(filename).name != filename + or filename in published + ): + raise ReleaseVerificationError( + f"PyPI release filename is invalid or duplicated: {filename!r}" + ) + published[filename] = entry + return published + + +def _artifact_url(entry: dict[str, Any], filename: str) -> str: + value = entry.get("url") + try: + parsed = urlsplit(value) if isinstance(value, str) else None + port = parsed.port if parsed is not None else None + except ValueError as exc: + raise ReleaseVerificationError(f"PyPI artifact URL is invalid: {filename}") from exc + if ( + parsed is None + or parsed.scheme != "https" + or parsed.netloc != "files.pythonhosted.org" + or parsed.username is not None + or parsed.password is not None + or port is not None + or parsed.query + or parsed.fragment + or Path(parsed.path).name != filename + ): + raise ReleaseVerificationError(f"PyPI artifact URL is invalid: {filename}") + return value + + +def verify_pypi_release( + directory: Path, + version: str, + *, + fetch: Fetch = _fetch_url, +) -> None: + """Require the exact wheel and sdist metadata, hashes, sizes, and bytes.""" + + if STABLE_VERSION.fullmatch(version) is None: + raise ReleaseVerificationError(f"version is not an exact stable X.Y.Z value: {version!r}") + local = _local_artifacts(directory) + wheel = next(name for name in local if name.endswith(".whl")) + sdist = next(name for name in local if name.endswith(".tar.gz")) + normalized_release = f"openadapt_evals-{version}" + if not wheel.startswith(f"{normalized_release}-"): + raise ReleaseVerificationError("local wheel does not identify the exact release") + if sdist != f"{normalized_release}.tar.gz": + raise ReleaseVerificationError("local sdist does not identify the exact release") + metadata_url = f"https://pypi.org/pypi/{PYPI_PROJECT}/{quote(version, safe='')}/json" + document = _metadata_object(fetch(metadata_url, MAX_METADATA_BYTES)) + info = document.get("info") + if ( + not isinstance(info, dict) + or _canonical_project(str(info.get("name") or "")) != PYPI_PROJECT + or info.get("version") != version + ): + raise ReleaseVerificationError( + "PyPI metadata does not identify the requested package version" + ) + + published = _published_files(document) + expected_names = set(local) + published_names = set(published) + extras = sorted(published_names - expected_names) + if extras: + raise ReleaseVerificationError(f"PyPI has unexpected immutable release files: {extras}") + missing = sorted(expected_names - published_names) + if missing: + raise PublicationPending(f"PyPI release files are not visible yet: {missing}") + + for filename, artifact in local.items(): + entry = published[filename] + digests = entry.get("digests") + remote_digest = digests.get("sha256") if isinstance(digests, dict) else None + remote_size = entry.get("size") + if ( + not isinstance(remote_digest, str) + or SHA256.fullmatch(remote_digest) is None + or isinstance(remote_size, bool) + or not isinstance(remote_size, int) + or remote_size <= 0 + or entry.get("packagetype") != artifact.package_type + or entry.get("yanked") is not False + ): + raise ReleaseVerificationError(f"PyPI release file metadata is invalid: {filename}") + if remote_digest != artifact.sha256 or remote_size != artifact.size: + raise ReleaseVerificationError( + f"PyPI release file metadata differs from the build: {filename}" + ) + remote_url = _artifact_url(entry, filename) + public_body = fetch(remote_url, min(MAX_ARTIFACT_BYTES, artifact.size + 1)) + if public_body != artifact.body: + raise ReleaseVerificationError(f"PyPI release bytes differ from the build: {filename}") + + +def _verify_with_wait( + directory: Path, + version: str, + *, + wait_seconds: int, + poll_seconds: int, +) -> None: + deadline = time.monotonic() + wait_seconds + while True: + try: + verify_pypi_release(directory, version) + return + except PublicationPending: + if time.monotonic() >= deadline: + raise + time.sleep(poll_seconds) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, default=Path("dist")) + parser.add_argument("--version", required=True) + parser.add_argument("--wait-seconds", type=int, default=300) + parser.add_argument("--poll-seconds", type=int, default=10) + args = parser.parse_args() + if args.wait_seconds < 0 or args.poll_seconds <= 0: + parser.error("wait seconds must be nonnegative and poll seconds must be positive") + try: + _verify_with_wait( + args.directory, + args.version, + wait_seconds=args.wait_seconds, + poll_seconds=args.poll_seconds, + ) + except (OSError, ReleaseVerificationError) as exc: + parser.exit(1, f"{exc}\n") + print(f"Verified immutable PyPI bytes for {PYPI_PROJECT} {args.version}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 12be996..4a90186 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -121,8 +121,23 @@ def test_release_configuration_is_fail_closed() -> None: assert "pypa/gh-action-pypi-publish@" in workflow assert "id-token: write" in workflow assert "skip-existing: true" in workflow + pypi_job = workflow.split(" publish-pypi:", 1)[1].split( + " publish-github-release:", 1 + )[0] + assert pypi_job.index("skip-existing: true") < pypi_job.index( + "python scripts/verify_pypi_release.py" + ) + assert "Verify immutable PyPI publication bytes" in pypi_job + assert '--directory dist' in pypi_job + assert '--version "$version"' in pypi_job + assert '--wait-seconds 300' in pypi_job assert "first_release_heading=$(grep -Em1" in workflow assert "gh release create" in workflow + assert "--json author,isDraft,isPrerelease,tagName" in publish_github_job + assert ( + "test \"$(jq -r '.author.login' <<<\"$state\")\" = " + "'openadapt-release[bot]'" + ) in publish_github_job assert "ADMIN_TOKEN" not in workflow assert "PYPI_API_TOKEN" not in workflow assert "secrets.GITHUB_TOKEN" not in workflow diff --git a/tests/test_verify_pypi_release.py b/tests/test_verify_pypi_release.py new file mode 100644 index 0000000..4618a0a --- /dev/null +++ b/tests/test_verify_pypi_release.py @@ -0,0 +1,170 @@ +"""Tests for exact immutable PyPI release verification.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from scripts.verify_pypi_release import ( + MAX_METADATA_BYTES, + PYPI_PROJECT, + PublicationPending, + ReleaseVerificationError, + verify_pypi_release, +) + +VERSION = "9.8.7" + + +def _fixture(tmp_path: Path) -> tuple[Path, dict[str, bytes], dict[str, bytes]]: + directory = tmp_path / "dist" + directory.mkdir() + local = { + f"openadapt_evals-{VERSION}-py3-none-any.whl": b"reviewed wheel bytes", + f"openadapt_evals-{VERSION}.tar.gz": b"reviewed sdist bytes", + } + urls = [] + bodies: dict[str, bytes] = {} + for filename, body in local.items(): + path = directory / filename + path.write_bytes(body) + url = f"https://files.pythonhosted.org/packages/test/{filename}" + bodies[url] = body + urls.append( + { + "filename": filename, + "url": url, + "size": len(body), + "digests": {"sha256": hashlib.sha256(body).hexdigest()}, + "packagetype": "bdist_wheel" if filename.endswith(".whl") else "sdist", + "yanked": False, + } + ) + metadata_url = f"https://pypi.org/pypi/{PYPI_PROJECT}/{VERSION}/json" + bodies[metadata_url] = json.dumps( + { + "info": {"name": "openadapt-evals", "version": VERSION}, + "urls": urls, + } + ).encode() + return directory, local, bodies + + +def _fetch(bodies: dict[str, bytes]): + def fetch(url: str, limit: int) -> bytes: + body = bodies[url] + assert len(body) <= limit + return body + + return fetch + + +def _metadata(bodies: dict[str, bytes]) -> tuple[str, dict]: + metadata_url = f"https://pypi.org/pypi/{PYPI_PROJECT}/{VERSION}/json" + return metadata_url, json.loads(bodies[metadata_url]) + + +def test_exact_public_files_match_names_metadata_hashes_and_bytes(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + calls: list[tuple[str, int]] = [] + + def fetch(url: str, limit: int) -> bytes: + calls.append((url, limit)) + return bodies[url] + + verify_pypi_release(directory, VERSION, fetch=fetch) + + assert calls[0] == ( + f"https://pypi.org/pypi/{PYPI_PROJECT}/{VERSION}/json", + MAX_METADATA_BYTES, + ) + assert {url for url, _ in calls[1:]} == { + url for url in bodies if url.startswith("https://files.pythonhosted.org/") + } + + +def test_missing_public_file_is_pending_for_bounded_recovery(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + metadata["urls"].pop() + bodies[metadata_url] = json.dumps(metadata).encode() + + with pytest.raises(PublicationPending, match="not visible yet"): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + + +def test_extra_immutable_public_file_is_refused(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + metadata["urls"].append(dict(metadata["urls"][0], filename="openadapt_evals-9.8.7-extra.whl")) + bodies[metadata_url] = json.dumps(metadata).encode() + + with pytest.raises(ReleaseVerificationError, match="unexpected immutable"): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + + +@pytest.mark.parametrize("field", ["digest", "size", "type", "yanked"]) +def test_changed_public_metadata_is_refused(tmp_path: Path, field: str) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + entry = metadata["urls"][0] + if field == "digest": + entry["digests"]["sha256"] = "0" * 64 + elif field == "size": + entry["size"] += 1 + elif field == "type": + entry["packagetype"] = "sdist" + else: + entry["yanked"] = True + bodies[metadata_url] = json.dumps(metadata).encode() + + with pytest.raises(ReleaseVerificationError, match="metadata"): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + + +def test_changed_public_bytes_are_refused_even_when_metadata_matches(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + artifact_url = next(url for url in bodies if url.startswith("https://files.pythonhosted.org/")) + original = bodies[artifact_url] + bodies[artifact_url] = b"X" * len(original) + + with pytest.raises(ReleaseVerificationError, match="bytes differ"): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + + +@pytest.mark.parametrize("mutation", ["project", "version", "host", "duplicate"]) +def test_wrong_release_identity_or_file_source_is_refused(tmp_path: Path, mutation: str) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + if mutation == "project": + metadata["info"]["name"] = "other-project" + elif mutation == "version": + metadata["info"]["version"] = "9.8.8" + elif mutation == "host": + metadata["urls"][0]["url"] = "https://example.com/package.whl" + else: + metadata["urls"].append(dict(metadata["urls"][0])) + bodies[metadata_url] = json.dumps(metadata).encode() + + with pytest.raises(ReleaseVerificationError): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + + +def test_local_release_requires_only_one_wheel_and_one_sdist(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + (directory / "notes.txt").write_text("unexpected", encoding="utf-8") + + with pytest.raises(ReleaseVerificationError, match="unexpected local"): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + + +def test_local_distribution_names_must_identify_the_exact_version(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + wheel = next(directory.glob("*.whl")) + wheel.rename(directory / wheel.name.replace(VERSION, "9.8.8")) + + with pytest.raises(ReleaseVerificationError, match="wheel does not identify"): + verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) From 788e1317d00a2066297dbb26ced7a2b5e0384df4 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:36:49 -0400 Subject: [PATCH 3/6] ci: preflight immutable eval publication --- .github/workflows/release.yml | 14 ++++++ scripts/verify_pypi_release.py | 39 +++++++++++--- tests/test_release_contract.py | 13 +++-- tests/test_verify_pypi_release.py | 84 ++++++++++++++++++++++++++++++- 4 files changed, 139 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2507253..dd15acb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -258,6 +258,20 @@ jobs: - name: Verify the public distribution boundary run: python scripts/check_source_boundary.py --require-dist + - name: Refuse conflicting immutable PyPI files + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + version="${RELEASE_TAG#v}" + test "$RELEASE_TAG" = "v${version}" + python scripts/verify_pypi_release.py \ + --directory dist \ + --version "$version" \ + --allow-matching-subset \ + --wait-seconds 300 \ + --poll-seconds 10 + - name: Publish to PyPI with OIDC uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: diff --git a/scripts/verify_pypi_release.py b/scripts/verify_pypi_release.py index 68c081f..8d96e6c 100644 --- a/scripts/verify_pypi_release.py +++ b/scripts/verify_pypi_release.py @@ -31,6 +31,10 @@ class PublicationPending(ReleaseVerificationError): """The exact immutable PyPI publication is not visible yet.""" +class PublicationAbsent(PublicationPending): + """The exact version or one of its inventoried files is absent.""" + + @dataclass(frozen=True) class Artifact: """One local immutable distribution file.""" @@ -62,7 +66,7 @@ def _fetch_url(url: str, limit: int) -> bytes: body = response.read(limit + 1) except urllib.error.HTTPError as exc: if exc.code == 404: - raise PublicationPending(f"PyPI has not published {url}") from exc + raise PublicationAbsent(f"PyPI has not published {url}") from exc raise ReleaseVerificationError(f"PyPI request failed with HTTP {exc.code}: {url}") from exc except (urllib.error.URLError, TimeoutError, OSError) as exc: raise PublicationPending(f"PyPI request is not ready: {url}: {exc}") from exc @@ -167,8 +171,9 @@ def verify_pypi_release( version: str, *, fetch: Fetch = _fetch_url, + allow_matching_subset: bool = False, ) -> None: - """Require the exact wheel and sdist metadata, hashes, sizes, and bytes.""" + """Verify all public files; optionally allow an absent or matching subset.""" if STABLE_VERSION.fullmatch(version) is None: raise ReleaseVerificationError(f"version is not an exact stable X.Y.Z value: {version!r}") @@ -181,7 +186,13 @@ def verify_pypi_release( if sdist != f"{normalized_release}.tar.gz": raise ReleaseVerificationError("local sdist does not identify the exact release") metadata_url = f"https://pypi.org/pypi/{PYPI_PROJECT}/{quote(version, safe='')}/json" - document = _metadata_object(fetch(metadata_url, MAX_METADATA_BYTES)) + try: + metadata_body = fetch(metadata_url, MAX_METADATA_BYTES) + except PublicationAbsent: + if allow_matching_subset: + return + raise + document = _metadata_object(metadata_body) info = document.get("info") if ( not isinstance(info, dict) @@ -199,10 +210,12 @@ def verify_pypi_release( if extras: raise ReleaseVerificationError(f"PyPI has unexpected immutable release files: {extras}") missing = sorted(expected_names - published_names) - if missing: + if missing and not allow_matching_subset: raise PublicationPending(f"PyPI release files are not visible yet: {missing}") for filename, artifact in local.items(): + if filename not in published: + continue entry = published[filename] digests = entry.get("digests") remote_digest = digests.get("sha256") if isinstance(digests, dict) else None @@ -233,11 +246,16 @@ def _verify_with_wait( *, wait_seconds: int, poll_seconds: int, + allow_matching_subset: bool, ) -> None: deadline = time.monotonic() + wait_seconds while True: try: - verify_pypi_release(directory, version) + verify_pypi_release( + directory, + version, + allow_matching_subset=allow_matching_subset, + ) return except PublicationPending: if time.monotonic() >= deadline: @@ -251,6 +269,11 @@ def main() -> int: parser.add_argument("--version", required=True) parser.add_argument("--wait-seconds", type=int, default=300) parser.add_argument("--poll-seconds", type=int, default=10) + parser.add_argument( + "--allow-matching-subset", + action="store_true", + help="permit an absent release or byte-identical subset before publication", + ) args = parser.parse_args() if args.wait_seconds < 0 or args.poll_seconds <= 0: parser.error("wait seconds must be nonnegative and poll seconds must be positive") @@ -260,10 +283,14 @@ def main() -> int: args.version, wait_seconds=args.wait_seconds, poll_seconds=args.poll_seconds, + allow_matching_subset=args.allow_matching_subset, ) except (OSError, ReleaseVerificationError) as exc: parser.exit(1, f"{exc}\n") - print(f"Verified immutable PyPI bytes for {PYPI_PROJECT} {args.version}.") + scope = ( + "existing immutable PyPI bytes" if args.allow_matching_subset else "immutable PyPI bytes" + ) + print(f"Verified {scope} for {PYPI_PROJECT} {args.version}.") return 0 diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 4a90186..9c1e9ff 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -124,9 +124,16 @@ def test_release_configuration_is_fail_closed() -> None: pypi_job = workflow.split(" publish-pypi:", 1)[1].split( " publish-github-release:", 1 )[0] - assert pypi_job.index("skip-existing: true") < pypi_job.index( - "python scripts/verify_pypi_release.py" - ) + preflight = pypi_job.index("Refuse conflicting immutable PyPI files") + publish = pypi_job.index("pypa/gh-action-pypi-publish@") + postflight = pypi_job.index("Verify immutable PyPI publication bytes") + assert preflight < publish < postflight + preflight_body = pypi_job[preflight:publish] + postflight_body = pypi_job[postflight:] + assert "python scripts/verify_pypi_release.py" in preflight_body + assert "--allow-matching-subset" in preflight_body + assert "python scripts/verify_pypi_release.py" in postflight_body + assert "--allow-matching-subset" not in postflight_body assert "Verify immutable PyPI publication bytes" in pypi_job assert '--directory dist' in pypi_job assert '--version "$version"' in pypi_job diff --git a/tests/test_verify_pypi_release.py b/tests/test_verify_pypi_release.py index 4618a0a..b61a146 100644 --- a/tests/test_verify_pypi_release.py +++ b/tests/test_verify_pypi_release.py @@ -11,6 +11,7 @@ from scripts.verify_pypi_release import ( MAX_METADATA_BYTES, PYPI_PROJECT, + PublicationAbsent, PublicationPending, ReleaseVerificationError, verify_pypi_release, @@ -96,14 +97,93 @@ def test_missing_public_file_is_pending_for_bounded_recovery(tmp_path: Path) -> verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) +def test_matching_public_subset_is_accepted_before_publication(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + missing = metadata["urls"].pop() + bodies.pop(missing["url"]) + bodies[metadata_url] = json.dumps(metadata).encode() + + verify_pypi_release( + directory, + VERSION, + fetch=_fetch(bodies), + allow_matching_subset=True, + ) + + +def test_absent_release_is_accepted_only_before_publication(tmp_path: Path) -> None: + directory, _, _ = _fixture(tmp_path) + + def absent(_url: str, _limit: int) -> bytes: + raise PublicationAbsent("not published") + + verify_pypi_release( + directory, + VERSION, + fetch=absent, + allow_matching_subset=True, + ) + with pytest.raises(PublicationAbsent): + verify_pypi_release(directory, VERSION, fetch=absent) + + +def test_empty_release_inventory_is_accepted_before_publication(tmp_path: Path) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + metadata["urls"] = [] + bodies[metadata_url] = json.dumps(metadata).encode() + + verify_pypi_release( + directory, + VERSION, + fetch=_fetch(bodies), + allow_matching_subset=True, + ) + + def test_extra_immutable_public_file_is_refused(tmp_path: Path) -> None: directory, _, bodies = _fixture(tmp_path) metadata_url, metadata = _metadata(bodies) metadata["urls"].append(dict(metadata["urls"][0], filename="openadapt_evals-9.8.7-extra.whl")) bodies[metadata_url] = json.dumps(metadata).encode() - with pytest.raises(ReleaseVerificationError, match="unexpected immutable"): - verify_pypi_release(directory, VERSION, fetch=_fetch(bodies)) + for allow_matching_subset in (False, True): + with pytest.raises(ReleaseVerificationError, match="unexpected immutable"): + verify_pypi_release( + directory, + VERSION, + fetch=_fetch(bodies), + allow_matching_subset=allow_matching_subset, + ) + + +@pytest.mark.parametrize("field", ["digest", "size", "type", "yanked", "bytes"]) +def test_changed_existing_subset_is_refused_before_publication(tmp_path: Path, field: str) -> None: + directory, _, bodies = _fixture(tmp_path) + metadata_url, metadata = _metadata(bodies) + missing = metadata["urls"].pop() + bodies.pop(missing["url"]) + entry = metadata["urls"][0] + if field == "digest": + entry["digests"]["sha256"] = "0" * 64 + elif field == "size": + entry["size"] += 1 + elif field == "type": + entry["packagetype"] = "sdist" + elif field == "yanked": + entry["yanked"] = True + else: + bodies[entry["url"]] = b"X" * entry["size"] + bodies[metadata_url] = json.dumps(metadata).encode() + + with pytest.raises(ReleaseVerificationError): + verify_pypi_release( + directory, + VERSION, + fetch=_fetch(bodies), + allow_matching_subset=True, + ) @pytest.mark.parametrize("field", ["digest", "size", "type", "yanked"]) From 23bbed2fb9a3e35daa1862f3f52d4ecd5f721224 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:55:07 -0400 Subject: [PATCH 4/6] Format release contract tests --- tests/test_release_contract.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 9c1e9ff..28e3116 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -14,8 +14,7 @@ def _write_release_files(root: Path, project_version: str, lock_version: str) -> None: (root / "pyproject.toml").write_text( - '[project]\nname = "example-package"\n' - f'version = "{project_version}"\n', + f'[project]\nname = "example-package"\nversion = "{project_version}"\n', encoding="utf-8", ) (root / "uv.lock").write_text( @@ -59,9 +58,7 @@ def test_sync_changes_only_editable_root_and_is_idempotent(tmp_path: Path) -> No def test_release_configuration_is_fail_closed() -> None: metadata = (ROOT / "pyproject.toml").read_text(encoding="utf-8") workflow = (ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") - test_workflow = (ROOT / ".github/workflows/test.yml").read_text( - encoding="utf-8" - ) + test_workflow = (ROOT / ".github/workflows/test.yml").read_text(encoding="utf-8") assert "major_on_zero = false" in metadata assert "allow_zero_version = true" in metadata assert ( @@ -69,9 +66,12 @@ def test_release_configuration_is_fail_closed() -> None: "python scripts/verify_release_lock.py --write && " "git add uv.lock && uv build" ) in metadata - assert metadata.index("python -m pip install uv==0.11.29") < metadata.index( - "python scripts/verify_release_lock.py --write" - ) < metadata.index("git add uv.lock") < metadata.index("uv build") + assert ( + metadata.index("python -m pip install uv==0.11.29") + < metadata.index("python scripts/verify_release_lock.py --write") + < metadata.index("git add uv.lock") + < metadata.index("uv build") + ) assert "environment: release-identity" in workflow assert "environment: pypi" in workflow assert "actions/create-github-app-token@" in workflow @@ -109,9 +109,7 @@ def test_release_configuration_is_fail_closed() -> None: app_pushes = [ line.strip() for line in workflow.splitlines() if line.strip().startswith("git push") ] - assert app_pushes == [ - 'git push origin "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}"' - ] + assert app_pushes == ['git push origin "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}"'] assert "refs/heads/main:refs/heads/main" not in workflow assert "gh pr create" not in workflow assert "automation/release" not in workflow @@ -121,9 +119,7 @@ def test_release_configuration_is_fail_closed() -> None: assert "pypa/gh-action-pypi-publish@" in workflow assert "id-token: write" in workflow assert "skip-existing: true" in workflow - pypi_job = workflow.split(" publish-pypi:", 1)[1].split( - " publish-github-release:", 1 - )[0] + pypi_job = workflow.split(" publish-pypi:", 1)[1].split(" publish-github-release:", 1)[0] preflight = pypi_job.index("Refuse conflicting immutable PyPI files") publish = pypi_job.index("pypa/gh-action-pypi-publish@") postflight = pypi_job.index("Verify immutable PyPI publication bytes") @@ -135,15 +131,14 @@ def test_release_configuration_is_fail_closed() -> None: assert "python scripts/verify_pypi_release.py" in postflight_body assert "--allow-matching-subset" not in postflight_body assert "Verify immutable PyPI publication bytes" in pypi_job - assert '--directory dist' in pypi_job + assert "--directory dist" in pypi_job assert '--version "$version"' in pypi_job - assert '--wait-seconds 300' in pypi_job + assert "--wait-seconds 300" in pypi_job assert "first_release_heading=$(grep -Em1" in workflow assert "gh release create" in workflow assert "--json author,isDraft,isPrerelease,tagName" in publish_github_job assert ( - "test \"$(jq -r '.author.login' <<<\"$state\")\" = " - "'openadapt-release[bot]'" + "test \"$(jq -r '.author.login' <<<\"$state\")\" = 'openadapt-release[bot]'" ) in publish_github_job assert "ADMIN_TOKEN" not in workflow assert "PYPI_API_TOKEN" not in workflow From 5a95ad4bc92d35d7501dee81470b38454fa27376 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 27 Aug 2026 13:28:07 -0400 Subject: [PATCH 5/6] feat(evidence): emit qualified lifecycle evidence --- CHANGELOG.md | 8 + README.md | 57 +- openadapt_evals/production_evidence.py | 1220 +++++++++++++++++ openadapt_evals/qualification_evidence.py | 917 +++++++++++++ pyproject.toml | 4 +- scripts/build_production_evidence.py | 181 +++ scripts/import_production_acceptance.py | 2 +- tests/test_build_production_evidence.py | 81 ++ tests/test_import_production_acceptance.py | 6 +- tests/test_production_evidence.py | 510 +++++++ .../test_public_evidence_workflow_boundary.py | 35 + tests/test_qualification_evidence_v3.py | 342 +++++ 12 files changed, 3346 insertions(+), 17 deletions(-) create mode 100644 openadapt_evals/production_evidence.py create mode 100644 openadapt_evals/qualification_evidence.py create mode 100644 scripts/build_production_evidence.py create mode 100644 tests/test_build_production_evidence.py create mode 100644 tests/test_production_evidence.py create mode 100644 tests/test_public_evidence_workflow_boundary.py create mode 100644 tests/test_qualification_evidence_v3.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7053b61..6852803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,18 @@ ## v0.94.1 (2026-08-26) +### Features + +- **evidence**: Build signed six-class campaigns and content-addressed public lifecycle summaries. + ### Bug Fixes - **evidence**: Match the retention digests the Cloud writer emits (#311) ([`df12b3c`](https://github.com/OpenAdaptAI/openadapt-evals/commit/df12b3ce72ad02992a7c7e389d8ccf2cfd01774c)) +### Continuous Integration + +- **release**: Keep a durable App-authored draft and verify its exact bytes before publication. + ## v0.94.0 (2026-08-26) diff --git a/README.md b/README.md index 0527818..6a50355 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # OpenAdapt Evals > [!IMPORTANT] -> **Status: Research infrastructure, not a required part of the product.** This -> package is evaluation and benchmarking tooling for GUI agents and for the -> OpenAdapt demonstration compiler. It is evidence-generating research work. It -> is not required to record, compile, or replay a workflow, and no end user -> needs it installed. +> **Lifecycle: Support.** This maintained public repository produces +> qualification and release evidence for OpenAdapt. The seven product targets +> use signed admissions to determine their Production state. Evals supplies +> evidence for those decisions and remains a separate operational tool. > -> The OpenAdapt product is the governed demonstration compiler, +> End users don't install Evals to run a workflow. The OpenAdapt product is the +> governed demonstration compiler, > [`openadapt-flow`](https://github.com/OpenAdaptAI/openadapt-flow), installed > via the [`OpenAdapt`](https://github.com/OpenAdaptAI/openadapt) launcher > (`pip install openadapt`). It compiles a demonstrated GUI workflow into a @@ -43,10 +43,45 @@ dimensions that matter to a governed compiler: - **Cost and model-call accounting**: model calls, latency, and dollar cost per successful task, including the ~0-model-call healthy replay path. -This is internal research tooling. It is not a packaged end-user product and it -is not needed to use OpenAdapt. If you want to record, compile, and replay a -workflow, use the [`OpenAdapt`](https://github.com/OpenAdaptAI/openadapt) -launcher, not this repository. +Operators use this package to run qualification and publication gates. If you +want to record, compile, and replay a workflow, use the +[`OpenAdapt`](https://github.com/OpenAdaptAI/openadapt) launcher. + +### Qualification and release evidence + +The v3 campaign producer records six required classes for each task: healthy, +safe halt, idempotency replay, uncertain delivery, declared attended, and +governed repair. Each task and condition needs at least three signed trials. +Evals derives silent incorrect success and over-halt from the signed runner, +observer, and delivery facts. It doesn't trust author-supplied failure counts. + +Campaign payloads and trial receipts stay inside the approved private evidence +boundary. A public lifecycle summary uses the remote-safe decision receipt, +the public qualification admission, and the production acceptance manifest. It +contains aggregate class counts and commitments, without task names, +application or environment values, or live identities. + +`scripts/build_production_evidence.py` builds that summary and the paired +content-addressed v2 references. The pair command preserves the raw Sigstore +bundle bytes. Run it only after the referenced objects exist in an exact merged +registry commit. The summary uses a later registry append, which avoids a +commit-hash cycle. + +```bash +python scripts/build_production_evidence.py summary \ + --input public-summary-input.json \ + --output production-acceptance-summary.json + +python scripts/build_production_evidence.py pair \ + --kind production-acceptance-summary \ + --object production-acceptance-summary.json \ + --sigstore-bundle production-acceptance-summary.sigstore.json \ + --registry-source-commit "$MERGED_REGISTRY_COMMIT" \ + --registry-revision "$REGISTRY_REVISION" \ + --registry-head-sha256 "$REGISTRY_HEAD_SHA256" \ + --output-root public-registry-candidate \ + --references-output production-acceptance-summary.references.json +``` ### Relationship to the rest of OpenAdapt @@ -79,7 +114,7 @@ does not claim maturity it has not measured. ## What is inside -- **openadapt-flow evaluation** (`openadapt_evals/flow/`): the paradigm-correct +- **openadapt-flow evaluation** (`openadapt_evals/flow/`): the release-path eval for a demonstration compiler. A `replay` runner compiles one demonstration into an openadapt-flow bundle and replays it against the WAA in-guest server with roughly zero model calls, and a `hybrid` agent runs compiled replay first diff --git a/openadapt_evals/production_evidence.py b/openadapt_evals/production_evidence.py new file mode 100644 index 0000000..132d9c1 --- /dev/null +++ b/openadapt_evals/production_evidence.py @@ -0,0 +1,1220 @@ +"""Build content-addressed references for public production evidence. + +The public evidence registry stores immutable JSON objects in ``OpenAdaptAI/.github``. +This module does not publish those objects. It builds the canonical bytes, semantic +identities, registry-entry commitments, and exact v2 references that a publisher can +write after it has obtained the current registry metadata. +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from fnmatch import fnmatchcase +from pathlib import PurePosixPath +from typing import Any, Mapping, Sequence + +OBJECT_REFERENCE_SCHEMA = "openadapt.production-evidence-object-reference/v2" +EVIDENCE_REPOSITORY = "OpenAdaptAI/.github" +EVIDENCE_REPOSITORY_ID = "858454062" +EVIDENCE_REPOSITORY_OWNER_ID = "132681217" +SIGSTORE_BUNDLE_MEDIA_TYPE = "application/vnd.dev.sigstore.bundle.v0.3+json" +OBJECT_PATH_PREFIX = "production-evidence/objects/sha256" + +REGISTRY_ENTRY_DIGEST_DOMAIN = b"OpenAdapt production evidence registry entry v1\0" +REGULAR_SEMANTIC_IDENTITY_DOMAIN = b"OpenAdapt production evidence semantic identity v1\0" +BUNDLE_SEMANTIC_IDENTITY_DOMAIN = b"OpenAdapt production evidence Sigstore bundle identity v1\0" + +SIGNER_REGISTRY_POINTER_SCHEMA = "openadapt.qualification-signer-registry-pointer/v1" +SIGNER_REGISTRY_IDENTITY_DOMAIN = b"OpenAdapt qualification signer registry v2\0" +PRODUCTION_ACCEPTANCE_SUMMARY_SCHEMA = "openadapt.production-lifecycle-evidence-summary/v2" +PRODUCTION_EVIDENCE_IDENTITY_DOMAIN = b"OpenAdapt production acceptance evidence identity v2\0" +PUBLICATION_STAGING_SCHEMA = "openadapt.production-release-staging-evidence/v1" +PUBLICATION_STAGING_DIGEST_DOMAIN = b"OpenAdapt production release staging evidence v1\0" +TAG_RULESET_SCHEMA = "openadapt.production-release-tag-ruleset/v1" +TAG_RULESETS_DIGEST_DOMAIN = b"OpenAdapt production release tag rulesets v1\0" +DECISION_RECEIPT_SCHEMA = "openadapt.qualification-evidence-decision-receipt/v1" +DECISION_CAMPAIGN_SUMMARY_SCHEMA = "openadapt.qualification-evidence-decision-campaign-summary/v1" + +_SHA256 = re.compile(r"^sha256:[a-f0-9]{64}$") +_HEX40 = re.compile(r"^[a-f0-9]{40}$") +_KIND = re.compile(r"^[a-z0-9][a-z0-9-]*$") + + +@dataclass(frozen=True) +class EvidenceKind: + """The approved schema and media type for one regular evidence object.""" + + schema_version: str + media_type: str + + +@dataclass(frozen=True) +class EvidenceObjectPair: + """One regular object followed by its raw Sigstore bundle.""" + + objects: tuple[bytes, bytes] + references: tuple[dict[str, Any], dict[str, Any]] + + +REGULAR_EVIDENCE_KINDS: Mapping[str, EvidenceKind] = { + "qualification-release": EvidenceKind( + "openadapt.qualification-release/v1", + "application/vnd.openadapt.qualification-release+json;version=1", + ), + "production-acceptance-manifest": EvidenceKind( + "openadapt.production-acceptance/v2", + "application/vnd.openadapt.production-acceptance+json;version=2", + ), + "production-acceptance-summary": EvidenceKind( + "openadapt.production-lifecycle-evidence-summary/v2", + "application/vnd.openadapt.production-lifecycle-evidence-summary+json;version=2", + ), + "qualification-authority-state-receipt": EvidenceKind( + "openadapt.qualification-authority-state-receipt/v2", + "application/vnd.openadapt.qualification-authority-state-receipt+json;version=2", + ), + "qualification-revocation-state-receipt": EvidenceKind( + "openadapt.qualification-revocation-state-receipt/v1", + "application/vnd.openadapt.qualification-revocation-state-receipt+json;version=1", + ), + "production-current-default": EvidenceKind( + "openadapt.production-current-default/v1", + "application/vnd.openadapt.production-current-default+json;version=1", + ), + "production-deployment-observation": EvidenceKind( + "openadapt.production-deployment-observation/v1", + "application/vnd.openadapt.production-deployment-observation+json;version=1", + ), + "production-lifecycle-checkpoint": EvidenceKind( + "openadapt.production-lifecycle-checkpoint/v1", + "application/vnd.openadapt.production-lifecycle-checkpoint+json;version=1", + ), + "production-cloud-deploy-authorization": EvidenceKind( + "openadapt.production-cloud-deploy-authorization/v1", + "application/vnd.openadapt.production-cloud-deploy-authorization+json;version=1", + ), + "production-cloud-deployment-result": EvidenceKind( + "openadapt.production-cloud-deployment-result/v1", + "application/vnd.openadapt.production-cloud-deployment-result+json;version=1", + ), + "qualification-campaign-permit-policy": EvidenceKind( + "openadapt.qualification-campaign-permit-policy/v3", + "application/vnd.openadapt.qualification-campaign-permit-policy+json;version=3", + ), + "qualification-campaign-permit-request": EvidenceKind( + "openadapt.qualification-campaign-permit-request/v3", + "application/vnd.openadapt.qualification-campaign-permit-request+json;version=3", + ), + "qualification-campaign-permit": EvidenceKind( + "openadapt.qualification-campaign-permit/v3", + "application/vnd.openadapt.qualification-campaign-permit+json;version=3", + ), + "qualification-campaign-permit-receipt": EvidenceKind( + "openadapt.qualification-campaign-permit-receipt/v3", + "application/vnd.openadapt.qualification-campaign-permit-receipt+json;version=3", + ), + "qualification-evidence-decision-receipt": EvidenceKind( + "openadapt.qualification-evidence-decision-receipt/v1", + "application/vnd.openadapt.qualification-evidence-decision-receipt+json;version=1", + ), + "qualification-admission": EvidenceKind( + "openadapt.qualification-admission/v3", + "application/vnd.openadapt.qualification-admission+json;version=3", + ), +} + +REFERENCE_KEYS = frozenset( + { + "schema_version", + "repository", + "repository_id", + "repository_owner_id", + "registry_source_commit", + "registry_revision", + "registry_head_sha256", + "registry_entry_sha256", + "kind", + "object_schema_version", + "object_path", + "object_sha256", + "size_bytes", + "object_media_type", + "semantic_identity_sha256", + "subject_sha256", + } +) + +REGISTRY_ENTRY_KEYS: Sequence[str] = ( + "kind", + "object_media_type", + "object_path", + "object_schema_version", + "object_sha256", + "semantic_identity_sha256", + "size_bytes", + "subject_sha256", +) + +_SUMMARY_KEYS = frozenset( + { + "schema_version", + "target", + "verdict", + "claim_scope", + "acceptance_policy_sha256", + "lifecycle_policy_sha256", + "release_identity", + "release_sha256", + "artifact_inventory_sha256", + "publication_staging", + "publication_staging_sha256", + "evidence_identity_sha256", + "qualification_evidence_decision_receipt_reference", + "qualification_evidence_decision_receipt_bundle_reference", + "qualification_admission_reference", + "qualification_admission_bundle_reference", + "production_acceptance_manifest_reference", + "production_acceptance_manifest_bundle_reference", + "campaign_summary", + "authority_state_sha256", + "revocation_state_sha256", + "signer_registry_sha256", + "issued_at", + "not_before", + "expires_at", + } +) +_RELEASE_IDENTITY_KEYS = frozenset( + {"schema_version", "channel", "sequence", "previous_admission_sha256"} +) +_CAMPAIGN_CLASSES = frozenset( + { + "healthy", + "safe_halt", + "idempotency_replay", + "uncertain_delivery", + "declared_attended", + "governed_repair", + } +) +_CAMPAIGN_CLASS_SUMMARY_KEYS = frozenset( + { + "task_condition_cell_count", + "minimum_trials_per_cell", + "observed_trial_count", + "silent_incorrect_success_count", + "over_halt_count", + "unsafe_effect_count", + "blind_retry_count", + "replay_dispatch_count", + "model_call_count", + "unplanned_intervention_count", + "reconciliation_required_count", + "authenticated_bound_decision_count", + "live_target_revalidation_count", + "policy_approved_repair_count", + "approved_repair_count", + "retained_repair_evidence_count", + "unverified_direct_action_count", + } +) +_WHOLE_SECOND_UTC = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") +_DECIMAL_ID = re.compile(r"^[1-9][0-9]*$") +_TAG = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$") +_ASSET_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,255}$") +_PUBLICATION_STAGING_KEYS = frozenset( + { + "schema_version", + "repository", + "repository_id", + "draft_release_id", + "tag", + "target_commitish", + "draft", + "prerelease", + "release_app_id", + "release_app_installation_id", + "release_app_bot_user_id", + "release_author_login", + "assets", + "immutable_releases_enabled", + "tag_rulesets", + "tag_rulesets_sha256", + "observed_at", + } +) +_ASSET_KEYS = frozenset( + {"asset_id", "name", "sha256", "size_bytes", "uploader_id", "uploader_login"} +) +_EXPECTED_ASSET_KEYS = frozenset({"name", "sha256", "size_bytes"}) +_TAG_RULESET_KEYS = frozenset( + { + "schema_version", + "role", + "repository", + "repository_id", + "ruleset_id", + "name", + "target", + "enforcement", + "bypass_actors", + "conditions", + "rules", + } +) +_BYPASS_ACTOR_KEYS = frozenset({"actor_id", "actor_type", "bypass_mode"}) +_CONDITION_KEYS = frozenset({"ref_name"}) +_REF_NAME_KEYS = frozenset({"include", "exclude"}) +_RULE_KEYS = frozenset({"type"}) +_DECISION_RECEIPT_KEYS = frozenset( + { + "schema_version", + "decision_commitment_sha256", + "evidence_manifest_sha256", + "campaign_artifact_sha256", + "organization_id_sha256", + "workflow_id_sha256", + "workflow_version_id_sha256", + "bundle_version", + "bundle_sha256", + "admitted_runtime_sha256", + "action_contract_sha256", + "application_contract_sha256", + "effect_contract_sha256", + "environment_contract_sha256", + "evidence_authority_contract_sha256", + "identity_contract_sha256", + "input_contract_sha256", + "policy_contract_sha256", + "campaign_permit_sha256", + "signer_registry_sha256", + "revocation_state_sha256", + "entity_class", + "campaign_summary", + "verdict", + "issued_at", + "not_before", + "expires_at", + "issuer_key_id", + "algorithm", + "signature", + } +) +_DECISION_CAMPAIGN_SUMMARY_KEYS = frozenset( + {"schema_version", "minimum_trials_per_task_condition", "task_count", "classes"} +) + + +class ProductionEvidenceError(ValueError): + """A public evidence object or reference violates the frozen contract.""" + + +def canonical_json_bytes(value: Any) -> bytes: + """Return the canonical compact JSON encoding used by evidence digests.""" + + _validate_json_value(value, "object") + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def sha256_digest(payload: bytes) -> str: + """Return a lowercase, algorithm-prefixed SHA-256 digest.""" + + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def signer_registry_identity(registry: Mapping[str, Any]) -> str: + """Commit to a signer registry with the approved v2 identity domain.""" + + return sha256_digest(SIGNER_REGISTRY_IDENTITY_DOMAIN + canonical_json_bytes(dict(registry))) + + +def build_signer_registry_pointer( + *, + object_path: str, + object_sha256: str, + registry_identity_sha256: str, + registry_revision: int, +) -> dict[str, Any]: + """Build the exact public signer-registry pointer projection.""" + + _validate_object_path(object_path) + _require_digest(object_sha256, "object_sha256") + _require_digest(registry_identity_sha256, "registry_identity_sha256") + _require_positive_int(registry_revision, "registry_revision") + return { + "schema_version": SIGNER_REGISTRY_POINTER_SCHEMA, + "object_path": object_path, + "object_sha256": object_sha256, + "registry_identity_sha256": registry_identity_sha256, + "registry_revision": registry_revision, + } + + +def build_evidence_object_pair( + *, + kind: str, + object_value: Mapping[str, Any], + sigstore_bundle: bytes, + registry_source_commit: str, + registry_revision: int, + registry_head_sha256: str, +) -> EvidenceObjectPair: + """Build one regular reference and its immediately following bundle reference.""" + + profile = REGULAR_EVIDENCE_KINDS.get(kind) + if profile is None: + raise ProductionEvidenceError(f"evidence kind is not approved: {kind!r}") + if object_value.get("schema_version") != profile.schema_version: + raise ProductionEvidenceError(f"{kind} schema_version must be {profile.schema_version!r}") + + regular_bytes = canonical_json_bytes(dict(object_value)) + bundle_bytes = _validate_raw_json_object(sigstore_bundle, "sigstore_bundle") + regular_sha256 = sha256_digest(regular_bytes) + regular_identity = sha256_digest( + REGULAR_SEMANTIC_IDENTITY_DOMAIN + + canonical_json_bytes( + { + "kind": kind, + "object_schema_version": profile.schema_version, + "object": dict(object_value), + } + ) + ) + regular = _build_reference( + kind=kind, + object_schema_version=profile.schema_version, + object_media_type=profile.media_type, + object_bytes=regular_bytes, + object_sha256=regular_sha256, + semantic_identity_sha256=regular_identity, + subject_sha256=None, + registry_source_commit=registry_source_commit, + registry_revision=registry_revision, + registry_head_sha256=registry_head_sha256, + ) + + bundle_kind = f"{kind}-sigstore-bundle" + bundle_sha256 = sha256_digest(bundle_bytes) + bundle_identity = sha256_digest( + BUNDLE_SEMANTIC_IDENTITY_DOMAIN + + canonical_json_bytes( + { + "kind": bundle_kind, + "object_sha256": bundle_sha256, + "subject_sha256": regular_sha256, + } + ) + ) + bundle = _build_reference( + kind=bundle_kind, + object_schema_version=SIGSTORE_BUNDLE_MEDIA_TYPE, + object_media_type=SIGSTORE_BUNDLE_MEDIA_TYPE, + object_bytes=bundle_bytes, + object_sha256=bundle_sha256, + semantic_identity_sha256=bundle_identity, + subject_sha256=regular_sha256, + registry_source_commit=registry_source_commit, + registry_revision=registry_revision, + registry_head_sha256=registry_head_sha256, + ) + return EvidenceObjectPair( + objects=(regular_bytes, bundle_bytes), + references=(regular, bundle), + ) + + +def build_production_acceptance_summary( + *, + target: str, + claim_scope: str, + acceptance_policy_sha256: str, + lifecycle_policy_sha256: str, + release_identity: Mapping[str, Any], + release_sha256: str, + artifact_inventory_sha256: str, + publication_staging: Mapping[str, Any], + expected_publication_assets: Sequence[Mapping[str, Any]], + qualification_evidence_decision_receipt: Mapping[str, Any], + qualification_evidence_decision_receipt_references: Sequence[Mapping[str, Any]], + qualification_admission_references: Sequence[Mapping[str, Any]], + production_acceptance_manifest_references: Sequence[Mapping[str, Any]], + authority_state_sha256: str, + revocation_state_sha256: str, + signer_registry_sha256: str, + issued_at: str, + not_before: str, + expires_at: str, +) -> dict[str, Any]: + """Build the remote-safe v2 summary after all three input pairs exist.""" + + decision_pair = _validated_pair( + qualification_evidence_decision_receipt_references, + "qualification-evidence-decision-receipt", + ) + admission_pair = _validated_pair(qualification_admission_references, "qualification-admission") + manifest_pair = _validated_pair( + production_acceptance_manifest_references, "production-acceptance-manifest" + ) + release = dict(release_identity) + receipt = dict(qualification_evidence_decision_receipt) + campaign = _validate_decision_receipt( + receipt, + reference=decision_pair[0], + expected_signer_registry_sha256=signer_registry_sha256, + expected_revocation_state_sha256=revocation_state_sha256, + ) + staging = dict(publication_staging) + staging_sha256 = _validate_publication_staging( + staging, expected_assets=expected_publication_assets + ) + summary: dict[str, Any] = { + "schema_version": PRODUCTION_ACCEPTANCE_SUMMARY_SCHEMA, + "target": target, + "verdict": "accepted", + "claim_scope": claim_scope, + "acceptance_policy_sha256": acceptance_policy_sha256, + "lifecycle_policy_sha256": lifecycle_policy_sha256, + "release_identity": release, + "release_sha256": release_sha256, + "artifact_inventory_sha256": artifact_inventory_sha256, + "publication_staging": staging, + "publication_staging_sha256": staging_sha256, + "evidence_identity_sha256": "sha256:" + "0" * 64, + "qualification_evidence_decision_receipt_reference": decision_pair[0], + "qualification_evidence_decision_receipt_bundle_reference": decision_pair[1], + "qualification_admission_reference": admission_pair[0], + "qualification_admission_bundle_reference": admission_pair[1], + "production_acceptance_manifest_reference": manifest_pair[0], + "production_acceptance_manifest_bundle_reference": manifest_pair[1], + "campaign_summary": campaign, + "authority_state_sha256": authority_state_sha256, + "revocation_state_sha256": revocation_state_sha256, + "signer_registry_sha256": signer_registry_sha256, + "issued_at": issued_at, + "not_before": not_before, + "expires_at": expires_at, + } + summary["evidence_identity_sha256"] = _production_evidence_identity(summary) + validate_production_acceptance_summary( + summary, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=expected_publication_assets, + ) + return summary + + +def validate_production_acceptance_summary( + summary: Mapping[str, Any], + *, + qualification_evidence_decision_receipt: Mapping[str, Any], + expected_publication_assets: Sequence[Mapping[str, Any]], +) -> None: + """Validate a v2 summary without reading any private campaign payload.""" + + value = dict(summary) + if set(value) != _SUMMARY_KEYS: + missing = sorted(_SUMMARY_KEYS - set(value)) + extra = sorted(set(value) - _SUMMARY_KEYS) + raise ProductionEvidenceError( + f"production summary keys differ: missing={missing}, extra={extra}" + ) + if value["schema_version"] != PRODUCTION_ACCEPTANCE_SUMMARY_SCHEMA: + raise ProductionEvidenceError("production summary schema_version is invalid") + for field in ("target", "claim_scope"): + if not isinstance(value[field], str) or not value[field]: + raise ProductionEvidenceError(f"production summary {field} is invalid") + if value["verdict"] != "accepted": + raise ProductionEvidenceError("production summary verdict must be accepted") + for field in ( + "acceptance_policy_sha256", + "lifecycle_policy_sha256", + "release_sha256", + "artifact_inventory_sha256", + "publication_staging_sha256", + "evidence_identity_sha256", + "authority_state_sha256", + "revocation_state_sha256", + "signer_registry_sha256", + ): + _require_digest(value[field], field) + _validate_release_identity(value["release_identity"]) + _validate_summary_reference_fields(value) + staging_sha256 = _validate_publication_staging( + value["publication_staging"], expected_assets=expected_publication_assets + ) + if value["publication_staging_sha256"] != staging_sha256: + raise ProductionEvidenceError("publication_staging_sha256 is invalid") + receipt_summary = _validate_decision_receipt( + qualification_evidence_decision_receipt, + reference=value["qualification_evidence_decision_receipt_reference"], + expected_signer_registry_sha256=value["signer_registry_sha256"], + expected_revocation_state_sha256=value["revocation_state_sha256"], + ) + _validate_campaign_summary(value["campaign_summary"]) + if value["campaign_summary"] != receipt_summary: + raise ProductionEvidenceError("campaign_summary differs from the public decision receipt") + issued = _validate_timestamp(value["issued_at"], "issued_at") + not_before = _validate_timestamp(value["not_before"], "not_before") + expires = _validate_timestamp(value["expires_at"], "expires_at") + if not (not_before <= issued < expires): + raise ProductionEvidenceError( + "production summary validity must satisfy not_before <= issued_at < expires_at" + ) + receipt_issued = _validate_timestamp( + qualification_evidence_decision_receipt["issued_at"], + "decision receipt issued_at", + ) + receipt_not_before = _validate_timestamp( + qualification_evidence_decision_receipt["not_before"], + "decision receipt not_before", + ) + receipt_expires = _validate_timestamp( + qualification_evidence_decision_receipt["expires_at"], + "decision receipt expires_at", + ) + if not ( + receipt_not_before <= receipt_issued <= issued + and not_before >= receipt_not_before + and expires <= receipt_expires + ): + raise ProductionEvidenceError( + "production summary validity exceeds the public decision receipt" + ) + if value["evidence_identity_sha256"] != _production_evidence_identity(value): + raise ProductionEvidenceError("evidence_identity_sha256 is invalid") + + +def validate_reference_pair( + references: Sequence[Mapping[str, Any]], + *, + expected_regular_kind: str | None = None, +) -> None: + """Reject a reference pair that changes kind, order, digest, or subject binding.""" + + if len(references) != 2: + raise ProductionEvidenceError( + "an evidence pair must contain one regular object and one bundle" + ) + regular = dict(references[0]) + bundle = dict(references[1]) + validate_object_reference(regular) + validate_object_reference(bundle) + kind = regular["kind"] + if expected_regular_kind is not None and kind != expected_regular_kind: + raise ProductionEvidenceError(f"regular evidence kind must be {expected_regular_kind!r}") + if bundle["kind"] != f"{kind}-sigstore-bundle": + raise ProductionEvidenceError("the Sigstore bundle must immediately follow its subject") + if regular["subject_sha256"] is not None: + raise ProductionEvidenceError("a regular evidence object subject must be null") + if bundle["subject_sha256"] != regular["object_sha256"]: + raise ProductionEvidenceError("the Sigstore bundle subject does not bind its object") + for field in ( + "registry_source_commit", + "registry_revision", + "registry_head_sha256", + ): + if bundle[field] != regular[field]: + raise ProductionEvidenceError(f"the evidence pair does not share {field}") + + +def validate_object_reference(reference: Mapping[str, Any]) -> None: + """Validate the complete 16-key v2 public object-reference contract.""" + + value = dict(reference) + if set(value) != REFERENCE_KEYS: + missing = sorted(REFERENCE_KEYS - set(value)) + extra = sorted(set(value) - REFERENCE_KEYS) + raise ProductionEvidenceError( + f"object reference keys differ: missing={missing}, extra={extra}" + ) + expected_constants = { + "schema_version": OBJECT_REFERENCE_SCHEMA, + "repository": EVIDENCE_REPOSITORY, + "repository_id": EVIDENCE_REPOSITORY_ID, + "repository_owner_id": EVIDENCE_REPOSITORY_OWNER_ID, + } + for field, expected in expected_constants.items(): + if value[field] != expected: + raise ProductionEvidenceError(f"object reference {field} is invalid") + if not isinstance(value["registry_source_commit"], str) or not _HEX40.fullmatch( + value["registry_source_commit"] + ): + raise ProductionEvidenceError("registry_source_commit must be 40 lowercase hex") + _require_positive_int(value["registry_revision"], "registry_revision") + for field in ( + "registry_head_sha256", + "registry_entry_sha256", + "object_sha256", + "semantic_identity_sha256", + ): + _require_digest(value[field], field) + _require_positive_int(value["size_bytes"], "size_bytes") + _validate_object_path(value["object_path"]) + + kind = value["kind"] + if not isinstance(kind, str) or not _KIND.fullmatch(kind): + raise ProductionEvidenceError("kind is invalid") + if kind.endswith("-sigstore-bundle"): + regular_kind = kind.removesuffix("-sigstore-bundle") + if regular_kind not in REGULAR_EVIDENCE_KINDS: + raise ProductionEvidenceError("bundle kind has no approved regular kind") + if ( + value["object_schema_version"] != SIGSTORE_BUNDLE_MEDIA_TYPE + or value["object_media_type"] != SIGSTORE_BUNDLE_MEDIA_TYPE + ): + raise ProductionEvidenceError("raw Sigstore bundle schema or media type is invalid") + _require_digest(value["subject_sha256"], "subject_sha256") + else: + profile = REGULAR_EVIDENCE_KINDS.get(kind) + if profile is None: + raise ProductionEvidenceError(f"evidence kind is not approved: {kind!r}") + if value["object_schema_version"] != profile.schema_version: + raise ProductionEvidenceError("regular object schema version is invalid") + if value["object_media_type"] != profile.media_type: + raise ProductionEvidenceError("regular object media type is invalid") + if value["subject_sha256"] is not None: + raise ProductionEvidenceError("a regular object subject must be null") + + raw_digest = value["object_sha256"].removeprefix("sha256:") + expected_path = f"{OBJECT_PATH_PREFIX}/{raw_digest[:2]}/{raw_digest}.{kind}.json" + if value["object_path"] != expected_path: + raise ProductionEvidenceError("object_path is not content-addressed by object_sha256") + expected_entry = _registry_entry_sha256(value) + if value["registry_entry_sha256"] != expected_entry: + raise ProductionEvidenceError("registry_entry_sha256 is invalid") + + +def _build_reference( + *, + kind: str, + object_schema_version: str, + object_media_type: str, + object_bytes: bytes, + object_sha256: str, + semantic_identity_sha256: str, + subject_sha256: str | None, + registry_source_commit: str, + registry_revision: int, + registry_head_sha256: str, +) -> dict[str, Any]: + if not _HEX40.fullmatch(registry_source_commit): + raise ProductionEvidenceError("registry_source_commit must be 40 lowercase hex") + _require_positive_int(registry_revision, "registry_revision") + _require_digest(registry_head_sha256, "registry_head_sha256") + _require_digest(object_sha256, "object_sha256") + _require_digest(semantic_identity_sha256, "semantic_identity_sha256") + if subject_sha256 is not None: + _require_digest(subject_sha256, "subject_sha256") + raw_digest = object_sha256.removeprefix("sha256:") + reference: dict[str, Any] = { + "schema_version": OBJECT_REFERENCE_SCHEMA, + "repository": EVIDENCE_REPOSITORY, + "repository_id": EVIDENCE_REPOSITORY_ID, + "repository_owner_id": EVIDENCE_REPOSITORY_OWNER_ID, + "registry_source_commit": registry_source_commit, + "registry_revision": registry_revision, + "registry_head_sha256": registry_head_sha256, + "registry_entry_sha256": "sha256:" + "0" * 64, + "kind": kind, + "object_schema_version": object_schema_version, + "object_path": (f"{OBJECT_PATH_PREFIX}/{raw_digest[:2]}/{raw_digest}.{kind}.json"), + "object_sha256": object_sha256, + "size_bytes": len(object_bytes), + "object_media_type": object_media_type, + "semantic_identity_sha256": semantic_identity_sha256, + "subject_sha256": subject_sha256, + } + reference["registry_entry_sha256"] = _registry_entry_sha256(reference) + validate_object_reference(reference) + return reference + + +def _registry_entry_sha256(reference: Mapping[str, Any]) -> str: + projection = {field: reference[field] for field in REGISTRY_ENTRY_KEYS} + return sha256_digest(REGISTRY_ENTRY_DIGEST_DOMAIN + canonical_json_bytes(projection)) + + +def _validated_pair( + references: Sequence[Mapping[str, Any]], expected_kind: str +) -> tuple[dict[str, Any], dict[str, Any]]: + validate_reference_pair(references, expected_regular_kind=expected_kind) + return dict(references[0]), dict(references[1]) + + +def _validate_summary_reference_fields(value: Mapping[str, Any]) -> None: + fields = ( + ( + "qualification_evidence_decision_receipt_reference", + "qualification_evidence_decision_receipt_bundle_reference", + "qualification-evidence-decision-receipt", + ), + ( + "qualification_admission_reference", + "qualification_admission_bundle_reference", + "qualification-admission", + ), + ( + "production_acceptance_manifest_reference", + "production_acceptance_manifest_bundle_reference", + "production-acceptance-manifest", + ), + ) + for regular_field, bundle_field, kind in fields: + regular = value[regular_field] + bundle = value[bundle_field] + if not isinstance(regular, Mapping) or not isinstance(bundle, Mapping): + raise ProductionEvidenceError(f"{kind} references must be objects") + validate_reference_pair((regular, bundle), expected_regular_kind=kind) + + +def _validate_release_identity(value: Any) -> None: + if not isinstance(value, Mapping) or set(value) != _RELEASE_IDENTITY_KEYS: + raise ProductionEvidenceError("release_identity keys are invalid") + if value["schema_version"] != "openadapt.monotonic-production-release/v1": + raise ProductionEvidenceError("release_identity schema_version is invalid") + if value["channel"] != "production": + raise ProductionEvidenceError("release_identity channel must be production") + _require_positive_int(value["sequence"], "release_identity.sequence") + _require_digest( + value["previous_admission_sha256"], + "release_identity.previous_admission_sha256", + ) + + +def _validate_decision_receipt( + value: Mapping[str, Any], + *, + reference: Mapping[str, Any], + expected_signer_registry_sha256: str, + expected_revocation_state_sha256: str, +) -> dict[str, Any]: + receipt = dict(value) + if set(receipt) != _DECISION_RECEIPT_KEYS: + missing = sorted(_DECISION_RECEIPT_KEYS - set(receipt)) + extra = sorted(set(receipt) - _DECISION_RECEIPT_KEYS) + raise ProductionEvidenceError( + f"decision receipt keys differ: missing={missing}, extra={extra}" + ) + if receipt["schema_version"] != DECISION_RECEIPT_SCHEMA: + raise ProductionEvidenceError("decision receipt schema_version is invalid") + for field in _DECISION_RECEIPT_KEYS: + if field.endswith("_sha256"): + _require_digest(receipt[field], f"decision receipt {field}") + if receipt["signer_registry_sha256"] != expected_signer_registry_sha256: + raise ProductionEvidenceError("decision receipt signer registry differs") + if receipt["revocation_state_sha256"] != expected_revocation_state_sha256: + raise ProductionEvidenceError("decision receipt revocation state differs") + if not isinstance(receipt["bundle_version"], str) or not receipt["bundle_version"]: + raise ProductionEvidenceError("decision receipt bundle_version is invalid") + if receipt["entity_class"] not in { + "insurance claim", + "item", + "loan application", + "patient record", + "record", + }: + raise ProductionEvidenceError("decision receipt entity_class is not remote-safe") + if receipt["verdict"] != "ADMIT": + raise ProductionEvidenceError("decision receipt verdict must be ADMIT") + if receipt["algorithm"] != "ed25519": + raise ProductionEvidenceError("decision receipt algorithm must be ed25519") + if not isinstance(receipt["issuer_key_id"], str) or not receipt["issuer_key_id"]: + raise ProductionEvidenceError("decision receipt issuer_key_id is invalid") + try: + signature = base64.b64decode(receipt["signature"], validate=True) + except (binascii.Error, TypeError) as exc: + raise ProductionEvidenceError("decision receipt signature is invalid base64") from exc + if len(signature) != 64: + raise ProductionEvidenceError("decision receipt signature is not 64-byte Ed25519") + issued = _validate_timestamp(receipt["issued_at"], "decision receipt issued_at") + not_before = _validate_timestamp(receipt["not_before"], "decision receipt not_before") + expires = _validate_timestamp(receipt["expires_at"], "decision receipt expires_at") + if not (not_before <= issued < expires): + raise ProductionEvidenceError("decision receipt validity is invalid") + if not isinstance(reference, Mapping): + raise ProductionEvidenceError("decision receipt reference must be an object") + validate_object_reference(reference) + if reference["kind"] != "qualification-evidence-decision-receipt": + raise ProductionEvidenceError("decision receipt reference kind is invalid") + if reference["object_sha256"] != sha256_digest(canonical_json_bytes(receipt)): + raise ProductionEvidenceError("decision receipt does not match its public reference") + + campaign_summary = receipt["campaign_summary"] + if not isinstance(campaign_summary, Mapping) or set(campaign_summary) != ( + _DECISION_CAMPAIGN_SUMMARY_KEYS + ): + raise ProductionEvidenceError("decision receipt campaign_summary keys are invalid") + if campaign_summary["schema_version"] != DECISION_CAMPAIGN_SUMMARY_SCHEMA: + raise ProductionEvidenceError("decision receipt campaign_summary schema_version is invalid") + minimum = _require_positive_int( + campaign_summary["minimum_trials_per_task_condition"], + "decision receipt minimum_trials_per_task_condition", + ) + task_count = _require_positive_int( + campaign_summary["task_count"], "decision receipt task_count" + ) + if minimum < 3: + raise ProductionEvidenceError("decision receipt requires fewer than three trials") + classes = campaign_summary["classes"] + _validate_campaign_summary(classes) + if minimum != min(counts["minimum_trials_per_cell"] for counts in classes.values()): + raise ProductionEvidenceError("decision receipt minimum trial count differs") + if any(counts["task_condition_cell_count"] < task_count for counts in classes.values()): + raise ProductionEvidenceError("decision receipt omits a task/class cell") + return {qualification_class: dict(counts) for qualification_class, counts in classes.items()} + + +def _validate_publication_staging( + value: Any, *, expected_assets: Sequence[Mapping[str, Any]] +) -> str: + if not isinstance(value, Mapping) or set(value) != _PUBLICATION_STAGING_KEYS: + raise ProductionEvidenceError("publication_staging keys are invalid") + staging = dict(value) + if staging["schema_version"] != PUBLICATION_STAGING_SCHEMA: + raise ProductionEvidenceError("publication_staging schema_version is invalid") + if ( + not isinstance(staging["repository"], str) + or re.fullmatch(r"OpenAdaptAI/[A-Za-z0-9._-]+", staging["repository"]) is None + ): + raise ProductionEvidenceError("publication_staging repository is invalid") + for field in ( + "repository_id", + "draft_release_id", + "release_app_id", + "release_app_installation_id", + "release_app_bot_user_id", + ): + _require_decimal_id(staging[field], f"publication_staging {field}") + if ( + staging["release_app_id"] != "4730708" + or staging["release_app_installation_id"] != "156835568" + or staging["release_app_bot_user_id"] != "321543906" + or staging["release_author_login"] != "openadapt-release[bot]" + ): + raise ProductionEvidenceError("publication_staging release App identity is invalid") + if staging["draft"] is not True or staging["prerelease"] is not False: + raise ProductionEvidenceError("publication_staging must be a non-prerelease draft") + if staging["immutable_releases_enabled"] is not True: + raise ProductionEvidenceError("immutable GitHub releases are not enabled") + if not isinstance(staging["tag"], str) or _TAG.fullmatch(staging["tag"]) is None: + raise ProductionEvidenceError("publication_staging tag is invalid") + if ( + not isinstance(staging["target_commitish"], str) + or _HEX40.fullmatch(staging["target_commitish"]) is None + ): + raise ProductionEvidenceError("publication_staging target_commitish is invalid") + _validate_timestamp(staging["observed_at"], "publication_staging observed_at") + _validate_publication_assets(staging["assets"], expected_assets=expected_assets) + tag_rulesets_sha256 = _validate_tag_rulesets( + staging["tag_rulesets"], + repository=staging["repository"], + repository_id=staging["repository_id"], + tag=staging["tag"], + ) + if staging["tag_rulesets_sha256"] != tag_rulesets_sha256: + raise ProductionEvidenceError("publication_staging tag_rulesets_sha256 is invalid") + return sha256_digest(PUBLICATION_STAGING_DIGEST_DOMAIN + canonical_json_bytes(staging)) + + +def _validate_publication_assets( + value: Any, *, expected_assets: Sequence[Mapping[str, Any]] +) -> None: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence) or not value: + raise ProductionEvidenceError("publication_staging assets must be a non-empty array") + normalized: list[dict[str, Any]] = [] + names: set[str] = set() + asset_ids: set[str] = set() + for index, raw_asset in enumerate(value): + if not isinstance(raw_asset, Mapping) or set(raw_asset) != _ASSET_KEYS: + raise ProductionEvidenceError(f"publication asset {index} keys are invalid") + asset = dict(raw_asset) + _require_decimal_id(asset["asset_id"], f"publication asset {index} asset_id") + _require_decimal_id(asset["uploader_id"], f"publication asset {index} uploader_id") + if ( + asset["uploader_id"] != "321543906" + or asset["uploader_login"] != "openadapt-release[bot]" + ): + raise ProductionEvidenceError(f"publication asset {index} uploader is invalid") + if not isinstance(asset["name"], str) or _ASSET_NAME.fullmatch(asset["name"]) is None: + raise ProductionEvidenceError(f"publication asset {index} name is invalid") + _require_digest(asset["sha256"], f"publication asset {index} sha256") + _require_positive_int(asset["size_bytes"], f"publication asset {index} size_bytes") + if asset["name"] in names or asset["asset_id"] in asset_ids: + raise ProductionEvidenceError("publication assets contain a duplicate") + names.add(asset["name"]) + asset_ids.add(asset["asset_id"]) + normalized.append(asset) + if normalized != sorted(normalized, key=lambda item: (item["name"], item["asset_id"])): + raise ProductionEvidenceError("publication assets are not canonically ordered") + + expected: list[dict[str, Any]] = [] + for index, raw_asset in enumerate(expected_assets): + if not isinstance(raw_asset, Mapping) or set(raw_asset) != _EXPECTED_ASSET_KEYS: + raise ProductionEvidenceError(f"expected publication asset {index} keys are invalid") + asset = dict(raw_asset) + if not isinstance(asset["name"], str) or _ASSET_NAME.fullmatch(asset["name"]) is None: + raise ProductionEvidenceError(f"expected publication asset {index} name is invalid") + _require_digest(asset["sha256"], f"expected publication asset {index} sha256") + _require_positive_int(asset["size_bytes"], f"expected publication asset {index} size_bytes") + expected.append(asset) + actual_projection = [ + {field: asset[field] for field in ("name", "sha256", "size_bytes")} for asset in normalized + ] + if actual_projection != sorted(expected, key=lambda item: item["name"]): + raise ProductionEvidenceError("publication assets differ from the release candidate") + + +def _validate_tag_rulesets(value: Any, *, repository: str, repository_id: str, tag: str) -> str: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ProductionEvidenceError("publication tag_rulesets must be an array") + rulesets = [dict(item) if isinstance(item, Mapping) else {} for item in value] + if [item.get("role") for item in rulesets] != ["creation_authority", "immutability"]: + raise ProductionEvidenceError("publication tag_rulesets roles or order are invalid") + for index, ruleset in enumerate(rulesets): + if set(ruleset) != _TAG_RULESET_KEYS: + raise ProductionEvidenceError(f"publication tag ruleset {index} keys are invalid") + if ruleset["schema_version"] != TAG_RULESET_SCHEMA: + raise ProductionEvidenceError(f"publication tag ruleset {index} schema is invalid") + if ruleset["repository"] != repository or ruleset["repository_id"] != repository_id: + raise ProductionEvidenceError(f"publication tag ruleset {index} repository differs") + _require_decimal_id(ruleset["ruleset_id"], f"tag ruleset {index} ruleset_id") + if not isinstance(ruleset["name"], str) or not ruleset["name"]: + raise ProductionEvidenceError(f"publication tag ruleset {index} name is invalid") + if ruleset["target"] != "tag" or ruleset["enforcement"] != "active": + raise ProductionEvidenceError(f"publication tag ruleset {index} is not active") + _validate_tag_conditions(ruleset["conditions"], tag=tag, index=index) + + bypass = ruleset["bypass_actors"] + if not isinstance(bypass, list): + raise ProductionEvidenceError(f"publication tag ruleset {index} bypass is invalid") + expected_bypass = ( + [ + { + "actor_id": "4730708", + "actor_type": "Integration", + "bypass_mode": "always", + } + ] + if ruleset["role"] == "creation_authority" + else [] + ) + if bypass != expected_bypass or any( + not isinstance(item, Mapping) or set(item) != _BYPASS_ACTOR_KEYS for item in bypass + ): + raise ProductionEvidenceError(f"publication tag ruleset {index} bypass differs") + + rules = ruleset["rules"] + expected_rule_types = ( + ["creation"] + if ruleset["role"] == "creation_authority" + else ["deletion", "non_fast_forward", "update"] + ) + if not isinstance(rules, list) or rules != [ + {"type": rule_type} for rule_type in expected_rule_types + ]: + raise ProductionEvidenceError(f"publication tag ruleset {index} rules differ") + if any(not isinstance(item, Mapping) or set(item) != _RULE_KEYS for item in rules): + raise ProductionEvidenceError(f"publication tag ruleset {index} rules are invalid") + return sha256_digest(TAG_RULESETS_DIGEST_DOMAIN + canonical_json_bytes(rulesets)) + + +def _validate_tag_conditions(value: Any, *, tag: str, index: int) -> None: + if not isinstance(value, Mapping) or set(value) != _CONDITION_KEYS: + raise ProductionEvidenceError(f"publication tag ruleset {index} conditions are invalid") + ref_name = value["ref_name"] + if not isinstance(ref_name, Mapping) or set(ref_name) != _REF_NAME_KEYS: + raise ProductionEvidenceError(f"publication tag ruleset {index} ref_name is invalid") + include = ref_name["include"] + exclude = ref_name["exclude"] + for label, patterns in (("include", include), ("exclude", exclude)): + if ( + not isinstance(patterns, list) + or any(not isinstance(pattern, str) or not pattern for pattern in patterns) + or patterns != sorted(set(patterns)) + ): + raise ProductionEvidenceError( + f"publication tag ruleset {index} {label} patterns are invalid" + ) + candidates = (tag, f"refs/tags/{tag}") + if not include or not any( + fnmatchcase(candidate, pattern) for pattern in include for candidate in candidates + ): + raise ProductionEvidenceError(f"publication tag ruleset {index} does not match the tag") + if any(fnmatchcase(candidate, pattern) for pattern in exclude for candidate in candidates): + raise ProductionEvidenceError(f"publication tag ruleset {index} excludes the tag") + + +def _require_decimal_id(value: Any, field: str) -> str: + if not isinstance(value, str) or _DECIMAL_ID.fullmatch(value) is None: + raise ProductionEvidenceError(f"{field} must be a positive decimal string") + return value + + +def _validate_campaign_summary(value: Any) -> None: + if not isinstance(value, Mapping) or set(value) != _CAMPAIGN_CLASSES: + raise ProductionEvidenceError( + "campaign_summary must contain exactly the six qualification classes" + ) + for qualification_class, raw_counts in value.items(): + if not isinstance(raw_counts, Mapping) or set(raw_counts) != _CAMPAIGN_CLASS_SUMMARY_KEYS: + raise ProductionEvidenceError( + f"campaign_summary.{qualification_class} keys are invalid" + ) + for field, count in raw_counts.items(): + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise ProductionEvidenceError( + f"campaign_summary.{qualification_class}.{field} must be non-negative" + ) + cells = raw_counts["task_condition_cell_count"] + minimum = raw_counts["minimum_trials_per_cell"] + observed = raw_counts["observed_trial_count"] + if cells < 1 or minimum < 3 or observed < cells * minimum: + raise ProductionEvidenceError( + f"campaign_summary.{qualification_class} lacks three trials per cell" + ) + for field in ( + "silent_incorrect_success_count", + "over_halt_count", + "unsafe_effect_count", + "blind_retry_count", + ): + if raw_counts[field] != 0: + raise ProductionEvidenceError( + f"accepted campaign has nonzero {qualification_class}.{field}" + ) + healthy = value["healthy"] + idempotency = value["idempotency_replay"] + uncertain = value["uncertain_delivery"] + attended = value["declared_attended"] + repair = value["governed_repair"] + for qualification_class, counts in ( + ("healthy", healthy), + ("idempotency_replay", idempotency), + ): + for field in ("model_call_count", "unplanned_intervention_count"): + if counts[field] != 0: + raise ProductionEvidenceError(f"{qualification_class}.{field} must be zero") + if idempotency["replay_dispatch_count"] != 0: + raise ProductionEvidenceError("idempotency_replay.replay_dispatch_count must be zero") + if ( + uncertain["reconciliation_required_count"] != uncertain["observed_trial_count"] + or uncertain["blind_retry_count"] != 0 + or uncertain["replay_dispatch_count"] != 0 + ): + raise ProductionEvidenceError( + "each uncertain-delivery trial must require reconciliation without redispatch" + ) + if ( + attended["authenticated_bound_decision_count"] != attended["observed_trial_count"] + or attended["live_target_revalidation_count"] != attended["observed_trial_count"] + ): + raise ProductionEvidenceError( + "each declared-attended trial must bind a decision and target revalidation" + ) + for field in ( + "policy_approved_repair_count", + "approved_repair_count", + "retained_repair_evidence_count", + "live_target_revalidation_count", + ): + if repair[field] != repair["observed_trial_count"]: + raise ProductionEvidenceError(f"each governed-repair trial must provide {field}") + if repair["unverified_direct_action_count"] != 0: + raise ProductionEvidenceError("governed-repair trials cannot use unverified direct action") + + +def _production_evidence_identity(value: Mapping[str, Any]) -> str: + projection = { + field: value[field] + for field in ( + "target", + "claim_scope", + "release_identity", + "release_sha256", + "artifact_inventory_sha256", + "publication_staging_sha256", + "qualification_evidence_decision_receipt_reference", + "qualification_admission_reference", + "production_acceptance_manifest_reference", + "campaign_summary", + "authority_state_sha256", + "revocation_state_sha256", + "signer_registry_sha256", + ) + } + return sha256_digest(PRODUCTION_EVIDENCE_IDENTITY_DOMAIN + canonical_json_bytes(projection)) + + +def _validate_timestamp(value: Any, field: str) -> datetime: + if not isinstance(value, str) or not _WHOLE_SECOND_UTC.fullmatch(value): + raise ProductionEvidenceError(f"{field} must be whole-second UTC") + return datetime.fromisoformat(value.removesuffix("Z") + "+00:00").astimezone(timezone.utc) + + +def _validate_json_value(value: Any, context: str) -> None: + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + raise ProductionEvidenceError(f"{context} contains a floating-point value") + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + raise ProductionEvidenceError(f"{context} contains a non-string key") + _validate_json_value(item, f"{context}.{key}") + return + if isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _validate_json_value(item, f"{context}[{index}]") + return + raise ProductionEvidenceError(f"{context} contains a non-JSON value") + + +def _require_digest(value: Any, field: str) -> str: + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise ProductionEvidenceError(f"{field} must be sha256:<64-lowercase-hex>") + return value + + +def _require_positive_int(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ProductionEvidenceError(f"{field} must be a positive integer") + return value + + +def _validate_object_path(value: Any) -> str: + if not isinstance(value, str) or not value or len(value) > 512: + raise ProductionEvidenceError("object_path is invalid") + if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,511}", value) is None: + raise ProductionEvidenceError("object_path is invalid") + path = PurePosixPath(value) + if ( + path.is_absolute() + or str(path) != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ProductionEvidenceError("object_path is invalid") + return value + + +def _validate_raw_json_object(value: Any, context: str) -> bytes: + if not isinstance(value, bytes) or not value: + raise ProductionEvidenceError(f"{context} must be non-empty raw bytes") + try: + decoded = json.loads(value) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ProductionEvidenceError(f"{context} must contain one JSON object") from exc + if not isinstance(decoded, dict): + raise ProductionEvidenceError(f"{context} must contain one JSON object") + return value diff --git a/openadapt_evals/qualification_evidence.py b/openadapt_evals/qualification_evidence.py new file mode 100644 index 0000000..dcb3145 --- /dev/null +++ b/openadapt_evals/qualification_evidence.py @@ -0,0 +1,917 @@ +"""Build and validate private qualification campaign evidence. + +The schemas in this module describe the open mechanism. Campaign payloads, failure +corpora, system-specific recipes, and empirical tuning remain inside the approved +private evidence boundary. A public lifecycle consumer must use only the separate +remote-safe decision receipt and public qualification admission. +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import re +from collections import Counter, defaultdict +from datetime import datetime, timezone +from typing import Any, Callable, Mapping, Sequence + +CAMPAIGN_SCHEMA = "openadapt.qualification-campaign/v3" +TRIAL_SCHEMA = "openadapt.qualification-trial-row/v3" +TRIAL_RECEIPT_SCHEMA = "openadapt.qualification-trial-receipt/v3" +CAMPAIGN_SUMMARY_SCHEMA = "openadapt.qualification-campaign-summary/v3" +TRIAL_RECEIPT_SIGNATURE_DOMAIN = b"OpenAdapt qualification trial receipt v3\0" + +QUALIFICATION_CLASSES = frozenset( + { + "healthy", + "safe_halt", + "idempotency_replay", + "uncertain_delivery", + "declared_attended", + "governed_repair", + } +) +EXPECTED_OUTCOME_BY_CLASS: Mapping[str, str] = { + "healthy": "VERIFIED", + "safe_halt": "HALTED", + "idempotency_replay": "VERIFIED", + "uncertain_delivery": "RECONCILIATION_REQUIRED", + "declared_attended": "VERIFIED", + "governed_repair": "VERIFIED", +} +TERMINAL_OUTCOMES = frozenset({"VERIFIED", "HALTED", "RECONCILIATION_REQUIRED", "PLATFORM_FAILURE"}) +RECEIPT_TYPES = frozenset( + { + "runner", + "observer", + "delivery", + "decision", + "policy", + "repair", + "cleanup", + "cleanup_absence", + } +) + +_CAMPAIGN_KEYS = frozenset( + { + "schema_version", + "campaign_id", + "campaign_permit_sha256", + "project_contract_sha256", + "source_evidence_manifest_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + "qualification_contract", + "oracle_contract", + "authority_contract", + "conditions", + "invariants", + "excluded_trials", + "receipt_envelopes", + "generated_at", + } +) +_CONDITION_KEYS = frozenset( + { + "task", + "condition", + "qualification_class", + "expected_terminal_outcome", + "required_trials", + "trials", + } +) +_TRIAL_KEYS = frozenset( + { + "schema_version", + "task", + "condition", + "qualification_class", + "trial_index", + "attempt_id_sha256", + "run_id_sha256", + "campaign_permit_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + "started_at", + "completed_at", + "observed_terminal_outcome", + "runner_receipt_sha256", + "observer_receipt_sha256", + "delivery_receipt_sha256", + "policy_receipt_sha256", + "decision_receipt_sha256", + "repair_receipt_sha256", + "cleanup_receipt_sha256", + "cleanup_absence_proof_sha256", + } +) +_RECEIPT_KEYS = frozenset( + { + "schema_version", + "receipt_type", + "issuer_key_id", + "algorithm", + "source_artifact_sha256", + "verified_projection", + "verified_at", + "signature", + } +) +_RECEIPT_UNSIGNED_KEYS = _RECEIPT_KEYS - {"signature"} +_PROJECTION_KEYS = frozenset( + { + "campaign_id", + "task", + "condition", + "qualification_class", + "trial_index", + "attempt_id_sha256", + "run_id_sha256", + "campaign_permit_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + "verdict", + "evidence_sha256", + "facts", + } +) +_ROW_RECEIPT_FIELDS: Mapping[str, str] = { + "runner": "runner_receipt_sha256", + "observer": "observer_receipt_sha256", + "delivery": "delivery_receipt_sha256", + "decision": "decision_receipt_sha256", + "policy": "policy_receipt_sha256", + "repair": "repair_receipt_sha256", + "cleanup": "cleanup_receipt_sha256", + "cleanup_absence": "cleanup_absence_proof_sha256", +} +_COMMON_RECEIPTS = frozenset({"runner", "observer", "delivery", "cleanup", "cleanup_absence"}) +_FACT_KEYS: Mapping[str, frozenset[str]] = { + "runner": frozenset( + { + "observed_terminal_outcome", + "model_call_count", + "unplanned_intervention_count", + "unsafe_effect_count", + } + ), + "observer": frozenset( + { + "independent_verdict", + "intended_effect_count", + "wrong_effect_count", + "wrong_record_count", + "duplicate_effect_count", + "collateral_effect_count", + } + ), + "delivery": frozenset( + { + "dispatch_state", + "blind_retry_count", + "replay_dispatch_count", + "idempotency_result", + "delivery_certainty", + } + ), + "decision": frozenset({"authenticated_typed_bound_decision", "live_target_revalidated"}), + "policy": frozenset({"policy_approved_model_path"}), + "repair": frozenset( + { + "human_approval_verified", + "retained_evidence_verified", + "target_revalidated", + "unverified_direct_action_count", + } + ), + "cleanup": frozenset({"cleanup_completed"}), + "cleanup_absence": frozenset({"absence_verified"}), +} +_COUNT_FACTS: Mapping[str, frozenset[str]] = { + "runner": frozenset( + {"model_call_count", "unplanned_intervention_count", "unsafe_effect_count"} + ), + "observer": frozenset( + { + "intended_effect_count", + "wrong_effect_count", + "wrong_record_count", + "duplicate_effect_count", + "collateral_effect_count", + } + ), + "delivery": frozenset({"blind_retry_count", "replay_dispatch_count"}), + "repair": frozenset({"unverified_direct_action_count"}), +} +_BOOL_FACTS: Mapping[str, frozenset[str]] = { + "decision": frozenset({"authenticated_typed_bound_decision", "live_target_revalidated"}), + "policy": frozenset({"policy_approved_model_path"}), + "repair": frozenset( + { + "human_approval_verified", + "retained_evidence_verified", + "target_revalidated", + } + ), + "cleanup": frozenset({"cleanup_completed"}), + "cleanup_absence": frozenset({"absence_verified"}), +} +_DISPATCH_STATES = frozenset({"dispatched", "not_dispatched"}) +_DELIVERY_CERTAINTY = frozenset({"delivered", "not_delivered", "uncertain"}) +_IDEMPOTENCY_RESULTS = frozenset( + { + "not_applicable", + "single_effect_verified", + "duplicate_suppressed", + "violated", + "unverifiable", + } +) +_OBSERVER_VERDICTS = frozenset({"PROVED", "REFUTED", "UNVERIFIABLE"}) +_SHA256 = re.compile(r"^sha256:[a-f0-9]{64}$") +_WHOLE_SECOND_UTC = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") + +ReceiptSigner = Callable[[bytes], bytes] +ReceiptSignatureVerifier = Callable[[str, bytes, bytes], bool] + + +class QualificationEvidenceError(ValueError): + """A campaign or signed receipt violates the qualification contract.""" + + +def canonical_json_bytes(value: Any) -> bytes: + """Return deterministic compact JSON bytes for signatures and commitments.""" + + _validate_json_value(value, "object") + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def sha256_digest(payload: bytes) -> str: + """Return a lowercase, algorithm-prefixed SHA-256 digest.""" + + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def receipt_sha256(receipt: Mapping[str, Any]) -> str: + """Commit to the complete signed receipt envelope.""" + + return sha256_digest(canonical_json_bytes(dict(receipt))) + + +def build_signed_trial_receipt( + *, + receipt_type: str, + issuer_key_id: str, + source_artifact_sha256: str, + verified_projection: Mapping[str, Any], + verified_at: str, + signer: ReceiptSigner, +) -> dict[str, Any]: + """Build an Ed25519 receipt without binding this package to a key store.""" + + if receipt_type not in RECEIPT_TYPES: + raise QualificationEvidenceError(f"receipt_type is invalid: {receipt_type!r}") + if not isinstance(issuer_key_id, str) or not issuer_key_id: + raise QualificationEvidenceError("issuer_key_id must be a non-empty string") + _require_digest(source_artifact_sha256, "source_artifact_sha256") + _validate_timestamp(verified_at, "verified_at") + projection = dict(verified_projection) + _validate_projection(projection, receipt_type) + unsigned = { + "schema_version": TRIAL_RECEIPT_SCHEMA, + "receipt_type": receipt_type, + "issuer_key_id": issuer_key_id, + "algorithm": "ed25519", + "source_artifact_sha256": source_artifact_sha256, + "verified_projection": projection, + "verified_at": verified_at, + } + signature = signer(TRIAL_RECEIPT_SIGNATURE_DOMAIN + canonical_json_bytes(unsigned)) + if not isinstance(signature, bytes) or len(signature) != 64: + raise QualificationEvidenceError( + "the Ed25519 receipt signer must return a 64-byte signature" + ) + return {**unsigned, "signature": base64.b64encode(signature).decode("ascii")} + + +def build_qualification_campaign( + *, + campaign_id: str, + campaign_permit_sha256: str, + project_contract_sha256: str, + source_evidence_manifest_sha256: str, + bundle_artifact_sha256: str, + runtime_identity_sha256: str, + evidence_identity_sha256: str, + qualification_contract: Mapping[str, Any], + oracle_contract: Mapping[str, Any], + authority_contract: Mapping[str, Any], + conditions: Sequence[Mapping[str, Any]], + invariants: Sequence[Mapping[str, Any]], + excluded_trials: Sequence[Mapping[str, Any]], + receipt_envelopes: Sequence[Mapping[str, Any]], + generated_at: str, + verify_receipt_signature: ReceiptSignatureVerifier, + require_admissible: bool = True, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build a canonical v3 campaign and return its derived summary.""" + + ordered_conditions = sorted( + (dict(condition) for condition in conditions), + key=lambda item: ( + str(item.get("task", "")), + str(item.get("condition", "")), + str(item.get("qualification_class", "")), + ), + ) + ordered_receipts = sorted((dict(receipt) for receipt in receipt_envelopes), key=receipt_sha256) + campaign = { + "schema_version": CAMPAIGN_SCHEMA, + "campaign_id": campaign_id, + "campaign_permit_sha256": campaign_permit_sha256, + "project_contract_sha256": project_contract_sha256, + "source_evidence_manifest_sha256": source_evidence_manifest_sha256, + "bundle_artifact_sha256": bundle_artifact_sha256, + "runtime_identity_sha256": runtime_identity_sha256, + "evidence_identity_sha256": evidence_identity_sha256, + "qualification_contract": dict(qualification_contract), + "oracle_contract": dict(oracle_contract), + "authority_contract": dict(authority_contract), + "conditions": ordered_conditions, + "invariants": [dict(invariant) for invariant in invariants], + "excluded_trials": [dict(trial) for trial in excluded_trials], + "receipt_envelopes": ordered_receipts, + "generated_at": generated_at, + } + summary = validate_qualification_campaign( + campaign, + verify_receipt_signature=verify_receipt_signature, + require_admissible=require_admissible, + ) + return campaign, summary + + +def validate_qualification_campaign( + campaign: Mapping[str, Any], + *, + verify_receipt_signature: ReceiptSignatureVerifier, + require_admissible: bool = True, +) -> dict[str, Any]: + """Verify all rows and receipts, derive counts, and enforce launch admission.""" + + value = dict(campaign) + _exact_keys(value, _CAMPAIGN_KEYS, "campaign") + if value["schema_version"] != CAMPAIGN_SCHEMA: + raise QualificationEvidenceError(f"campaign schema must be {CAMPAIGN_SCHEMA!r}") + campaign_id = _require_nonempty(value["campaign_id"], "campaign_id") + for field in ( + "campaign_permit_sha256", + "project_contract_sha256", + "source_evidence_manifest_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + ): + _require_digest(value[field], field) + for field in ("qualification_contract", "oracle_contract", "authority_contract"): + _require_mapping(value[field], field) + _require_sequence(value["invariants"], "invariants") + _require_sequence(value["excluded_trials"], "excluded_trials") + _validate_timestamp(value["generated_at"], "generated_at") + + receipts = _receipt_index( + value["receipt_envelopes"], verify_receipt_signature=verify_receipt_signature + ) + conditions = _require_sequence(value["conditions"], "conditions") + if not conditions: + raise QualificationEvidenceError("campaign must contain conditions") + + failures: list[str] = [] + classes_by_task: dict[str, set[str]] = defaultdict(set) + condition_ids: set[tuple[str, str, str]] = set() + required_trial_count = 0 + observed_trial_count = 0 + minimum_trials_per_condition: int | None = None + class_counts: dict[str, Counter[str]] = { + name: Counter() for name in sorted(QUALIFICATION_CLASSES) + } + reliability = Counter( + { + "unsafe_effect_count": 0, + "silent_incorrect_success_count": 0, + "over_halt_count": 0, + "blind_retry_count": 0, + "replay_dispatch_count": 0, + "model_call_count": 0, + "unplanned_intervention_count": 0, + "uncertain_delivery_trial_count": 0, + "reconciliation_required_count": 0, + } + ) + referenced_receipts: set[str] = set() + + previous_condition_key: tuple[str, str, str] | None = None + for condition_index, raw_condition in enumerate(conditions): + context = f"conditions[{condition_index}]" + condition = _require_mapping(raw_condition, context) + _exact_keys(condition, _CONDITION_KEYS, context) + task = _require_nonempty(condition["task"], f"{context}.task") + name = _require_nonempty(condition["condition"], f"{context}.condition") + qualification_class = condition["qualification_class"] + if qualification_class not in QUALIFICATION_CLASSES: + raise QualificationEvidenceError(f"{context}.qualification_class is invalid") + expected = EXPECTED_OUTCOME_BY_CLASS[qualification_class] + if condition["expected_terminal_outcome"] != expected: + raise QualificationEvidenceError( + f"{context}.expected_terminal_outcome must be {expected}" + ) + required = _require_int( + condition["required_trials"], f"{context}.required_trials", minimum=3 + ) + key = (task, name, qualification_class) + if key in condition_ids: + raise QualificationEvidenceError(f"duplicate condition cell: {key!r}") + if previous_condition_key is not None and key < previous_condition_key: + raise QualificationEvidenceError("campaign conditions are not canonically ordered") + previous_condition_key = key + condition_ids.add(key) + classes_by_task[task].add(qualification_class) + + trials = _require_sequence(condition["trials"], f"{context}.trials") + if len(trials) < required: + failures.append(f"{task}/{name} has {len(trials)} trials; {required} required") + required_trial_count += required + observed_trial_count += len(trials) + minimum_trials_per_condition = ( + required + if minimum_trials_per_condition is None + else min(minimum_trials_per_condition, required) + ) + class_counts[qualification_class]["condition_count"] += 1 + class_counts[qualification_class]["required_trial_count"] += required + class_counts[qualification_class]["observed_trial_count"] += len(trials) + + for expected_index, raw_trial in enumerate(trials, start=1): + trial_context = f"{context}.trials[{expected_index - 1}]" + trial = _require_mapping(raw_trial, trial_context) + _exact_keys(trial, _TRIAL_KEYS, trial_context) + if trial["schema_version"] != TRIAL_SCHEMA: + raise QualificationEvidenceError( + f"{trial_context}.schema_version must be {TRIAL_SCHEMA!r}" + ) + if ( + trial["task"] != task + or trial["condition"] != name + or trial["qualification_class"] != qualification_class + ): + raise QualificationEvidenceError( + f"{trial_context} does not bind its condition cell" + ) + if trial["trial_index"] != expected_index: + raise QualificationEvidenceError( + f"{trial_context}.trial_index must be {expected_index}" + ) + for field in ( + "attempt_id_sha256", + "run_id_sha256", + "campaign_permit_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + ): + _require_digest(trial[field], f"{trial_context}.{field}") + for field in ( + "campaign_permit_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + ): + if trial[field] != value[field]: + raise QualificationEvidenceError( + f"{trial_context}.{field} does not bind the campaign" + ) + started_at = _validate_timestamp(trial["started_at"], f"{trial_context}.started_at") + completed_at = _validate_timestamp( + trial["completed_at"], f"{trial_context}.completed_at" + ) + if completed_at < started_at: + raise QualificationEvidenceError( + f"{trial_context}.completed_at precedes started_at" + ) + outcome = trial["observed_terminal_outcome"] + if outcome not in TERMINAL_OUTCOMES: + raise QualificationEvidenceError( + f"{trial_context}.observed_terminal_outcome is invalid" + ) + + required_receipt_types = set(_COMMON_RECEIPTS) + if qualification_class == "declared_attended": + required_receipt_types.add("decision") + if qualification_class == "governed_repair": + required_receipt_types.update({"policy", "repair"}) + trial_receipts: dict[str, Mapping[str, Any]] = {} + for receipt_type, row_field in _ROW_RECEIPT_FIELDS.items(): + digest = trial[row_field] + if receipt_type in required_receipt_types: + _require_digest(digest, f"{trial_context}.{row_field}") + if digest not in receipts: + raise QualificationEvidenceError( + f"{trial_context}.{row_field} has no signed receipt" + ) + if digest in referenced_receipts: + raise QualificationEvidenceError( + f"receipt {digest} is reused by more than one row slot" + ) + referenced_receipts.add(digest) + envelope = receipts[digest] + if envelope["receipt_type"] != receipt_type: + raise QualificationEvidenceError( + f"{trial_context}.{row_field} has the wrong receipt type" + ) + _validate_receipt_binding( + envelope, + campaign_id=campaign_id, + trial=trial, + context=f"{trial_context}.{row_field}", + ) + trial_receipts[receipt_type] = envelope + elif digest is not None: + raise QualificationEvidenceError( + f"{trial_context}.{row_field} must be null for {qualification_class}" + ) + + facts = { + receipt_type: envelope["verified_projection"]["facts"] + for receipt_type, envelope in trial_receipts.items() + } + runner = facts["runner"] + observer = facts["observer"] + delivery = facts["delivery"] + if runner["observed_terminal_outcome"] != outcome: + raise QualificationEvidenceError( + f"{trial_context} outcome differs from the runner receipt" + ) + class_counts[qualification_class][outcome] += 1 + reliability["unsafe_effect_count"] += runner["unsafe_effect_count"] + reliability["model_call_count"] += runner["model_call_count"] + reliability["unplanned_intervention_count"] += runner["unplanned_intervention_count"] + reliability["blind_retry_count"] += delivery["blind_retry_count"] + reliability["replay_dispatch_count"] += delivery["replay_dispatch_count"] + if outcome == "RECONCILIATION_REQUIRED": + reliability["reconciliation_required_count"] += 1 + if qualification_class == "uncertain_delivery": + reliability["uncertain_delivery_trial_count"] += 1 + + exact_intended_effect = ( + observer["independent_verdict"] == "PROVED" + and observer["intended_effect_count"] == 1 + and observer["wrong_effect_count"] == 0 + and observer["wrong_record_count"] == 0 + and observer["duplicate_effect_count"] == 0 + and observer["collateral_effect_count"] == 0 + ) + if outcome == "VERIFIED" and not exact_intended_effect: + reliability["silent_incorrect_success_count"] += 1 + failures.append(f"{task}/{name}/{expected_index} is a silent incorrect success") + if expected == "VERIFIED" and outcome == "HALTED": + reliability["over_halt_count"] += 1 + failures.append(f"{task}/{name}/{expected_index} is an over-halt") + if outcome != expected: + failures.append( + f"{task}/{name}/{expected_index} returned {outcome}; expected {expected}" + ) + if runner["unsafe_effect_count"]: + failures.append(f"{task}/{name}/{expected_index} has unsafe effects") + if delivery["blind_retry_count"]: + failures.append(f"{task}/{name}/{expected_index} used a blind retry") + if not facts["cleanup"]["cleanup_completed"]: + failures.append(f"{task}/{name}/{expected_index} cleanup did not complete") + if not facts["cleanup_absence"]["absence_verified"]: + failures.append(f"{task}/{name}/{expected_index} cleanup absence was not verified") + + if qualification_class in {"healthy", "idempotency_replay"}: + if runner["model_call_count"]: + failures.append(f"{task}/{name}/{expected_index} used a model") + if runner["unplanned_intervention_count"]: + failures.append(f"{task}/{name}/{expected_index} used unplanned intervention") + if qualification_class == "idempotency_replay": + if delivery["idempotency_result"] not in { + "duplicate_suppressed", + "single_effect_verified", + }: + failures.append(f"{task}/{name}/{expected_index} did not verify idempotency") + if observer["duplicate_effect_count"]: + failures.append(f"{task}/{name}/{expected_index} produced a duplicate effect") + if qualification_class == "uncertain_delivery": + if delivery["delivery_certainty"] != "uncertain": + failures.append( + f"{task}/{name}/{expected_index} lacks uncertain delivery evidence" + ) + if delivery["replay_dispatch_count"]: + failures.append(f"{task}/{name}/{expected_index} used replay dispatch") + if exact_intended_effect: + failures.append( + f"{task}/{name}/{expected_index} required reconciliation despite complete effect proof" + ) + if qualification_class == "safe_halt": + if any( + observer[field] + for field in ( + "intended_effect_count", + "wrong_effect_count", + "wrong_record_count", + "duplicate_effect_count", + "collateral_effect_count", + ) + ): + failures.append(f"{task}/{name}/{expected_index} safe halt observed an effect") + if ( + delivery["dispatch_state"] != "not_dispatched" + or delivery["delivery_certainty"] != "not_delivered" + ): + failures.append( + f"{task}/{name}/{expected_index} safe halt dispatched an action" + ) + if qualification_class == "declared_attended": + decision = facts["decision"] + if not all(decision.values()): + failures.append( + f"{task}/{name}/{expected_index} lacks a bound attended decision" + ) + if qualification_class == "governed_repair": + policy = facts["policy"] + repair = facts["repair"] + if not policy["policy_approved_model_path"]: + failures.append( + f"{task}/{name}/{expected_index} lacks a policy-approved model path" + ) + for field in ( + "human_approval_verified", + "retained_evidence_verified", + "target_revalidated", + ): + if not repair[field]: + failures.append(f"{task}/{name}/{expected_index} repair lacks {field}") + if repair["unverified_direct_action_count"]: + failures.append(f"{task}/{name}/{expected_index} used unverified direct action") + + for task, observed_classes in sorted(classes_by_task.items()): + missing = sorted(QUALIFICATION_CLASSES - observed_classes) + if missing: + failures.append(f"{task} is missing qualification classes: {missing}") + unused_receipts = sorted(set(receipts) - referenced_receipts) + if unused_receipts: + raise QualificationEvidenceError( + f"campaign contains unreferenced receipt envelopes: {unused_receipts}" + ) + + campaign_sha256 = sha256_digest(canonical_json_bytes(value)) + summary = { + "schema_version": CAMPAIGN_SUMMARY_SCHEMA, + "campaign_sha256": campaign_sha256, + "campaign_permit_sha256": value["campaign_permit_sha256"], + "project_contract_sha256": value["project_contract_sha256"], + "source_evidence_manifest_sha256": value["source_evidence_manifest_sha256"], + "bundle_artifact_sha256": value["bundle_artifact_sha256"], + "runtime_identity_sha256": value["runtime_identity_sha256"], + "evidence_identity_sha256": value["evidence_identity_sha256"], + "task_count": len(classes_by_task), + "condition_count": len(condition_ids), + "required_trial_count": required_trial_count, + "observed_trial_count": observed_trial_count, + "minimum_trials_per_condition": minimum_trials_per_condition, + "class_summaries": { + qualification_class: dict(sorted(counts.items())) + for qualification_class, counts in class_counts.items() + }, + "reliability": dict(sorted(reliability.items())), + "admissible": not failures, + "violations": sorted(set(failures)), + } + if require_admissible and failures: + raise QualificationEvidenceError( + "campaign is not admissible: " + "; ".join(sorted(set(failures))) + ) + return summary + + +def _receipt_index( + raw_receipts: Any, + *, + verify_receipt_signature: ReceiptSignatureVerifier, +) -> dict[str, Mapping[str, Any]]: + receipts = _require_sequence(raw_receipts, "receipt_envelopes") + if not callable(verify_receipt_signature): + raise QualificationEvidenceError("a receipt signature verifier is required") + by_digest: dict[str, Mapping[str, Any]] = {} + previous_digest: str | None = None + for index, raw_receipt in enumerate(receipts): + context = f"receipt_envelopes[{index}]" + receipt = _require_mapping(raw_receipt, context) + _exact_keys(receipt, _RECEIPT_KEYS, context) + if receipt["schema_version"] != TRIAL_RECEIPT_SCHEMA: + raise QualificationEvidenceError( + f"{context}.schema_version must be {TRIAL_RECEIPT_SCHEMA!r}" + ) + receipt_type = receipt["receipt_type"] + if receipt_type not in RECEIPT_TYPES: + raise QualificationEvidenceError(f"{context}.receipt_type is invalid") + key_id = _require_nonempty(receipt["issuer_key_id"], f"{context}.issuer_key_id") + if receipt["algorithm"] != "ed25519": + raise QualificationEvidenceError(f"{context}.algorithm must be ed25519") + _require_digest(receipt["source_artifact_sha256"], f"{context}.source_artifact_sha256") + _validate_timestamp(receipt["verified_at"], f"{context}.verified_at") + projection = _require_mapping( + receipt["verified_projection"], f"{context}.verified_projection" + ) + _validate_projection(projection, receipt_type) + try: + signature = base64.b64decode(receipt["signature"], validate=True) + except (binascii.Error, TypeError) as exc: + raise QualificationEvidenceError(f"{context}.signature is invalid base64") from exc + if len(signature) != 64: + raise QualificationEvidenceError( + f"{context}.signature is not a 64-byte Ed25519 signature" + ) + unsigned = {field: receipt[field] for field in _RECEIPT_UNSIGNED_KEYS} + preimage = TRIAL_RECEIPT_SIGNATURE_DOMAIN + canonical_json_bytes(unsigned) + if not verify_receipt_signature(key_id, preimage, signature): + raise QualificationEvidenceError(f"{context}.signature is not valid") + digest = receipt_sha256(receipt) + if digest in by_digest: + raise QualificationEvidenceError(f"duplicate receipt envelope: {digest}") + if previous_digest is not None and digest < previous_digest: + raise QualificationEvidenceError("receipt_envelopes are not canonically ordered") + previous_digest = digest + by_digest[digest] = receipt + return by_digest + + +def _validate_projection(projection: Mapping[str, Any], receipt_type: str) -> None: + _exact_keys(projection, _PROJECTION_KEYS, "verified_projection") + _require_nonempty(projection["campaign_id"], "verified_projection.campaign_id") + for field in ("task", "condition", "qualification_class", "verdict"): + _require_nonempty(projection[field], f"verified_projection.{field}") + if projection["qualification_class"] not in QUALIFICATION_CLASSES: + raise QualificationEvidenceError("verified_projection.qualification_class is invalid") + _require_int(projection["trial_index"], "verified_projection.trial_index", minimum=1) + for field in ( + "attempt_id_sha256", + "run_id_sha256", + "campaign_permit_sha256", + "bundle_artifact_sha256", + "runtime_identity_sha256", + "evidence_identity_sha256", + "evidence_sha256", + ): + _require_digest(projection[field], f"verified_projection.{field}") + facts = _require_mapping(projection["facts"], "verified_projection.facts") + _exact_keys(facts, _FACT_KEYS[receipt_type], f"{receipt_type} facts") + for field in _COUNT_FACTS.get(receipt_type, frozenset()): + _require_int(facts[field], f"{receipt_type} facts.{field}", minimum=0) + for field in _BOOL_FACTS.get(receipt_type, frozenset()): + if not isinstance(facts[field], bool): + raise QualificationEvidenceError(f"{receipt_type} facts.{field} must be boolean") + if receipt_type == "runner" and facts["observed_terminal_outcome"] not in TERMINAL_OUTCOMES: + raise QualificationEvidenceError("runner observed_terminal_outcome is invalid") + if receipt_type == "observer" and facts["independent_verdict"] not in _OBSERVER_VERDICTS: + raise QualificationEvidenceError("observer independent_verdict is invalid") + if receipt_type == "delivery": + if facts["dispatch_state"] not in _DISPATCH_STATES: + raise QualificationEvidenceError("delivery dispatch_state is invalid") + if facts["delivery_certainty"] not in _DELIVERY_CERTAINTY: + raise QualificationEvidenceError("delivery delivery_certainty is invalid") + if facts["idempotency_result"] not in _IDEMPOTENCY_RESULTS: + raise QualificationEvidenceError("delivery idempotency_result is invalid") + expected_verdict = _derived_receipt_verdict(receipt_type, facts) + if projection["verdict"] != expected_verdict: + raise QualificationEvidenceError( + f"{receipt_type} receipt verdict must be {expected_verdict!r}" + ) + + +def _derived_receipt_verdict(receipt_type: str, facts: Mapping[str, Any]) -> str: + if receipt_type == "runner": + return str(facts["observed_terminal_outcome"]) + if receipt_type == "observer": + return str(facts["independent_verdict"]) + if receipt_type == "delivery": + return str(facts["delivery_certainty"]).upper() + if receipt_type == "decision": + return "VERIFIED" if all(facts.values()) else "REFUTED" + if receipt_type == "policy": + return "VERIFIED" if facts["policy_approved_model_path"] else "REFUTED" + if receipt_type == "repair": + verified = ( + facts["human_approval_verified"] + and facts["retained_evidence_verified"] + and facts["target_revalidated"] + and facts["unverified_direct_action_count"] == 0 + ) + return "VERIFIED" if verified else "REFUTED" + if receipt_type == "cleanup": + return "VERIFIED" if facts["cleanup_completed"] else "REFUTED" + if receipt_type == "cleanup_absence": + return "VERIFIED" if facts["absence_verified"] else "REFUTED" + raise QualificationEvidenceError(f"receipt_type is invalid: {receipt_type!r}") + + +def _validate_receipt_binding( + receipt: Mapping[str, Any], + *, + campaign_id: str, + trial: Mapping[str, Any], + context: str, +) -> None: + projection = receipt["verified_projection"] + expected = { + "campaign_id": campaign_id, + "task": trial["task"], + "condition": trial["condition"], + "qualification_class": trial["qualification_class"], + "trial_index": trial["trial_index"], + "attempt_id_sha256": trial["attempt_id_sha256"], + "run_id_sha256": trial["run_id_sha256"], + "campaign_permit_sha256": trial["campaign_permit_sha256"], + "bundle_artifact_sha256": trial["bundle_artifact_sha256"], + "runtime_identity_sha256": trial["runtime_identity_sha256"], + "evidence_identity_sha256": trial["evidence_identity_sha256"], + } + for field, expected_value in expected.items(): + if projection[field] != expected_value: + raise QualificationEvidenceError(f"{context} does not bind {field}") + + +def _exact_keys(value: Mapping[str, Any], expected: frozenset[str], context: str) -> None: + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise QualificationEvidenceError(f"{context} keys differ: missing={missing}, extra={extra}") + + +def _require_mapping(value: Any, context: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise QualificationEvidenceError(f"{context} must be an object") + return value + + +def _require_sequence(value: Any, context: str) -> Sequence[Any]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise QualificationEvidenceError(f"{context} must be an array") + return value + + +def _require_nonempty(value: Any, context: str) -> str: + if not isinstance(value, str) or not value: + raise QualificationEvidenceError(f"{context} must be a non-empty string") + return value + + +def _require_digest(value: Any, context: str) -> str: + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise QualificationEvidenceError(f"{context} must be sha256:<64-lowercase-hex>") + return value + + +def _require_int(value: Any, context: str, *, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise QualificationEvidenceError(f"{context} must be an integer >= {minimum}") + return value + + +def _validate_timestamp(value: Any, context: str) -> datetime: + if not isinstance(value, str) or not _WHOLE_SECOND_UTC.fullmatch(value): + raise QualificationEvidenceError(f"{context} must be whole-second UTC") + return datetime.fromisoformat(value.removesuffix("Z") + "+00:00").astimezone(timezone.utc) + + +def _validate_json_value(value: Any, context: str) -> None: + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + raise QualificationEvidenceError(f"{context} contains a floating-point value") + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + raise QualificationEvidenceError(f"{context} contains a non-string key") + _validate_json_value(item, f"{context}.{key}") + return + if isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _validate_json_value(item, f"{context}[{index}]") + return + raise QualificationEvidenceError(f"{context} contains a non-JSON value") diff --git a/pyproject.toml b/pyproject.toml index 1761951..8dbc71b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "openadapt-evals" version = "0.94.1" -description = "Evaluation infrastructure for GUI agent benchmarks" +description = "Qualification evidence and evaluation infrastructure for OpenAdapt" readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -16,7 +16,7 @@ maintainers = [ {name = "OpenAdaptAI", email = "contact@openadapt.ai"} ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", diff --git a/scripts/build_production_evidence.py b/scripts/build_production_evidence.py new file mode 100644 index 0000000..55a2ae3 --- /dev/null +++ b/scripts/build_production_evidence.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Build public production summaries and content-addressed evidence pairs.""" + +from __future__ import annotations + +import argparse +import json +import os +import tempfile +from pathlib import Path +from typing import Any, Sequence + +from openadapt_evals.production_evidence import ( + ProductionEvidenceError, + build_evidence_object_pair, + build_production_acceptance_summary, + canonical_json_bytes, +) + +_SUMMARY_INPUT_KEYS = frozenset( + { + "target", + "claim_scope", + "acceptance_policy_sha256", + "lifecycle_policy_sha256", + "release_identity", + "release_sha256", + "artifact_inventory_sha256", + "publication_staging", + "expected_publication_assets", + "qualification_evidence_decision_receipt", + "qualification_evidence_decision_receipt_references", + "qualification_admission_references", + "production_acceptance_manifest_references", + "authority_state_sha256", + "revocation_state_sha256", + "signer_registry_sha256", + "issued_at", + "not_before", + "expires_at", + } +) + + +def _reject_duplicate_pairs(pairs: Sequence[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ProductionEvidenceError(f"JSON contains duplicate key: {key}") + value[key] = item + return value + + +def _read_json_object(path: Path, context: str) -> dict[str, Any]: + try: + value = json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=_reject_duplicate_pairs, + parse_float=lambda value: (_ for _ in ()).throw( + ProductionEvidenceError(f"{context} contains a floating-point value: {value}") + ), + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ProductionEvidenceError(f"cannot read {context}: {path}") from exc + if not isinstance(value, dict): + raise ProductionEvidenceError(f"{context} must be one JSON object") + return value + + +def _write_atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + if path.read_bytes() == payload: + return + raise ProductionEvidenceError(f"refusing to replace different evidence: {path}") + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + if temporary.exists(): + temporary.unlink() + + +def build_summary(input_path: Path, output_path: Path) -> dict[str, Any]: + """Build one public v2 lifecycle summary from public inputs only.""" + + payload = _read_json_object(input_path, "production summary input") + if set(payload) != _SUMMARY_INPUT_KEYS: + missing = sorted(_SUMMARY_INPUT_KEYS - set(payload)) + extra = sorted(set(payload) - _SUMMARY_INPUT_KEYS) + raise ProductionEvidenceError( + f"production summary input keys differ: missing={missing}, extra={extra}" + ) + summary = build_production_acceptance_summary(**payload) + _write_atomic(output_path, canonical_json_bytes(summary)) + return summary + + +def build_pair( + *, + kind: str, + object_path: Path, + sigstore_bundle_path: Path, + registry_source_commit: str, + registry_revision: int, + registry_head_sha256: str, + output_root: Path, + references_output: Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Write one regular object and its bundle to their exact public paths.""" + + object_value = _read_json_object(object_path, "regular production evidence object") + try: + raw_bundle = sigstore_bundle_path.read_bytes() + except OSError as exc: + raise ProductionEvidenceError( + f"cannot read raw Sigstore bundle: {sigstore_bundle_path}" + ) from exc + pair = build_evidence_object_pair( + kind=kind, + object_value=object_value, + sigstore_bundle=raw_bundle, + registry_source_commit=registry_source_commit, + registry_revision=registry_revision, + registry_head_sha256=registry_head_sha256, + ) + for payload, reference in zip(pair.objects, pair.references, strict=True): + _write_atomic(output_root / reference["object_path"], payload) + references = [dict(reference) for reference in pair.references] + _write_atomic(references_output, canonical_json_bytes(references)) + return pair.references + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + summary = commands.add_parser("summary", help="build a public lifecycle summary") + summary.add_argument("--input", type=Path, required=True) + summary.add_argument("--output", type=Path, required=True) + + pair = commands.add_parser("pair", help="build a content-addressed object pair") + pair.add_argument("--kind", required=True) + pair.add_argument("--object", type=Path, required=True) + pair.add_argument("--sigstore-bundle", type=Path, required=True) + pair.add_argument("--registry-source-commit", required=True) + pair.add_argument("--registry-revision", type=int, required=True) + pair.add_argument("--registry-head-sha256", required=True) + pair.add_argument("--output-root", type=Path, required=True) + pair.add_argument("--references-output", type=Path, required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "summary": + build_summary(args.input, args.output) + else: + build_pair( + kind=args.kind, + object_path=args.object, + sigstore_bundle_path=args.sigstore_bundle, + registry_source_commit=args.registry_source_commit, + registry_revision=args.registry_revision, + registry_head_sha256=args.registry_head_sha256, + output_root=args.output_root, + references_output=args.references_output, + ) + except ProductionEvidenceError as exc: + raise SystemExit(f"production evidence refused: {exc}") from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/import_production_acceptance.py b/scripts/import_production_acceptance.py index ab4c42e..5dca270 100644 --- a/scripts/import_production_acceptance.py +++ b/scripts/import_production_acceptance.py @@ -37,7 +37,7 @@ TRIAL_SCHEMA = "openadapt.qualification-trial-row/v2" RESULT_SCHEMA = "openadapt.evals-derived-production-acceptance/v1" CLAIM_SCOPE = "qualified_browser_workflow_on_bound_environment" -PRODUCTION_ACCEPTANCE_SCHEMA = "openadapt.production-acceptance/v1" +PRODUCTION_ACCEPTANCE_SCHEMA = "openadapt.production-acceptance/v2" PRODUCTION_ACCEPTANCE_POLICY_SCHEMA = "openadapt.production-acceptance-policy/v1" PRODUCTION_ACCEPTANCE_POLICY_DOMAIN = b"OpenAdapt production acceptance policy v1\0" PRODUCTION_LIFECYCLE_POLICY_SCHEMA = "openadapt.production-lifecycle-policy/v1" diff --git a/tests/test_build_production_evidence.py b/tests/test_build_production_evidence.py new file mode 100644 index 0000000..55db337 --- /dev/null +++ b/tests/test_build_production_evidence.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +from openadapt_evals import production_evidence as evidence + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "build_production_evidence.py" +SPEC = importlib.util.spec_from_file_location("build_production_evidence", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def _digest(character: str) -> str: + return "sha256:" + character * 64 + + +def test_pair_command_writes_exact_content_addressed_paths(tmp_path: Path) -> None: + regular = tmp_path / "admission.json" + raw_bundle = tmp_path / "admission.sigstore.json" + output_root = tmp_path / "registry" + references_output = tmp_path / "references.json" + regular.write_text( + json.dumps( + { + "schema_version": "openadapt.qualification-admission/v3", + "verdict": "ADMIT", + } + ), + encoding="utf-8", + ) + bundle_bytes = b'{ "mediaType": "application/vnd.dev.sigstore.bundle.v0.3+json" }\n' + raw_bundle.write_bytes(bundle_bytes) + + regular_reference, bundle_reference = MODULE.build_pair( + kind="qualification-admission", + object_path=regular, + sigstore_bundle_path=raw_bundle, + registry_source_commit="a" * 40, + registry_revision=12, + registry_head_sha256=_digest("b"), + output_root=output_root, + references_output=references_output, + ) + + assert (output_root / regular_reference["object_path"]).read_bytes() == ( + evidence.canonical_json_bytes(json.loads(regular.read_text(encoding="utf-8"))) + ) + assert (output_root / bundle_reference["object_path"]).read_bytes() == bundle_bytes + assert json.loads(references_output.read_text(encoding="utf-8")) == [ + regular_reference, + bundle_reference, + ] + evidence.validate_reference_pair( + (regular_reference, bundle_reference), + expected_regular_kind="qualification-admission", + ) + + +def test_pair_command_refuses_to_replace_different_evidence(tmp_path: Path) -> None: + path = tmp_path / "evidence.json" + path.write_bytes(b"old") + with pytest.raises(evidence.ProductionEvidenceError, match="refusing to replace"): + MODULE._write_atomic(path, b"new") + + +def test_json_reader_rejects_duplicate_keys_and_floats(tmp_path: Path) -> None: + duplicate = tmp_path / "duplicate.json" + duplicate.write_text('{"schema_version":"one","schema_version":"two"}', encoding="utf-8") + with pytest.raises(evidence.ProductionEvidenceError, match="duplicate key"): + MODULE._read_json_object(duplicate, "test object") + + floating = tmp_path / "float.json" + floating.write_text('{"size":1.5}', encoding="utf-8") + with pytest.raises(evidence.ProductionEvidenceError, match="floating-point"): + MODULE._read_json_object(floating, "test object") diff --git a/tests/test_import_production_acceptance.py b/tests/test_import_production_acceptance.py index bb21982..c7815c1 100644 --- a/tests/test_import_production_acceptance.py +++ b/tests/test_import_production_acceptance.py @@ -2474,7 +2474,7 @@ def test_production_acceptance_target_scope_map_is_closed() -> None: "openadapt": "qualified_workflow_launcher_release", } assert MODULE.production_acceptance_policy_sha256() == ( - "sha256:77130f494453c2443ed824a863069bd055a831da483880386e95b092eabaf5bb" + "sha256:6cd6cfeeb6d430c0e9d3a4a55770cb2d9962c8769edc52fac5cc46bff31bee21" ) @@ -2505,7 +2505,7 @@ def test_builds_only_complete_target_neutral_production_manifest() -> None: "reliability", "retention", } - assert manifest["schema_version"] == "openadapt.production-acceptance/v1" + assert manifest["schema_version"] == "openadapt.production-acceptance/v2" assert manifest["target"] == "flow" assert manifest["claim_scope"] == "qualified_workflow_runtime_release" assert manifest["verdict"] == "accepted" @@ -2587,7 +2587,7 @@ def test_browser_source_refuses_target_without_its_evidence_adapter(target: str) @pytest.mark.parametrize( "path,replacement", [ - (("schema_version",), "openadapt.production-acceptance/v2"), + (("schema_version",), "openadapt.production-acceptance/v1"), (("target",), "cloud"), (("claim_scope",), "qualified_workflow_control_plane_deployment"), (("verdict",), "rejected"), diff --git a/tests/test_production_evidence.py b/tests/test_production_evidence.py new file mode 100644 index 0000000..fd12989 --- /dev/null +++ b/tests/test_production_evidence.py @@ -0,0 +1,510 @@ +from __future__ import annotations + +import base64 +import copy +import hashlib +import json + +import pytest + +from openadapt_evals import production_evidence as evidence + + +def _digest(character: str) -> str: + return "sha256:" + character * 64 + + +def _object(kind: str) -> dict[str, object]: + return { + "schema_version": evidence.REGULAR_EVIDENCE_KINDS[kind].schema_version, + "value": kind, + } + + +def _pair( + kind: str, + *, + revision: int = 4, + object_value: dict[str, object] | None = None, +) -> evidence.EvidenceObjectPair: + return evidence.build_evidence_object_pair( + kind=kind, + object_value=object_value or _object(kind), + sigstore_bundle=(b'{ "mediaType": "application/vnd.dev.sigstore.bundle.v0.3+json" }\n'), + registry_source_commit="a" * 40, + registry_revision=revision, + registry_head_sha256=_digest("b"), + ) + + +def _campaign_summary() -> dict[str, dict[str, int]]: + fields = evidence._CAMPAIGN_CLASS_SUMMARY_KEYS + summary = { + qualification_class: {field: 0 for field in fields} + for qualification_class in evidence._CAMPAIGN_CLASSES + } + for counts in summary.values(): + counts["task_condition_cell_count"] = 1 + counts["minimum_trials_per_cell"] = 3 + counts["observed_trial_count"] = 3 + summary["uncertain_delivery"]["reconciliation_required_count"] = 3 + summary["declared_attended"]["authenticated_bound_decision_count"] = 3 + summary["declared_attended"]["live_target_revalidation_count"] = 3 + for field in ( + "policy_approved_repair_count", + "approved_repair_count", + "retained_repair_evidence_count", + "live_target_revalidation_count", + ): + summary["governed_repair"][field] = 3 + return summary + + +def _decision_receipt(campaign: dict[str, dict[str, int]]) -> dict[str, object]: + receipt: dict[str, object] = { + "schema_version": evidence.DECISION_RECEIPT_SCHEMA, + "decision_commitment_sha256": _digest("a"), + "evidence_manifest_sha256": _digest("b"), + "campaign_artifact_sha256": _digest("c"), + "organization_id_sha256": _digest("d"), + "workflow_id_sha256": _digest("e"), + "workflow_version_id_sha256": _digest("f"), + "bundle_version": "7", + "bundle_sha256": _digest("0"), + "admitted_runtime_sha256": _digest("1"), + "action_contract_sha256": _digest("2"), + "application_contract_sha256": _digest("3"), + "effect_contract_sha256": _digest("4"), + "environment_contract_sha256": _digest("5"), + "evidence_authority_contract_sha256": _digest("6"), + "identity_contract_sha256": _digest("7"), + "input_contract_sha256": _digest("8"), + "policy_contract_sha256": _digest("9"), + "campaign_permit_sha256": _digest("a"), + "signer_registry_sha256": _digest("8"), + "revocation_state_sha256": _digest("7"), + "entity_class": "record", + "campaign_summary": { + "schema_version": evidence.DECISION_CAMPAIGN_SUMMARY_SCHEMA, + "minimum_trials_per_task_condition": 3, + "task_count": 1, + "classes": campaign, + }, + "verdict": "ADMIT", + "issued_at": "2026-08-27T11:00:00Z", + "not_before": "2026-08-27T11:00:00Z", + "expires_at": "2026-09-10T12:00:00Z", + "issuer_key_id": "qualification-evidence-key-1", + "algorithm": "ed25519", + "signature": base64.b64encode(b"s" * 64).decode("ascii"), + } + return receipt + + +def _publication_staging() -> tuple[dict[str, object], list[dict[str, object]]]: + expected_assets = [ + { + "name": "openadapt_evals-0.94.1-py3-none-any.whl", + "sha256": _digest("a"), + "size_bytes": 200, + }, + { + "name": "openadapt_evals-0.94.1.tar.gz", + "sha256": _digest("b"), + "size_bytes": 100, + }, + ] + assets = [ + { + **asset, + "asset_id": str(index), + "uploader_id": "321543906", + "uploader_login": "openadapt-release[bot]", + } + for index, asset in enumerate(expected_assets, start=11) + ] + + def ruleset( + role: str, ruleset_id: str, rules: list[str], *, creation_bypass: bool + ) -> dict[str, object]: + return { + "schema_version": evidence.TAG_RULESET_SCHEMA, + "role": role, + "repository": "OpenAdaptAI/openadapt-evals", + "repository_id": "123456", + "ruleset_id": ruleset_id, + "name": f"release tag {role}", + "target": "tag", + "enforcement": "active", + "bypass_actors": ( + [ + { + "actor_id": "4730708", + "actor_type": "Integration", + "bypass_mode": "always", + } + ] + if creation_bypass + else [] + ), + "conditions": {"ref_name": {"include": ["refs/tags/v*"], "exclude": []}}, + "rules": [{"type": rule} for rule in rules], + } + + tag_rulesets = [ + ruleset("creation_authority", "71", ["creation"], creation_bypass=True), + ruleset( + "immutability", + "72", + ["deletion", "non_fast_forward", "update"], + creation_bypass=False, + ), + ] + staging = { + "schema_version": evidence.PUBLICATION_STAGING_SCHEMA, + "repository": "OpenAdaptAI/openadapt-evals", + "repository_id": "123456", + "draft_release_id": "456789", + "tag": "v0.94.1", + "target_commitish": "c" * 40, + "draft": True, + "prerelease": False, + "release_app_id": "4730708", + "release_app_installation_id": "156835568", + "release_app_bot_user_id": "321543906", + "release_author_login": "openadapt-release[bot]", + "assets": assets, + "immutable_releases_enabled": True, + "tag_rulesets": tag_rulesets, + "tag_rulesets_sha256": evidence.sha256_digest( + evidence.TAG_RULESETS_DIGEST_DOMAIN + evidence.canonical_json_bytes(tag_rulesets) + ), + "observed_at": "2026-08-27T11:30:00Z", + } + return staging, expected_assets + + +def _summary() -> tuple[dict[str, object], dict[str, object], list[dict[str, object]]]: + campaign = _campaign_summary() + receipt = _decision_receipt(campaign) + decision = _pair("qualification-evidence-decision-receipt", object_value=receipt) + admission = _pair("qualification-admission") + manifest = _pair("production-acceptance-manifest") + staging, expected_assets = _publication_staging() + summary = evidence.build_production_acceptance_summary( + target="flow", + claim_scope="qualified_workflow_runtime_release", + acceptance_policy_sha256=_digest("1"), + lifecycle_policy_sha256=_digest("2"), + release_identity={ + "schema_version": "openadapt.monotonic-production-release/v1", + "channel": "production", + "sequence": 7, + "previous_admission_sha256": _digest("3"), + }, + release_sha256=_digest("4"), + artifact_inventory_sha256=_digest("5"), + publication_staging=staging, + expected_publication_assets=expected_assets, + qualification_evidence_decision_receipt=receipt, + qualification_evidence_decision_receipt_references=decision.references, + qualification_admission_references=admission.references, + production_acceptance_manifest_references=manifest.references, + authority_state_sha256=_digest("6"), + revocation_state_sha256=_digest("7"), + signer_registry_sha256=_digest("8"), + issued_at="2026-08-27T12:00:00Z", + not_before="2026-08-27T12:00:00Z", + expires_at="2026-09-03T12:00:00Z", + ) + return summary, receipt, expected_assets + + +def test_approved_kind_map_is_exact() -> None: + assert set(evidence.REGULAR_EVIDENCE_KINDS) == { + "qualification-release", + "production-acceptance-manifest", + "production-acceptance-summary", + "qualification-authority-state-receipt", + "qualification-revocation-state-receipt", + "production-current-default", + "production-deployment-observation", + "production-lifecycle-checkpoint", + "production-cloud-deploy-authorization", + "production-cloud-deployment-result", + "qualification-campaign-permit-policy", + "qualification-campaign-permit-request", + "qualification-campaign-permit", + "qualification-campaign-permit-receipt", + "qualification-evidence-decision-receipt", + "qualification-admission", + } + assert evidence.REGULAR_EVIDENCE_KINDS["production-acceptance-summary"] == ( + evidence.EvidenceKind( + "openadapt.production-lifecycle-evidence-summary/v2", + "application/vnd.openadapt.production-lifecycle-evidence-summary+json;version=2", + ) + ) + assert evidence.REGULAR_EVIDENCE_KINDS["qualification-admission"] == ( + evidence.EvidenceKind( + "openadapt.qualification-admission/v3", + "application/vnd.openadapt.qualification-admission+json;version=3", + ) + ) + + +def test_builds_exact_content_addressed_v2_reference_pair() -> None: + pair = _pair("production-acceptance-summary") + regular, bundle = pair.references + + assert pair.objects[1].startswith(b'{ "mediaType"') + assert set(regular) == evidence.REFERENCE_KEYS + assert "url" not in regular + assert regular["schema_version"] == evidence.OBJECT_REFERENCE_SCHEMA + assert regular["repository"] == "OpenAdaptAI/.github" + assert regular["repository_id"] == "858454062" + assert regular["repository_owner_id"] == "132681217" + assert regular["object_schema_version"] == ( + "openadapt.production-lifecycle-evidence-summary/v2" + ) + assert regular["object_media_type"] == ( + "application/vnd.openadapt.production-lifecycle-evidence-summary+json;version=2" + ) + assert regular["subject_sha256"] is None + regular_hex = str(regular["object_sha256"]).removeprefix("sha256:") + assert regular["object_path"] == ( + f"production-evidence/objects/sha256/{regular_hex[:2]}/" + f"{regular_hex}.production-acceptance-summary.json" + ) + assert bundle["kind"] == "production-acceptance-summary-sigstore-bundle" + assert bundle["object_schema_version"] == evidence.SIGSTORE_BUNDLE_MEDIA_TYPE + assert bundle["object_media_type"] == evidence.SIGSTORE_BUNDLE_MEDIA_TYPE + assert bundle["subject_sha256"] == regular["object_sha256"] + evidence.validate_reference_pair( + pair.references, expected_regular_kind="production-acceptance-summary" + ) + + +def test_reference_digests_use_frozen_domains_and_projections() -> None: + pair = _pair("qualification-admission") + regular, bundle = pair.references + regular_object = json.loads(pair.objects[0]) + expected_regular_identity = evidence.sha256_digest( + evidence.REGULAR_SEMANTIC_IDENTITY_DOMAIN + + evidence.canonical_json_bytes( + { + "kind": "qualification-admission", + "object_schema_version": "openadapt.qualification-admission/v3", + "object": regular_object, + } + ) + ) + assert regular["semantic_identity_sha256"] == expected_regular_identity + expected_bundle_identity = evidence.sha256_digest( + evidence.BUNDLE_SEMANTIC_IDENTITY_DOMAIN + + evidence.canonical_json_bytes( + { + "kind": "qualification-admission-sigstore-bundle", + "object_sha256": bundle["object_sha256"], + "subject_sha256": regular["object_sha256"], + } + ) + ) + assert bundle["semantic_identity_sha256"] == expected_bundle_identity + for reference in pair.references: + entry = {field: reference[field] for field in evidence.REGISTRY_ENTRY_KEYS} + assert reference["registry_entry_sha256"] == evidence.sha256_digest( + evidence.REGISTRY_ENTRY_DIGEST_DOMAIN + evidence.canonical_json_bytes(entry) + ) + + +def test_rejects_bundle_before_regular_or_wrong_subject() -> None: + pair = _pair("qualification-campaign-permit") + with pytest.raises(evidence.ProductionEvidenceError): + evidence.validate_reference_pair(tuple(reversed(pair.references))) + + regular, bundle = copy.deepcopy(pair.references) + bundle["subject_sha256"] = _digest("f") + bundle["registry_entry_sha256"] = evidence._registry_entry_sha256(bundle) + with pytest.raises(evidence.ProductionEvidenceError, match="subject"): + evidence.validate_reference_pair((regular, bundle)) + + +def test_signer_registry_pointer_has_only_frozen_fields() -> None: + registry = { + "schema_version": "openadapt.qualification-signer-registry/v2", + "revision": 9, + } + pointer = evidence.build_signer_registry_pointer( + object_path="production-evidence/authority/signer-registry.json", + object_sha256=_digest("a"), + registry_identity_sha256=evidence.signer_registry_identity(registry), + registry_revision=9, + ) + assert pointer == { + "schema_version": "openadapt.qualification-signer-registry-pointer/v1", + "object_path": "production-evidence/authority/signer-registry.json", + "object_sha256": _digest("a"), + "registry_identity_sha256": evidence.signer_registry_identity(registry), + "registry_revision": 9, + } + + +def test_builds_remote_safe_summary_from_three_prior_reference_pairs() -> None: + summary, receipt, expected_assets = _summary() + assert set(summary) == evidence._SUMMARY_KEYS + assert summary["schema_version"] == ("openadapt.production-lifecycle-evidence-summary/v2") + assert summary["verdict"] == "accepted" + assert set(summary["campaign_summary"]) == evidence._CAMPAIGN_CLASSES + rendered = json.dumps(summary["campaign_summary"], sort_keys=True) + for private_field in ( + "task_name", + "condition_name", + "application", + "environment", + "live_identity", + ): + assert private_field not in rendered + evidence.validate_production_acceptance_summary( + summary, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=expected_assets, + ) + + +def test_summary_binds_durable_release_app_staging_and_two_tag_rulesets() -> None: + summary, receipt, expected_assets = _summary() + staging = summary["publication_staging"] + assert staging["draft"] is True + assert staging["prerelease"] is False + assert staging["release_app_id"] == "4730708" + assert staging["release_app_installation_id"] == "156835568" + assert staging["release_app_bot_user_id"] == "321543906" + assert [ruleset["role"] for ruleset in staging["tag_rulesets"]] == [ + "creation_authority", + "immutability", + ] + assert summary["publication_staging_sha256"] == evidence.sha256_digest( + evidence.PUBLICATION_STAGING_DIGEST_DOMAIN + evidence.canonical_json_bytes(staging) + ) + + mutated = copy.deepcopy(summary) + mutated["publication_staging"]["immutable_releases_enabled"] = False + with pytest.raises(evidence.ProductionEvidenceError, match="immutable"): + evidence.validate_production_acceptance_summary( + mutated, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=expected_assets, + ) + + +def test_summary_rejects_draft_asset_or_tag_authority_drift() -> None: + summary, receipt, expected_assets = _summary() + changed_assets = copy.deepcopy(expected_assets) + changed_assets[0]["size_bytes"] += 1 + with pytest.raises(evidence.ProductionEvidenceError, match="release candidate"): + evidence.validate_production_acceptance_summary( + summary, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=changed_assets, + ) + + changed_rules = copy.deepcopy(summary) + changed_rules["publication_staging"]["tag_rulesets"][1]["bypass_actors"] = [ + { + "actor_id": "4730708", + "actor_type": "Integration", + "bypass_mode": "always", + } + ] + with pytest.raises(evidence.ProductionEvidenceError, match="bypass"): + evidence.validate_production_acceptance_summary( + changed_rules, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=expected_assets, + ) + + +def test_summary_extracts_only_the_remote_safe_receipt_classes() -> None: + summary, receipt, expected_assets = _summary() + changed_receipt = copy.deepcopy(receipt) + changed_receipt["entity_class"] = "custom customer noun" + with pytest.raises(evidence.ProductionEvidenceError, match="remote-safe"): + evidence.validate_production_acceptance_summary( + summary, + qualification_evidence_decision_receipt=changed_receipt, + expected_publication_assets=expected_assets, + ) + + +def test_summary_identity_uses_the_exact_frozen_projection() -> None: + summary, _, _ = _summary() + projection = { + field: summary[field] + for field in ( + "target", + "claim_scope", + "release_identity", + "release_sha256", + "artifact_inventory_sha256", + "publication_staging_sha256", + "qualification_evidence_decision_receipt_reference", + "qualification_admission_reference", + "production_acceptance_manifest_reference", + "campaign_summary", + "authority_state_sha256", + "revocation_state_sha256", + "signer_registry_sha256", + ) + } + assert summary["evidence_identity_sha256"] == ( + "sha256:" + + hashlib.sha256( + evidence.PRODUCTION_EVIDENCE_IDENTITY_DOMAIN + evidence.canonical_json_bytes(projection) + ).hexdigest() + ) + + +def test_summary_rejects_private_counts_or_receipt_drift() -> None: + summary, receipt, expected_assets = _summary() + mutated = copy.deepcopy(summary) + mutated["campaign_summary"]["healthy"]["task_name"] = 1 + with pytest.raises(evidence.ProductionEvidenceError, match="keys"): + evidence.validate_production_acceptance_summary( + mutated, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=expected_assets, + ) + + different_receipt = copy.deepcopy(receipt) + different_receipt["campaign_summary"]["classes"]["healthy"]["observed_trial_count"] = 4 + with pytest.raises(evidence.ProductionEvidenceError, match="public reference"): + evidence.validate_production_acceptance_summary( + summary, + qualification_evidence_decision_receipt=different_receipt, + expected_publication_assets=expected_assets, + ) + + +@pytest.mark.parametrize( + "qualification_class,field,value", + [ + ("healthy", "silent_incorrect_success_count", 1), + ("safe_halt", "over_halt_count", 1), + ("uncertain_delivery", "replay_dispatch_count", 1), + ("uncertain_delivery", "reconciliation_required_count", 2), + ("declared_attended", "authenticated_bound_decision_count", 2), + ("governed_repair", "unverified_direct_action_count", 1), + ], +) +def test_summary_rejects_nonqualifying_class_counts( + qualification_class: str, field: str, value: int +) -> None: + summary, receipt, expected_assets = _summary() + summary["campaign_summary"][qualification_class][field] = value + with pytest.raises(evidence.ProductionEvidenceError): + evidence.validate_production_acceptance_summary( + summary, + qualification_evidence_decision_receipt=receipt, + expected_publication_assets=expected_assets, + ) diff --git a/tests/test_public_evidence_workflow_boundary.py b/tests/test_public_evidence_workflow_boundary.py new file mode 100644 index 0000000..9f439da --- /dev/null +++ b/tests/test_public_evidence_workflow_boundary.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = ROOT / ".github" / "workflows" +LEGACY_REFUSAL = WORKFLOWS / "import-production-acceptance.yml" + + +def test_no_evals_workflow_can_accept_a_private_payload() -> None: + """Keep the legacy importer unreachable until the public cutover removes it.""" + + forbidden_public_inputs = { + "private_export_contract:", + "--private-export-contract", + "--campaign /", + "--qualification-admission /", + } + for workflow in WORKFLOWS.glob("*.yml"): + if workflow == LEGACY_REFUSAL: + continue + text = workflow.read_text(encoding="utf-8") + for token in forbidden_public_inputs: + assert token not in text, f"{workflow.name} accepts private input via {token!r}" + + legacy = LEGACY_REFUSAL.read_text(encoding="utf-8") + assert "Staging private evidence is not implemented, and the import gate is closed." in legacy + assert "if python scripts/import_production_acceptance.py" in legacy + assert "--certificate /nonexistent/certificate.json" in legacy + assert "--campaign /nonexistent/campaign.json" in legacy + assert "--qualification-admission /nonexistent/admission.json" in legacy + assert "--attestation-bundle /nonexistent/bundle.jsonl" in legacy + assert "the importer produced a result while the gate is closed" in legacy + assert 'test ! -e "${RUNNER_TEMP}/derived.json"' in legacy + assert "actions/upload-artifact" not in legacy diff --git a/tests/test_qualification_evidence_v3.py b/tests/test_qualification_evidence_v3.py new file mode 100644 index 0000000..7ef0b1c --- /dev/null +++ b/tests/test_qualification_evidence_v3.py @@ -0,0 +1,342 @@ +from __future__ import annotations + +import copy +import hashlib +import hmac +from typing import Any + +import pytest + +from openadapt_evals import qualification_evidence as evidence + +SECRET = b"test-only-receipt-key" +CLASSES = ( + "declared_attended", + "governed_repair", + "healthy", + "idempotency_replay", + "safe_halt", + "uncertain_delivery", +) + + +def _digest(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode()).hexdigest() + + +def _sign(preimage: bytes) -> bytes: + digest = hmac.new(SECRET, preimage, hashlib.sha256).digest() + return digest + digest + + +def _verify(_key_id: str, preimage: bytes, signature: bytes) -> bool: + return hmac.compare_digest(_sign(preimage), signature) + + +def _facts( + receipt_type: str, + qualification_class: str, + outcome: str, + *, + silent: bool = False, + replay_dispatch_count: int = 0, +) -> dict[str, Any]: + if receipt_type == "runner": + return { + "observed_terminal_outcome": outcome, + "model_call_count": 1 if qualification_class == "governed_repair" else 0, + "unplanned_intervention_count": 0, + "unsafe_effect_count": 0, + } + if receipt_type == "observer": + if qualification_class in {"safe_halt", "uncertain_delivery"}: + return { + "independent_verdict": ( + "PROVED" if qualification_class == "safe_halt" else "UNVERIFIABLE" + ), + "intended_effect_count": 0, + "wrong_effect_count": 0, + "wrong_record_count": 0, + "duplicate_effect_count": 0, + "collateral_effect_count": 0, + } + return { + "independent_verdict": "REFUTED" if silent else "PROVED", + "intended_effect_count": 0 if silent else 1, + "wrong_effect_count": 0, + "wrong_record_count": 0, + "duplicate_effect_count": 0, + "collateral_effect_count": 0, + } + if receipt_type == "delivery": + return { + "dispatch_state": ( + "not_dispatched" if qualification_class == "safe_halt" else "dispatched" + ), + "blind_retry_count": 0, + "replay_dispatch_count": replay_dispatch_count, + "idempotency_result": ( + "duplicate_suppressed" + if qualification_class == "idempotency_replay" + else "not_applicable" + ), + "delivery_certainty": ( + "not_delivered" + if qualification_class == "safe_halt" + else ("uncertain" if qualification_class == "uncertain_delivery" else "delivered") + ), + } + if receipt_type == "decision": + return { + "authenticated_typed_bound_decision": True, + "live_target_revalidated": True, + } + if receipt_type == "policy": + return {"policy_approved_model_path": True} + if receipt_type == "repair": + return { + "human_approval_verified": True, + "retained_evidence_verified": True, + "target_revalidated": True, + "unverified_direct_action_count": 0, + } + if receipt_type == "cleanup": + return {"cleanup_completed": True} + if receipt_type == "cleanup_absence": + return {"absence_verified": True} + raise AssertionError(receipt_type) + + +def _campaign( + *, + silent_class: str | None = None, + over_halt_class: str | None = None, + uncertain_replay_dispatch_count: int = 0, +) -> tuple[dict[str, Any], dict[str, Any]]: + campaign_id = "qualification-campaign-2026-08-27" + permit = _digest("permit") + bundle = _digest("bundle") + runtime = _digest("runtime") + identity = _digest("evidence-identity") + conditions: list[dict[str, Any]] = [] + envelopes: list[dict[str, Any]] = [] + + for qualification_class in CLASSES: + expected = evidence.EXPECTED_OUTCOME_BY_CLASS[qualification_class] + trials: list[dict[str, Any]] = [] + for trial_index in range(1, 4): + outcome = "HALTED" if qualification_class == over_halt_class else expected + common = { + "campaign_id": campaign_id, + "task": "task-a", + "condition": f"{qualification_class}-fault", + "qualification_class": qualification_class, + "trial_index": trial_index, + "attempt_id_sha256": _digest(f"{qualification_class}-{trial_index}-attempt"), + "run_id_sha256": _digest(f"{qualification_class}-{trial_index}-run"), + "campaign_permit_sha256": permit, + "bundle_artifact_sha256": bundle, + "runtime_identity_sha256": runtime, + "evidence_identity_sha256": identity, + } + receipt_types = [ + "runner", + "observer", + "delivery", + "cleanup", + "cleanup_absence", + ] + if qualification_class == "declared_attended": + receipt_types.append("decision") + if qualification_class == "governed_repair": + receipt_types.extend(["policy", "repair"]) + receipt_digests: dict[str, str] = {} + for receipt_type in receipt_types: + facts = _facts( + receipt_type, + qualification_class, + outcome, + silent=( + receipt_type == "observer" + and qualification_class == silent_class + and trial_index == 1 + ), + replay_dispatch_count=( + uncertain_replay_dispatch_count + if receipt_type == "delivery" + and qualification_class == "uncertain_delivery" + and trial_index == 1 + else 0 + ), + ) + projection = { + **common, + "verdict": evidence._derived_receipt_verdict(receipt_type, facts), + "evidence_sha256": _digest( + f"{qualification_class}-{trial_index}-{receipt_type}-evidence" + ), + "facts": facts, + } + receipt = evidence.build_signed_trial_receipt( + receipt_type=receipt_type, + issuer_key_id="test-ed25519-key", + source_artifact_sha256=_digest( + f"{qualification_class}-{trial_index}-{receipt_type}-source" + ), + verified_projection=projection, + verified_at="2026-08-27T12:01:00Z", + signer=_sign, + ) + envelopes.append(receipt) + receipt_digests[receipt_type] = evidence.receipt_sha256(receipt) + trials.append( + { + "schema_version": evidence.TRIAL_SCHEMA, + "task": common["task"], + "condition": common["condition"], + "qualification_class": qualification_class, + "trial_index": trial_index, + "attempt_id_sha256": common["attempt_id_sha256"], + "run_id_sha256": common["run_id_sha256"], + "campaign_permit_sha256": permit, + "bundle_artifact_sha256": bundle, + "runtime_identity_sha256": runtime, + "evidence_identity_sha256": identity, + "started_at": "2026-08-27T12:00:00Z", + "completed_at": "2026-08-27T12:02:00Z", + "observed_terminal_outcome": outcome, + "runner_receipt_sha256": receipt_digests["runner"], + "observer_receipt_sha256": receipt_digests["observer"], + "delivery_receipt_sha256": receipt_digests["delivery"], + "policy_receipt_sha256": receipt_digests.get("policy"), + "decision_receipt_sha256": receipt_digests.get("decision"), + "repair_receipt_sha256": receipt_digests.get("repair"), + "cleanup_receipt_sha256": receipt_digests["cleanup"], + "cleanup_absence_proof_sha256": receipt_digests["cleanup_absence"], + } + ) + conditions.append( + { + "task": "task-a", + "condition": f"{qualification_class}-fault", + "qualification_class": qualification_class, + "expected_terminal_outcome": expected, + "required_trials": 3, + "trials": trials, + } + ) + return evidence.build_qualification_campaign( + campaign_id=campaign_id, + campaign_permit_sha256=permit, + project_contract_sha256=_digest("project"), + source_evidence_manifest_sha256=_digest("source-manifest"), + bundle_artifact_sha256=bundle, + runtime_identity_sha256=runtime, + evidence_identity_sha256=identity, + qualification_contract={"six_class_launch_matrix": True}, + oracle_contract={"independent_effect_observer": True}, + authority_contract={"receipt_signer_registry_sha256": _digest("registry")}, + conditions=conditions, + invariants=[{"id": "no-unsafe-effect", "holds": True}], + excluded_trials=[], + receipt_envelopes=envelopes, + generated_at="2026-08-27T12:03:00Z", + verify_receipt_signature=_verify, + require_admissible=False, + ) + + +def test_v3_campaign_has_six_classes_and_three_trials_per_cell() -> None: + campaign, summary = _campaign() + assert campaign["schema_version"] == "openadapt.qualification-campaign/v3" + assert campaign["source_evidence_manifest_sha256"] == _digest("source-manifest") + assert summary["admissible"] is True + assert summary["task_count"] == 1 + assert summary["condition_count"] == 6 + assert summary["required_trial_count"] == 18 + assert summary["observed_trial_count"] == 18 + assert set(summary["class_summaries"]) == evidence.QUALIFICATION_CLASSES + assert summary["reliability"]["silent_incorrect_success_count"] == 0 + assert summary["reliability"]["over_halt_count"] == 0 + assert summary["reliability"]["unsafe_effect_count"] == 0 + assert summary["reliability"]["blind_retry_count"] == 0 + assert summary["reliability"]["uncertain_delivery_trial_count"] == 3 + assert summary["reliability"]["reconciliation_required_count"] == 3 + assert { + trial["observed_terminal_outcome"] + for condition in campaign["conditions"] + if condition["qualification_class"] == "uncertain_delivery" + for trial in condition["trials"] + } == {"RECONCILIATION_REQUIRED"} + + evidence.validate_qualification_campaign( + campaign, + verify_receipt_signature=_verify, + require_admissible=True, + ) + + +def test_campaign_derives_silent_incorrect_success_from_observer_facts() -> None: + campaign, summary = _campaign(silent_class="healthy") + assert summary["admissible"] is False + assert summary["reliability"]["silent_incorrect_success_count"] == 1 + with pytest.raises(evidence.QualificationEvidenceError, match="silent incorrect"): + evidence.validate_qualification_campaign( + campaign, + verify_receipt_signature=_verify, + require_admissible=True, + ) + + +def test_campaign_derives_over_halt_from_expected_verified_class() -> None: + campaign, summary = _campaign(over_halt_class="healthy") + assert summary["admissible"] is False + assert summary["reliability"]["over_halt_count"] == 3 + with pytest.raises(evidence.QualificationEvidenceError, match="over-halt"): + evidence.validate_qualification_campaign( + campaign, + verify_receipt_signature=_verify, + require_admissible=True, + ) + + +def test_uncertain_delivery_refuses_replay_dispatch() -> None: + campaign, summary = _campaign(uncertain_replay_dispatch_count=1) + assert summary["admissible"] is False + assert summary["reliability"]["replay_dispatch_count"] == 1 + with pytest.raises(evidence.QualificationEvidenceError, match="replay dispatch"): + evidence.validate_qualification_campaign( + campaign, + verify_receipt_signature=_verify, + require_admissible=True, + ) + + +def test_receipt_signature_tamper_fails_closed() -> None: + campaign, _ = _campaign() + mutated = copy.deepcopy(campaign) + signature = mutated["receipt_envelopes"][0]["signature"] + mutated["receipt_envelopes"][0]["signature"] = ( + "A" if signature[0] != "A" else "B" + ) + signature[1:] + with pytest.raises(evidence.QualificationEvidenceError, match="signature is not valid"): + evidence.validate_qualification_campaign( + mutated, + verify_receipt_signature=_verify, + ) + + +def test_type_specific_receipt_slots_must_be_null() -> None: + campaign, _ = _campaign() + mutated = copy.deepcopy(campaign) + healthy = next( + condition + for condition in mutated["conditions"] + if condition["qualification_class"] == "healthy" + ) + healthy["trials"][0]["decision_receipt_sha256"] = _digest("unexpected") + with pytest.raises(evidence.QualificationEvidenceError, match="must be null"): + evidence.validate_qualification_campaign( + mutated, + verify_receipt_signature=_verify, + ) From bd16ad3e44ce648d98615b10899387fd5e425f5a Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 27 Aug 2026 18:28:05 -0400 Subject: [PATCH 6/6] test: repin the production acceptance policy digest after rebase The pinned digest covers the canonical JSON of production_acceptance_policy(). Rebasing onto main combined main's policy with this branch's manifest_schema bump, so neither the old branch pin nor main's pin is correct for the result. Recomputing the policy on both sides shows exactly one field differs: manifest_schema: openadapt.production-acceptance/v1 -> /v2 which is this branch's intended change. The new pin is the digest of that policy. --- tests/test_import_production_acceptance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_import_production_acceptance.py b/tests/test_import_production_acceptance.py index c7815c1..1238f4b 100644 --- a/tests/test_import_production_acceptance.py +++ b/tests/test_import_production_acceptance.py @@ -2474,7 +2474,7 @@ def test_production_acceptance_target_scope_map_is_closed() -> None: "openadapt": "qualified_workflow_launcher_release", } assert MODULE.production_acceptance_policy_sha256() == ( - "sha256:6cd6cfeeb6d430c0e9d3a4a55770cb2d9962c8769edc52fac5cc46bff31bee21" + "sha256:7aadc8434aeea62e88a5d8622e68a3dd325b28a8ca90480b561d40ace0995b48" )