From cc7462f8948a90c0776b72906c879100615766e4 Mon Sep 17 00:00:00 2001 From: yifeif <277870278+yifeif-nv@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:45:10 -0700 Subject: [PATCH] ci: check private base access before allocation Signed-off-by: yifeif <277870278+yifeif-nv@users.noreply.github.com> --- .../tests/test_community_dependency_image.py | 205 +++++++++++++++++- .github/workflows/community-ci.yml | 123 ++++++++++- tools/community_gpu_ci.py | 8 +- tools/community_gpu_images.py | 32 ++- tools/tests/test_community_ci.py | 198 ++++++++++++++++- tools/tests/test_community_gpu_images.py | 71 ++++++ 6 files changed, 623 insertions(+), 14 deletions(-) diff --git a/.github/tests/test_community_dependency_image.py b/.github/tests/test_community_dependency_image.py index bc54cc02a1..77f1d344bd 100644 --- a/.github/tests/test_community_dependency_image.py +++ b/.github/tests/test_community_dependency_image.py @@ -15,6 +15,7 @@ import subprocess import sys import time +import urllib.error import urllib.response from email.message import Message from pathlib import Path @@ -1535,7 +1536,7 @@ def test_access_check_cli_uses_only_public_token_and_preserves_safe_result( ("pull_request_target", "refs/heads/main", "maintain", False), ], ) -def test_access_check_job_authorizes_current_actor_without_private_reader( +def test_access_check_job_authorizes_current_actor_without_reader_in_negative_step( tmp_path, event, ref, role, allowed ): caller = yaml.load( @@ -1544,11 +1545,13 @@ def test_access_check_job_authorizes_current_actor_without_private_reader( job = caller["jobs"]["check-dependency-image-access"] assert job["permissions"] == {"contents": "read", "packages": "read"} assert job["environment"]["name"] == "gpu-ci-dispatch" - assert "BREV" not in json.dumps(job) and "REGISTRY_READ_TOKEN" not in json.dumps(job) + assert "BREV" not in json.dumps(job) assert not job.get("needs") step = job["steps"][0] assert step["env"]["REQUEST_ACTOR"] == "${{ github.triggering_actor }}" check = next(s for s in job["steps"] if "access-check \\" in s.get("run", "")) + assert "REGISTRY_READ_TOKEN" not in json.dumps(check) + assert "REGISTRY_READ_TOKEN" not in json.dumps(step) assert check["env"]["GH_TOKEN"] == "${{ github.token }}" assert check["env"]["TRTMC_COMMUNITY_REGISTRY"] == "${{ secrets.TRTMC_COMMUNITY_REGISTRY }}" assert "collaborators/$REQUEST_ACTOR/permission" in check["run"] @@ -1578,6 +1581,204 @@ def test_access_check_job_authorizes_current_actor_without_private_reader( assert "current-rerun-actor" in trace.read_text() +def dedicated_reader_step(): + caller = yaml.load( + (ROOT / ".github/workflows/community-ci.yml").read_text(), Loader=yaml.BaseLoader + ) + job = caller["jobs"]["check-dependency-image-access"] + step = next( + s + for s in job["steps"] + if s["name"] == "Verify the dedicated reader for the same shared base" + ) + assert step["if"] == "${{ inputs.audit_family == 'base' }}" + assert step["timeout-minutes"] == "3" + assert step["env"]["REGISTRY_TOKEN"] == "${{ secrets.TRTMC_COMMUNITY_REGISTRY_READ_TOKEN }}" + negative = next( + s for s in job["steps"] if s["name"] == "Check anonymous and public repository token denial" + ) + evidence = next(s for s in job["steps"] if s["name"] == "Preserve the access denial evidence") + assert job["steps"].index(negative) < job["steps"].index(evidence) < job["steps"].index(step) + return step + + +@pytest.fixture +def dedicated_reader_inline(tmp_path, monkeypatch): + """Execute the actual workflow Python with only I/O boundaries substituted.""" + from tools import community_gpu_ci as ci + from tools import community_gpu_images as images + + inputs = {name: hashlib.sha256(name.encode()).hexdigest() for name in images.BASE_INPUTS} + prefix = "ghcr.io/private-fixture-org/hidden-image-prefix" + digest = "sha256:" + "e" * 64 + catalog = { + "schema_version": 1, + "ci_sha": "f" * 40, + "registry_prefix": prefix, + "families": {}, + "base": { + "schema_version": 1, + "kind": "community-base", + "platform": "linux/amd64", + "registry_visibility": "private", + "image": prefix + "/base@" + digest, + "environment_source_sha": "a" * 40, + "inputs": inputs, + "input_key": images.base_key(inputs), + "cpu_environment_verified": True, + "local_image_id": "sha256:" + "b" * 64, + "package_id": 123, + }, + } + context = SimpleNamespace( + images=images, + catalog=catalog, + prefix=prefix, + digest=digest, + token="sensitive-reader-token", + username="sensitive-reader-account", + roots=[], + requests=[], + docker=[], + configs=[], + metadata_status=None, + docker_failure=None, + ) + monkeypatch.chdir(tmp_path) + for key, value in { + "REGISTRY_TOKEN": context.token, + "REGISTRY_PREFIX": prefix, + "PROTECTED_DEPENDENCY_CATALOG": json.dumps(catalog), + "REQUESTED_DIGEST": digest, + "GITHUB_SHA": catalog["ci_sha"], + }.items(): + monkeypatch.setenv(key, value) + + def export(repository, sha, output, source): + assert repository == tmp_path and sha == catalog["ci_sha"] + assert json.loads(source.read_text()) == catalog + assert stat.S_IMODE(source.stat().st_mode) == 0o600 + context.roots.append(output.parent) + output.write_text(json.dumps(catalog)) + + def transport(opener, request, *, timeout): + assert timeout == 30 and request.get_method() == "GET" + assert request.get_header("Authorization") == "Bearer " + context.token + context.requests.append(request.full_url) + if request.full_url == "https://api.github.com/user": + payload = {"login": context.username} + else: + assert request.full_url == ( + "https://api.github.com/orgs/private-fixture-org/packages/container/" + "hidden-image-prefix%2Fbase" + ) + if context.metadata_status: + headers = Message() + headers["X-OAuth-Scopes"] = "read:packages" + raise urllib.error.HTTPError( + request.full_url, + context.metadata_status, + context.token + context.prefix, + headers, + io.BytesIO(json.dumps({"message": context.token + context.prefix}).encode()), + ) + payload = { + "id": 123, + "name": "hidden-image-prefix/base", + "visibility": "private", + "package_type": "container", + } + response = io.BytesIO(json.dumps(payload).encode()) + response.status = 200 + return response + + def docker(command, environment, stdin, timeout, capture): + assert command[:2] == ["docker", "--config"] + config = Path(command[2]) + assert config.exists() and stat.S_IMODE(config.stat().st_mode) == 0o700 + assert context.token not in json.dumps(environment) + operation = command[3:] + context.docker.append(operation) + context.configs.append(config) + if operation[0] == "login": + assert operation == [ + "login", + "ghcr.io", + "--username", + context.username, + "--password-stdin", + ] + assert stdin == context.token and timeout == 30 + (config / "config.json").write_text(context.token) + else: + assert operation == ["manifest", "inspect", prefix + "/base@" + digest] + assert stdin is None and timeout == 60 + assert (config / "config.json").read_text() == context.token + assert capture is False + if operation[0] == context.docker_failure: + raise subprocess.CalledProcessError(1, command, stderr=context.token + context.prefix) + return context.token + context.prefix + + monkeypatch.setattr(ci, "export_dependency_catalog", export) + monkeypatch.setattr(images.urllib.request.OpenerDirector, "open", transport) + monkeypatch.setattr(images, "_captured", docker) + script = dedicated_reader_step()["run"].split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + context.execute = lambda: exec(compile(script, "dedicated-reader-workflow", "exec"), {}) + return context + + +@pytest.mark.parametrize("metadata_status", [None, 403]) +@pytest.mark.parametrize("docker_failure", [None, "login", "manifest"]) +def test_dedicated_reader_checks_metadata_and_same_manifest_then_erases_auth( + dedicated_reader_inline, metadata_status, docker_failure, capsys +): + context = dedicated_reader_inline + context.metadata_status = metadata_status + context.docker_failure = docker_failure + failure = "" + if metadata_status or docker_failure: + with pytest.raises(SystemExit) as error: + context.execute() + failure = str(error.value) + else: + context.execute() + assert len(context.requests) == 2 + assert [command[0] for command in context.docker] == ( + ["login"] if docker_failure == "login" else ["login", "manifest"] + ) + assert context.roots and all(not path.exists() for path in context.roots + context.configs) + captured = capsys.readouterr() + output = captured.out + captured.err + failure + assert all(value not in output for value in (context.token, context.prefix, context.username)) + if metadata_status: + assert "HTTP 403" in output + if not docker_failure: + assert "Dedicated reader pinned manifest: accessible" in output + if not metadata_status and not docker_failure: + assert "Dedicated reader verified:" in output + + +@pytest.mark.parametrize("invalid", ["digest", "registry", "mutable_catalog"]) +def test_dedicated_reader_rejects_different_or_mutable_reference_before_login( + dedicated_reader_inline, monkeypatch, invalid, capsys +): + context = dedicated_reader_inline + if invalid == "digest": + monkeypatch.setenv("REQUESTED_DIGEST", "sha256:" + "0" * 64) + elif invalid == "registry": + monkeypatch.setenv("REGISTRY_PREFIX", "ghcr.io/different-owner/different-image") + else: + context.catalog["base"]["image"] = context.prefix + "/base:latest" + monkeypatch.setenv("PROTECTED_DEPENDENCY_CATALOG", json.dumps(context.catalog)) + with pytest.raises(SystemExit) as error: + context.execute() + assert context.requests == [] and context.docker == [] + assert context.roots and all(not path.exists() for path in context.roots) + captured = capsys.readouterr() + output = captured.out + captured.err + str(error.value) + assert context.token not in output and context.prefix not in output + + PRIVATE_PREFIX = "ghcr.io/test-owner/private-dependencies" PRIVATE_TOKEN = "candidate-read-secret" CANDIDATE_DIGEST = "sha256:" + "c" * 64 diff --git a/.github/workflows/community-ci.yml b/.github/workflows/community-ci.yml index b5d2d0ec17..088cecf3dd 100644 --- a/.github/workflows/community-ci.yml +++ b/.github/workflows/community-ci.yml @@ -235,6 +235,61 @@ jobs: if-no-files-found: ignore retention-days: 7 + - name: Verify the dedicated reader for the same shared base + if: ${{ inputs.audit_family == 'base' }} + timeout-minutes: 3 + env: + REGISTRY_TOKEN: ${{ secrets.TRTMC_COMMUNITY_REGISTRY_READ_TOKEN }} + REGISTRY_PREFIX: ${{ secrets.TRTMC_COMMUNITY_REGISTRY }} + PROTECTED_DEPENDENCY_CATALOG: ${{ secrets.TRTMC_COMMUNITY_DEPENDENCY_CATALOG }} + REQUESTED_DIGEST: ${{ inputs.audit_digest }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + import tempfile + from pathlib import Path + from tools import community_gpu_ci as ci + from tools import community_gpu_images as images + + try: + with tempfile.TemporaryDirectory(prefix="private-base-reader-") as directory: + root = Path(directory) + source, catalog = root / "input.json", root / "catalog.json" + source.write_text(os.environ["PROTECTED_DEPENDENCY_CATALOG"]) + source.chmod(0o600) + ci.export_dependency_catalog(Path.cwd(), os.environ["GITHUB_SHA"], catalog, source) + data = ci._dependency_catalog(catalog) + reference = os.environ["REGISTRY_PREFIX"] + "/base@" + os.environ["REQUESTED_DIGEST"] + if data["base"]["image"] != reference: + raise images.ImagePreparationError("Access checks must use the same protected base") + token = os.environ.get("REGISTRY_TOKEN", "").strip() + username = images.registry_reader_login(token) + failures = [] + try: + images.require_private_package(data["registry_prefix"], token, data["base"].get("package_id")) + print("Private package metadata: verified") + except images.ImagePreparationError as error: + failures.append(str(error)) + print("Private package metadata: " + str(error)) + config = root / "docker-auth" + config.mkdir(mode=0o700) + try: + images.docker(["login", "ghcr.io", "--username", username, "--password-stdin"], + stdin=token, config=str(config), private=True, timeout=30) + images.docker(["manifest", "inspect", reference], + config=str(config), private=True, timeout=60) + print("Dedicated reader pinned manifest: accessible") + except images.ImagePreparationError as error: + failures.append(str(error)) + print("Dedicated reader pinned manifest: unavailable") + if failures: + raise images.ImagePreparationError("; ".join(failures)) + print("Dedicated reader verified: private package metadata and pinned base manifest") + except (RuntimeError, OSError, ValueError) as error: + raise SystemExit("Private base reader preflight failed: " + str(error)) from None + PY + produce-dependency-image: if: ${{ github.event_name == 'workflow_dispatch' && inputs.task == 'dependency-image' }} permissions: @@ -1111,6 +1166,7 @@ jobs: contents: read outputs: conclusion: ${{ steps.result.outputs.conclusion }} + failure_class: ${{ steps.classification.outputs.failure_class }} instance_name: ${{ steps.reserve.outputs.instance_name }} lease_artifact_name: ${{ steps.reserve.outputs.lease_artifact_name }} instance_type: ${{ steps.reserve.outputs.instance_type }} @@ -1437,6 +1493,51 @@ jobs: --summary-file "$GITHUB_STEP_SUMMARY" \ > "$RUNNER_TEMP/trtmc-community-gpu-summary.json" + - name: Classify the GPU result + if: always() + id: classification + env: + SUMMARY_FILE: ${{ runner.temp }}/trtmc-community-gpu-summary.json + run: | + python3 - <<'PY' + import json + import os + from pathlib import Path + from tools.community_gpu_ci import _checked_summary + + # Only a validated terminal coordinator receipt can attribute a + # failure to the PR. Provision/setup failures have no such receipt. + category = "infra_failure" + try: + path = Path(os.environ["SUMMARY_FILE"]) + if path.stat().st_size > 4 * 1024 * 1024: + raise ValueError("oversized GPU summary") + summary = _checked_summary(json.loads(path.read_text())) + if summary is not None: + active = [row for row in summary["families"] if row["status"] != "deferred"] + failures = [row for row in active if row["status"] != "passed"] + valid_passes = all( + row.get("entrypoint_started") is True + and row.get("failure_class") is None + and bool(row["cases"]) + and all(value == "passed" for value in row["cases"].values()) + for row in active if row["status"] == "passed" + ) + if summary["passed"] and summary["complete"] and valid_passes: + category = "none" + elif failures and valid_passes and all( + row["status"] == "failed" + and row.get("entrypoint_started") is True + and row.get("failure_class") == "pr_failure" + for row in failures + ): + category = "pr_failure" + except (OSError, ValueError, TypeError, RecursionError): + pass + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"failure_class={category}\n") + PY + - name: Record the step conclusion if: always() id: result @@ -1642,6 +1743,7 @@ jobs: GPU_AUTHORIZED: ${{ needs.gpu-authorize.result }} GPU_RESULT: ${{ needs.provision-and-test.result }} TEST_RESULT: ${{ needs.provision-and-test.outputs.conclusion }} + GPU_FAILURE_CLASS: ${{ needs.provision-and-test.outputs.failure_class }} CLEANUP_RESULT: ${{ needs.cleanup.result }} run: | set -euo pipefail @@ -1659,6 +1761,7 @@ jobs: test "$RUN_GPU" = true test "$GPU_RESULT" = success test "$TEST_RESULT" = success + test "$GPU_FAILURE_CLASS" = none test "$CLEANUP_RESULT" = success fi @@ -1667,7 +1770,10 @@ jobs: env: GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ needs.authorize.outputs.head_sha }} + CPU_RESULT: ${{ needs.required.result }} + RUN_GPU: ${{ needs.gpu-authorize.outputs.run_gpu }} GPU_RESULT: ${{ needs.provision-and-test.result }} + GPU_FAILURE_CLASS: ${{ needs.provision-and-test.outputs.failure_class }} OWNER_RELEASE_CONFIRMED: ${{ needs.provision-and-test.outputs.cleanup_confirmed }} BACKSTOP_RELEASE_CONFIRMED: ${{ needs.cleanup.outputs.cleanup_confirmed }} NO_ALLOCATION_CONFIRMED: ${{ needs.provision-and-test.outputs.no_allocation_confirmed }} @@ -1677,16 +1783,25 @@ jobs: [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] state=failure description="Complete Community CI: failure" - if [ "${NO_ALLOCATION_CONFIRMED:-}" = true ]; then - description="Complete Community CI: infra_failure" - elif [ "$GPU_RESULT" != skipped ] \ + if [ "$GPU_RESULT" != skipped ] \ && [ "$OWNER_RELEASE_CONFIRMED" != true ] \ && [ "$BACKSTOP_RELEASE_CONFIRMED" != true ]; then state=pending description="Community CI release remains unconfirmed" - elif [ "$COMPLETE_RESULT" = success ]; then + elif [ "$COMPLETE_RESULT" = success ] \ + && { [ "$RUN_GPU" != true ] || [ "$GPU_FAILURE_CLASS" = none ]; }; then state=success description="Complete Community CI: success" + elif [ "$CPU_RESULT" = success ] && [ "$RUN_GPU" = true ]; then + # Missing, malformed or interrupted GPU receipts must not blame + # the PR for a failure before its entry point was reached. + state=error + description="Complete Community CI: infra_failure" + if [ "$GPU_FAILURE_CLASS" = pr_failure ] \ + && [ "${NO_ALLOCATION_CONFIRMED:-}" != true ]; then + state=failure + description="Complete Community CI: pr_failure" + fi fi gh api --silent --method POST \ "/repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ diff --git a/tools/community_gpu_ci.py b/tools/community_gpu_ci.py index 0ce180eaa9..9e3390630c 100644 --- a/tools/community_gpu_ci.py +++ b/tools/community_gpu_ci.py @@ -352,10 +352,14 @@ def staging_hub_requirement(repository: Path, ci_sha: str) -> str: def require_shared_base(catalog_path: Path, token: str) -> str: catalog = _dependency_catalog(catalog_path) - _image_preparation().validate_base(catalog.get("base"), _dependency_registry(catalog)) + preparation = _image_preparation() + prefix = _dependency_registry(catalog) + base = preparation.validate_base(catalog.get("base"), prefix) if not token.strip(): raise CommunityGpuError("Configure the dedicated private base reader before allocation") - return _image_preparation().registry_reader_login(token) + login = preparation.registry_reader_login(token) + preparation.require_private_package(prefix, token.strip(), base.get("package_id")) + return login def native_cli_library(declaration: Path) -> str | None: diff --git a/tools/community_gpu_images.py b/tools/community_gpu_images.py index ad50aa42f6..e32082e2db 100644 --- a/tools/community_gpu_images.py +++ b/tools/community_gpu_images.py @@ -21,6 +21,7 @@ import subprocess import tempfile import time +import urllib.error import urllib.parse import urllib.request @@ -183,8 +184,37 @@ def require_private_package(prefix: str, token: str, expected_id: int | None) -> if response.status != 200 or len(raw) > MAX_FILE: raise ImagePreparationError("Private base metadata is unavailable") metadata = object_json(raw) + except urllib.error.HTTPError as error: + details = [f"HTTP {error.code}"] + headers = error.headers or {} + scopes = headers.get("X-OAuth-Scopes") + if scopes is not None: + present = "read:packages" in {scope.strip() for scope in scopes.split(",")} + details.append("token read:packages scope=" + ("present" if present else "absent")) + if headers.get("X-GitHub-SSO", "").lower().startswith("required"): + details.append("organization SSO authorization required") + # Return only recognized reason labels, never the API response, private + # package coordinates, authorization URL, token, or arbitrary text. + try: + raw = error.read(8193) + message = object_json(raw).get("message", "") if len(raw) <= 8192 else "" + message = message.lower() if isinstance(message, str) else "" + except (ImagePreparationError, OSError, ValueError, TypeError): + message = "" + for words, reason in ( + (("ip allow list", "ip allowlist"), "organization IP allowlist restriction"), + (("oauth app access restrictions",), "organization OAuth app restriction"), + (("rate limit",), "GitHub API rate limit"), + (("bad credentials",), "credential rejected"), + (("resource not accessible by personal access token",), "token access rejected"), + ): + if any(word in message for word in words): + details.append(reason) + raise ImagePreparationError( + "Private base access could not be verified (" + "; ".join(details) + ")" + ) from None except (OSError, ValueError): - raise ImagePreparationError("Private base access could not be verified") from None + raise ImagePreparationError("Private base API network request failed") from None if ( metadata.get("name") != package or metadata.get("visibility") != "private" diff --git a/tools/tests/test_community_ci.py b/tools/tests/test_community_ci.py index a6c3fd7b03..79382e9671 100644 --- a/tools/tests/test_community_ci.py +++ b/tools/tests/test_community_ci.py @@ -815,6 +815,7 @@ def test_gpu_step_conclusion_requires_completed_success( "GPU_AUTHORIZED", "GPU_RESULT", "TEST_RESULT", + "GPU_FAILURE_CLASS", "CLEANUP_RESULT", ], ) @@ -831,6 +832,7 @@ def test_complete_pipeline_requires_every_cpu_and_gpu_stage(failed, bad_result): ), "success", ) + states["GPU_FAILURE_CLASS"] = "none" if failed: states[failed] = bad_result result = subprocess.run( @@ -892,6 +894,7 @@ def test_gpu_status_and_cleanup_fail_closed( assert "|| true" not in cleanup["run"] assert job["outputs"] == { "conclusion": "${{ steps.result.outputs.conclusion }}", + "failure_class": "${{ steps.classification.outputs.failure_class }}", "instance_name": "${{ steps.reserve.outputs.instance_name }}", "lease_artifact_name": "${{ steps.reserve.outputs.lease_artifact_name }}", "instance_type": "${{ steps.reserve.outputs.instance_type }}", @@ -1715,13 +1718,21 @@ def test_trusted_inner_status_requires_release_before_a_gpu_verdict( assert "always()" in step["if"] and "workflow_dispatch" in step["if"] assert "inputs.ci_lane == 'dev'" in step["if"] assert step["env"]["HEAD_SHA"] == "${{ needs.authorize.outputs.head_sha }}" + assert step["env"]["CPU_RESULT"] == "${{ needs.required.result }}" + assert step["env"]["RUN_GPU"] == "${{ needs.gpu-authorize.outputs.run_gpu }}" + assert step["env"]["GPU_FAILURE_CLASS"] == ( + "${{ needs.provision-and-test.outputs.failure_class }}" + ) writes = tmp_path / "writes" result = subprocess.run( ["bash", "-c", 'gh() { printf "%s\\n" "$@" > "$STATUS_WRITES"; }\n' + step["run"]], env={ **os.environ, "HEAD_SHA": "a" * 40, + "CPU_RESULT": "success" if gpu != "skipped" else "failure", + "RUN_GPU": "true" if gpu != "skipped" else "false", "GPU_RESULT": gpu, + "GPU_FAILURE_CLASS": "none" if gpu == "success" else "pr_failure", "OWNER_RELEASE_CONFIRMED": owner, "BACKSTOP_RELEASE_CONFIRMED": backstop, "COMPLETE_RESULT": complete, @@ -1852,7 +1863,10 @@ def test_preparation_failure_finalizes_infra_without_brev_credentials(tmp_path): env={ "PATH": os.environ["PATH"], "HEAD_SHA": "a" * 40, + "CPU_RESULT": "success", + "RUN_GPU": "true", "GPU_RESULT": "failure", + "GPU_FAILURE_CLASS": "infra_failure", "OWNER_RELEASE_CONFIRMED": "true", "BACKSTOP_RELEASE_CONFIRMED": "true", "NO_ALLOCATION_CONFIRMED": "true", @@ -1867,10 +1881,173 @@ def test_preparation_failure_finalizes_infra_without_brev_credentials(tmp_path): timeout=5, ) assert result.returncode == 0 and not calls.exists() - assert "state=failure" in writes.read_text().splitlines() + assert "state=error" in writes.read_text().splitlines() assert "description=Complete Community CI: infra_failure" in writes.read_text().splitlines() +@pytest.mark.parametrize( + "receipt,expected_class", + [ + ("missing", "infra_failure"), + ("null", "infra_failure"), + ("malformed", "infra_failure"), + ("oversized", "infra_failure"), + ("wrong-schema", "infra_failure"), + ("pre-entry", "infra_failure"), + ("interrupted", "infra_failure"), + ("missing-entry-marker", "infra_failure"), + ("mixed", "infra_failure"), + ("false-success", "infra_failure"), + ("pr-failure", "pr_failure"), + ("post-entry-oom", "pr_failure"), + ("success", "none"), + ], +) +@pytest.mark.parametrize("released", [False, True]) +def test_gpu_receipt_classification_reaches_only_a_released_final_status( + tmp_path, receipt, expected_class, released +): + """Run the real workflow scripts, including missing pre-coordinator output.""" + row = { + "family": "example", + "status": "failed", + "phase": "validation", + "failure_class": "pr_failure", + "entrypoint_started": True, + "requested_cases": ["one"], + "cases": {"one": "failed"}, + } + report = {"schema_version": 1, "families": [row], "passed": True, "complete": True} + if receipt == "wrong-schema": + report["schema_version"] = 2 + elif receipt == "pre-entry": + row.update(phase="checkpoints", failure_class="infra_failure", entrypoint_started=False) + row["cases"]["one"] = "not_run" + elif receipt == "interrupted": + row.update(status="running", phase="container") + row["cases"]["one"] = "not_run" + elif receipt == "missing-entry-marker": + row.pop("entrypoint_started") + elif receipt == "mixed": + report["families"].append( + { + **row, + "family": "other", + "failure_class": "infra_failure", + "entrypoint_started": False, + } + ) + elif receipt == "post-entry-oom": + row.update(phase="container", evidence="Docker reported OOMKilled for this container") + row["cases"]["one"] = "not_run" + elif receipt in {"success", "false-success"}: + row.update(status="passed", phase="complete", failure_class=None) + row["cases"]["one"] = "passed" if receipt == "success" else "not_run" + path = tmp_path / "summary.json" + if receipt != "missing": + payload = { + "null": "null\n", + "malformed": "{\n", + "oversized": " " * (4 * 1024 * 1024 + 1), + }.get(receipt, json.dumps(report)) + path.write_text(payload) + output = tmp_path / "output" + classify = _workflow_step_script( + "community-ci.yml", "provision-and-test", "Classify the GPU result" + ) + result = subprocess.run( + ["bash", "-c", classify], + cwd=REPO_ROOT, + env={**os.environ, "SUMMARY_FILE": str(path), "GITHUB_OUTPUT": str(output)}, + capture_output=True, + text=True, + timeout=10, + ) + assert result.returncode == 0, result.stderr + assert output.read_text() == f"failure_class={expected_class}\n" + + publish = _workflow_step_script( + "community-ci.yml", "publish", "Publish the released Dev workflow conclusion" + ) + writes = tmp_path / "writes" + success = receipt in {"success", "false-success"} + result = subprocess.run( + ["bash", "-c", 'gh() { printf "%s\\n" "$@" > "$STATUS_WRITES"; }\n' + publish], + env={ + **os.environ, + "HEAD_SHA": "a" * 40, + "CPU_RESULT": "success", + "RUN_GPU": "true", + "GPU_RESULT": "success" if success else "failure", + "GPU_FAILURE_CLASS": expected_class, + "OWNER_RELEASE_CONFIRMED": "true" if released else "", + "BACKSTOP_RELEASE_CONFIRMED": "", + "NO_ALLOCATION_CONFIRMED": "", + "COMPLETE_RESULT": "success" if success else "failure", + "GITHUB_REPOSITORY": "example/source", + "GITHUB_SERVER_URL": "https://github.com", + "GITHUB_RUN_ID": "42", + "STATUS_WRITES": str(writes), + }, + capture_output=True, + text=True, + timeout=5, + ) + assert result.returncode == 0, result.stderr + arguments = writes.read_text().splitlines() + expected_state = {"none": "success", "infra_failure": "error", "pr_failure": "failure"}[ + expected_class + ] + if not released: + assert "state=pending" in arguments + assert "description=Community CI release remains unconfirmed" in arguments + else: + assert f"state={expected_state}" in arguments + description = "success" if expected_class == "none" else expected_class + assert f"description=Complete Community CI: {description}" in arguments + + +@pytest.mark.parametrize("category", ["", "unrecognized", "infra_failure"]) +@pytest.mark.parametrize("cpu", ["success", "failure"]) +def test_absent_classification_output_defaults_to_infra_only_after_cpu_passes( + tmp_path, category, cpu +): + """A skipped/failed classifier cannot blame the PR or relabel CPU failures.""" + publish = _workflow_step_script( + "community-ci.yml", "publish", "Publish the released Dev workflow conclusion" + ) + writes = tmp_path / "writes" + result = subprocess.run( + ["bash", "-c", 'gh() { printf "%s\\n" "$@" > "$STATUS_WRITES"; }\n' + publish], + env={ + **os.environ, + "HEAD_SHA": "a" * 40, + "CPU_RESULT": cpu, + "RUN_GPU": "true", + "GPU_RESULT": "failure", + "GPU_FAILURE_CLASS": category, + "OWNER_RELEASE_CONFIRMED": "true", + "BACKSTOP_RELEASE_CONFIRMED": "", + "NO_ALLOCATION_CONFIRMED": "", + "COMPLETE_RESULT": "failure", + "GITHUB_REPOSITORY": "example/source", + "GITHUB_SERVER_URL": "https://github.com", + "GITHUB_RUN_ID": "42", + "STATUS_WRITES": str(writes), + }, + capture_output=True, + text=True, + timeout=5, + ) + assert result.returncode == 0, result.stderr + arguments = writes.read_text().splitlines() + expected_state, expected_class = ( + ("error", "infra_failure") if cpu == "success" else ("failure", "failure") + ) + assert f"state={expected_state}" in arguments + assert f"description=Complete Community CI: {expected_class}" in arguments + + @pytest.mark.parametrize("ready_after", [1, 2, 7]) def test_pr_trigger_waits_for_github_merge_generation_with_a_bounded_retry(tmp_path, ready_after): head, base, merge, tree = (value * 40 for value in "abcd") @@ -2592,16 +2769,22 @@ def git(*args): } lock.write_text('{"resources":{"host_ram_gib":256},"image":"attacker:latest"}') # The actual trusted Python admission runs in a subprocess. Supply a fixed - # /user response at its HTTP boundary; resolver validation has unit coverage. + # identity/package responses at its HTTP boundary; parsers have unit coverage. http_fixture = tmp_path / "identity-http-fixture" http_fixture.mkdir() (http_fixture / "sitecustomize.py").write_text( - "import io, json, os, urllib.request\n" + "import io, json, os, urllib.request, urllib.error\n" "from types import SimpleNamespace\n" "def identity(request, timeout):\n" - " assert request.full_url == 'https://api.github.com/user'\n" " assert request.get_header('Authorization') == 'Bearer dedicated-test-reader-token'\n" " assert timeout > 0\n" + " if request.full_url == 'https://api.github.com/orgs/test-owner/packages/container/private-dependencies%2Fbase':\n" + " if os.environ.get('PACKAGE_FIXTURE_MODE') == 'denied':\n" + " raise urllib.error.HTTPError(request.full_url, 403, 'Forbidden', {}, io.BytesIO(b'{}'))\n" + " response = io.BytesIO(json.dumps({'name':'private-dependencies/base','package_type':'container','visibility':'private'}).encode())\n" + " response.status = 200\n" + " return response\n" + " assert request.full_url == 'https://api.github.com/user'\n" " if os.environ.get('IDENTITY_FIXTURE_MODE') == 'error':\n" " raise OSError('dedicated-test-reader-token upstream error')\n" " login = 'test-reader' if os.environ.get('IDENTITY_FIXTURE_MODE') != 'invalid' else 'invalid\\nlogin'\n" @@ -2682,7 +2865,8 @@ def test_trusted_host_selection_step_runs_before_any_allocation( @pytest.mark.parametrize( - "fault", ["registry", "base", "token", "identity", "invalid_identity", "base_inputs"] + "fault", + ["registry", "base", "token", "identity", "invalid_identity", "base_inputs", "package_access"], ) def test_invalid_protected_catalog_stops_the_real_preallocation_step(tmp_path, gpu_job, fault): repository, catalog, environment = _host_selection_fixture(tmp_path) @@ -2692,6 +2876,8 @@ def test_invalid_protected_catalog_stops_the_real_preallocation_step(tmp_path, g del catalog["base"] elif fault in {"identity", "invalid_identity"}: environment["IDENTITY_FIXTURE_MODE"] = "error" if fault == "identity" else "invalid" + elif fault == "package_access": + environment["PACKAGE_FIXTURE_MODE"] = "denied" elif fault == "base_inputs": from tools import community_gpu_images as images @@ -2715,6 +2901,8 @@ def test_invalid_protected_catalog_stops_the_real_preallocation_step(tmp_path, g assert secret not in result.stdout + result.stderr assert "dedicated-test-reader-token" not in result.stdout + result.stderr assert not list(tmp_path.glob("trtmc-protected-dependency-catalog.*")) + if fault == "package_access": + assert "HTTP 403" in result.stderr def test_failed_blocking_reservation_stops_normal_test_admission(tmp_path, gpu_job): diff --git a/tools/tests/test_community_gpu_images.py b/tools/tests/test_community_gpu_images.py index 5690249297..006107ac39 100644 --- a/tools/tests/test_community_gpu_images.py +++ b/tools/tests/test_community_gpu_images.py @@ -3,6 +3,7 @@ import copy import hashlib +import io import json import os from pathlib import Path @@ -410,6 +411,76 @@ def test_missing_or_invalid_reader_token_never_sends_an_auth_request(monkeypatch images.registry_reader_login(token) +@pytest.mark.parametrize( + "status,headers,message,expected", + [ + ( + 403, + { + "X-OAuth-Scopes": "read:packages", + "X-GitHub-SSO": "required;url=https://private.invalid/secret", + }, + "Forbidden", + "organization SSO authorization required", + ), + (403, {"X-OAuth-Scopes": "repo"}, "Forbidden", "token read:packages scope=absent"), + ( + 403, + {}, + "Private organization has an IP allow list containing secret details", + "organization IP allowlist restriction", + ), + (403, {}, "API rate limit exceeded: secret", "GitHub API rate limit"), + (401, {}, "Bad credentials: secret", "credential rejected"), + ( + 403, + {}, + "Resource not accessible by personal access token: secret", + "token access rejected", + ), + (404, {}, "Not Found: private coordinates and secret", "HTTP 404"), + ], +) +def test_package_access_errors_keep_safe_http_evidence( + monkeypatch, status, headers, message, expected +): + def open_request(request, *, timeout): + assert request.get_header("Authorization") == "Bearer reader-secret" + assert ( + request.full_url + == "https://api.github.com/orgs/example/packages/container/private-ci%2Fbase" + ) + raise urllib.error.HTTPError( + request.full_url, + status, + "private response", + headers, + io.BytesIO(json.dumps({"message": message}).encode()), + ) + + monkeypatch.setattr( + images.urllib.request, "build_opener", lambda *_: SimpleNamespace(open=open_request) + ) + with pytest.raises(images.ImagePreparationError) as error: + images.require_private_package(PREFIX, "reader-secret", None) + text = str(error.value) + assert f"HTTP {status}" in text and expected in text + assert "reader-secret" not in text and "private.invalid" not in text + assert "private-ci" not in text and "private coordinates" not in text and "secret" not in text + + +def test_package_network_error_is_not_misreported_as_token_rejection(monkeypatch): + def fail(*args, **kwargs): + raise OSError("private-url reader-secret certificate error") + + monkeypatch.setattr( + images.urllib.request, "build_opener", lambda *_: SimpleNamespace(open=fail) + ) + with pytest.raises(images.ImagePreparationError, match="API network request failed") as error: + images.require_private_package(PREFIX, "reader-secret", None) + assert "reader-secret" not in str(error.value) and "private-url" not in str(error.value) + + def test_real_process_output_is_bounded_during_read(monkeypatch): monkeypatch.setattr(images, "MAX_FILE", 1024) with pytest.raises(images.ImagePreparationError, match="size limit"):