diff --git a/.github/workflows/trivy-changes.yml b/.github/workflows/trivy-changes.yml new file mode 100644 index 0000000000..cf66ca0e81 --- /dev/null +++ b/.github/workflows/trivy-changes.yml @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Trivy Changes + +on: + pull_request: + merge_group: + types: [checks_requested] + workflow_dispatch: + inputs: + base_sha: + description: Base commit SHA to compare + required: true + type: string + head_sha: + description: Candidate commit SHA to compare + required: true + type: string + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + changes: + name: Detect deployment configuration changes + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.default.outputs.should_run || steps.changed.outputs.any_changed }} + steps: + - id: default + if: github.event_name != 'pull_request' + run: echo "should_run=true" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: github.event_name == 'pull_request' + with: + persist-credentials: false + + - id: changed + if: github.event_name == 'pull_request' + uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11 # v42.1.0 + with: + files: | + deploy/docker/** + deploy/helm/** + .trivyignore.yaml + flake.nix + flake.lock + tasks/scripts/trivy-scan.sh + .github/workflows/trivy-changes.yml + + scan: + name: Scan changed deployment configuration + needs: changes + if: needs.changes.outputs.should_run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || inputs.base_sha }} + HEAD_REF: ${{ inputs.head_sha || github.sha }} + defaults: + run: + shell: nix develop --command bash -euo pipefail {0} + steps: + - name: Check out candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.HEAD_REF }} + persist-credentials: false + + - name: Check out baseline + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.BASE_REF }} + path: .trivy-base + persist-credentials: false + + - name: Set up Nix + uses: ./.github/actions/setup-nix + + - name: Scan baseline + env: + TRIVY_SOURCE_ROOT: ${{ github.workspace }}/.trivy-base + TRIVY_IGNORE_FILE: ${{ github.workspace }}/.trivyignore.yaml + TRIVY_REPORT_DIR: ${{ runner.temp }}/trivy-base + run: | + mkdir -p "$TRIVY_REPORT_DIR" + "$GITHUB_WORKSPACE/tasks/scripts/trivy-scan.sh" config + + - name: Scan candidate + env: + TRIVY_REPORT_DIR: ${{ runner.temp }}/trivy-head + run: | + mkdir -p "$TRIVY_REPORT_DIR" + tasks/scripts/trivy-scan.sh config + + - name: Reject new high or critical findings + run: | + tasks/scripts/trivy-scan.sh gate-config-diff \ + "$RUNNER_TEMP/trivy-base" "$RUNNER_TEMP/trivy-head" + + - name: Upload reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: trivy-changes-${{ github.run_id }} + path: | + ${{ runner.temp }}/trivy-base + ${{ runner.temp }}/trivy-head + if-no-files-found: ignore + retention-days: 14 + + result: + name: OpenShell / Trivy Changes + needs: [changes, scan] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check scan result + env: + CHANGES_RESULT: ${{ needs.changes.result }} + SHOULD_RUN: ${{ needs.changes.outputs.should_run }} + SCAN_RESULT: ${{ needs.scan.result }} + run: | + set -euo pipefail + if [ "$CHANGES_RESULT" != "success" ]; then + echo "::error::Change detection concluded $CHANGES_RESULT." + exit 1 + fi + if [ "$SHOULD_RUN" = "true" ] && [ "$SCAN_RESULT" != "success" ]; then + echo "::error::Trivy scan concluded $SCAN_RESULT." + exit 1 + fi + if [ "$SHOULD_RUN" != "true" ]; then + echo "No Helm or Dockerfile changes to scan." + fi diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml new file mode 100644 index 0000000000..b2bc396b4b --- /dev/null +++ b/.github/workflows/trivy-scan.yml @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Trivy Scan + +# Scans the artifacts a release publishes: the final container images and the +# deployment configuration. Callers pass OCI references, so this workflow is +# self-contained and knows nothing about how a release is assembled. +# +# Findings are informational while we learn what this reports in practice: they +# produce a warning, not a failure. A scanner that cannot run still fails, so a +# broken scan cannot look clean. Set fail-on-findings to flip the gate on. + +on: + workflow_call: + inputs: + images: + description: Newline-separated image references to scan + required: false + type: string + default: "" + chart-ref: + description: | + Packaged Helm chart to scan, for example + oci://ghcr.io/nvidia/openshell/helm-chart:0.0.116 + required: false + type: string + default: "" + severity: + description: Severities that fail the workflow + required: false + type: string + default: HIGH,CRITICAL + ignore-unfixed: + description: Ignore image vulnerabilities with no upstream fix + required: false + type: boolean + default: true + fail-on-findings: + description: Fail the run on findings instead of warning + required: false + type: boolean + default: false + upload-sarif: + description: Upload results to GitHub Code Scanning + required: false + type: boolean + default: true + secrets: + CACHIX_AUTH_TOKEN: + description: Token used to write Nix build outputs to Cachix + required: false + + workflow_dispatch: + inputs: + images: + description: Newline-separated image references to scan + required: false + type: string + default: "" + chart-ref: + description: Packaged Helm chart OCI reference to scan + required: false + type: string + default: "" + severity: + description: Severities that fail the workflow + required: false + type: string + default: HIGH,CRITICAL + ignore-unfixed: + description: Ignore image vulnerabilities with no upstream fix + required: false + type: boolean + default: true + fail-on-findings: + description: Fail the run on findings instead of warning + required: false + type: boolean + default: false + upload-sarif: + description: Upload results to GitHub Code Scanning + required: false + type: boolean + default: true + +permissions: + contents: read + +defaults: + run: + shell: nix develop --command bash -euo pipefail {0} + +env: + TRIVY_SEVERITY: ${{ inputs.severity }} + TRIVY_REPORT_DIR: reports/trivy + +jobs: + image: + name: Image vulnerabilities (informational) + if: inputs.images != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Trivy reads ~/.docker/config.json, which `nix develop` leaves alone. + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Set up Nix + uses: ./.github/actions/setup-nix + with: + cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} + + - name: Scan images + id: scan + env: + IMAGES: ${{ inputs.images }} + TRIVY_IGNORE_UNFIXED: ${{ inputs.ignore-unfixed }} + run: | + mapfile -t refs < <(printf '%s\n' "$IMAGES" | sed '/^[[:space:]]*$/d') + tasks/scripts/trivy-scan.sh images "${refs[@]}" + + - name: Upload SARIF to Code Scanning + if: ${{ !cancelled() && steps.scan.conclusion == 'success' && inputs.upload-sarif }} + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + with: + sarif_file: reports/trivy + category: trivy-image + + - name: Upload reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: trivy-image-${{ github.run_id }} + path: reports/trivy + if-no-files-found: ignore + retention-days: 14 + + # Separate from the scan so a tripped gate still publishes its reports. + - name: Report findings + if: ${{ !cancelled() && steps.scan.conclusion == 'success' }} + env: + FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + run: | + set +e + tasks/scripts/trivy-scan.sh gate + status=$? + set -e + case "$status" in + 0) echo "No findings at ${TRIVY_SEVERITY}." ;; + 10) + if [ "$FAIL_ON_FINDINGS" = "true" ]; then + echo "::error::Image findings at ${TRIVY_SEVERITY}." + exit 1 + fi + echo "::warning::Image findings at ${TRIVY_SEVERITY}; this check is informational." + ;; + *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; + esac + + config: + name: Configuration misconfigurations (informational) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # `helm pull` needs registry credentials only when a packaged chart is + # requested, but logging in unconditionally keeps the step list flat. + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Set up Nix + uses: ./.github/actions/setup-nix + with: + cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} + + - name: Scan configuration + id: scan + env: + CHART_REF: ${{ inputs.chart-ref }} + run: | + args=() + if [ -n "$CHART_REF" ]; then + args+=(--chart-ref "$CHART_REF") + fi + tasks/scripts/trivy-scan.sh config "${args[@]}" + + - name: Upload SARIF to Code Scanning + if: ${{ !cancelled() && steps.scan.conclusion == 'success' && inputs.upload-sarif }} + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + with: + sarif_file: reports/trivy + category: trivy-config + + - name: Upload reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: trivy-config-${{ github.run_id }} + path: reports/trivy + if-no-files-found: ignore + retention-days: 14 + + - name: Report findings + if: ${{ !cancelled() && steps.scan.conclusion == 'success' }} + env: + FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + run: | + set +e + tasks/scripts/trivy-scan.sh gate + status=$? + set -e + case "$status" in + 0) echo "No findings at ${TRIVY_SEVERITY}." ;; + 10) + if [ "$FAIL_ON_FINDINGS" = "true" ]; then + echo "::error::Configuration findings at ${TRIVY_SEVERITY}." + exit 1 + fi + echo "::warning::Configuration findings at ${TRIVY_SEVERITY}; this check is informational." + ;; + *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; + esac + + # Republishes whether the scans ran, not what they found, so a broken scanner + # cannot pass as healthy while findings stay informational. + result: + name: OpenShell / Trivy (informational) + needs: [image, config] + if: always() + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The image job is skipped when no references are passed, which + # check-job-results treats as a failure, so drop skipped jobs first. + - name: Drop skipped jobs + id: required + env: + JOB_RESULTS: ${{ toJSON(needs) }} + run: | + { + echo 'results<>"$GITHUB_OUTPUT" + + - uses: ./.github/actions/check-job-results + with: + results: ${{ steps.required.outputs.results }} diff --git a/.gitignore b/.gitignore index 3ef1a37697..a7867fe2fe 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,9 @@ pip-delete-this-directory.txt coverage.out coverage/ htmlcov/ + +# Trivy scan reports (tasks/scripts/trivy-scan-*.sh) +/reports/ .tox/ .nox/ .coverage diff --git a/.trivyignore.yaml b/.trivyignore.yaml new file mode 100644 index 0000000000..893ac16502 --- /dev/null +++ b/.trivyignore.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Trivy exceptions. Passed explicitly with --ignorefile by tasks/scripts/trivy-scan.sh, +# because Trivy auto-loads a plain `.trivyignore` but not the YAML variant. +# +# An entry belongs here only when the finding is wrong: the condition it +# reports is not true of this repository, or it is an artifact of how the scan +# renders the chart. Nothing else qualifies. Findings that describe hardening +# we have not done, or a risk we have accepted, stay in the report where they +# can be seen and argued about, even when that means the gate fails. +# +# Always scope an entry with `paths`, naming individual files. An `id` on its +# own disables the check everywhere, which would also hide a genuine occurrence +# elsewhere. +# +# Use `**/.yaml`. Paths match the location Trivy reports, which is +# relative to the scanned target, and the same template is reported two ways: +# `helm/openshell/templates/x.yaml` when scanning deploy/, and +# `helm-chart-.tgz:templates/x.yaml` when scanning the published chart. +# Only a leading `**/` matches both. `*templates/x.yaml` silently stops applying +# to the repository scan, and `**/templates/x.yaml` to the packaged one. +# +# `paths` is also as narrow as this file can get: for misconfigurations Trivy +# offers no per-occurrence scoping, so a second, legitimate finding of the same +# check in a listed file would be hidden too. Inline `#trivy:ignore:` comments +# would fix that and do work for Dockerfiles, but Trivy 0.74 does not apply them +# to Helm templates. Revisit when it does. + +misconfigurations: + # helm template renders without a namespace, so every workload appears to be + # in "default". The namespace comes from `helm install -n` and no template + # hardcodes one. + - id: KSV-0110 + paths: + - "**/statefulset.yaml" + - "**/deployment.yaml" + statement: >- + An artifact of rendering the chart outside a cluster. The namespace is + supplied at install time. + + # This ConfigMap stores the *name* of a key inside an external Secret + # (proxy_auth_secret_key = "proxy-auth"), not the credential. Scoped to the + # one file, because elsewhere this check is what would catch a real + # credential committed into a ConfigMap. + - id: KSV-01010 + paths: + - "**/gateway-config.yaml" + statement: >- + The ConfigMap holds the name of a key in an external Secret, not a + credential. + + # ghcr.io/nvidia/openshell is where this project publishes its own images. + # Trivy's default trusted-registry list cannot be extended in the version we + # run, so the check cannot be taught about our registry. Scoped to the two + # workload templates so third-party images referenced elsewhere still report. + - id: KSV-0125 + paths: + - "**/statefulset.yaml" + - "**/deployment.yaml" + statement: >- + Images come from ghcr.io/nvidia/openshell, this project's own registry. diff --git a/CI.md b/CI.md index c29abdba0c..480f5626e1 100644 --- a/CI.md +++ b/CI.md @@ -24,7 +24,9 @@ Three opt-in labels enable the long-running E2E suites: When multiple labels are present, `Branch E2E Checks` builds each generic multi-architecture artifact set once and fans out enabled suites in parallel. Runtime-specific reusable workflows define the Docker, Podman, VM, and Kubernetes lanes. Composite actions own the replaceable Podman, KVM, kind, and mise setup. Each lane depends only on the artifact categories it consumes: VM does not wait for container-driver artifacts or supervisor images, and GPU does not wait for the gateway image. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E reuse matching prebuilt gateway and CLI binaries instead of compiling debug binaries in test jobs. Standalone-driver lanes additionally reuse driver-free gateway and compute-driver artifacts. Kubernetes managed-driver lanes consume published gateway and supervisor images, while the standalone-driver lane composes its gateway image from prebuilt binaries. The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. -The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. +The GitHub ruleset should require the `OpenShell / ...` statuses published by +`Required CI Gates` plus the direct `OpenShell / Trivy Changes` result, not the +push-triggered workflow jobs themselves. ## Informational security reports @@ -79,6 +81,81 @@ nix develop --command actionlint -shellcheck= -pyflakes= nix develop --command zizmor --offline --persona=regular --min-severity=high --no-exit-codes . ``` +## Artifact scanning + +`Trivy Scan` differs from the reports above in what it looks at rather than in +how it reports: it scans what a release publishes instead of what a change +contains — the final container images, the Helm charts, the final image +Dockerfiles, and the raw Kubernetes manifests. Nix provides Trivy and Helm, and +the jobs run on GitHub-hosted runners like the other scanners. + +Findings are informational for now, while we learn what the scanner reports in +practice. They raise a warning and the run stays green; a scanner that cannot +run still fails, so a broken scan cannot look clean. The `fail-on-findings` +input flips that to a hard failure once the findings have been worked through. + +The workflow is reusable and takes OCI references as input, so it knows nothing +about how a release is assembled. `HIGH` and `CRITICAL` are what get reported as +findings; everything below is listed without comment. Image scanning +additionally ignores vulnerabilities with no upstream fix, because a base-image +CVE without a patch would otherwise be permanent noise. That option does not +apply to misconfigurations. + +It is not listed in any release workflow's `needs:`, so no publication depends +on it. Wiring it into `release-dev.yml` and `release-tag.yml` is a separate +change, and one that only makes sense once findings fail. + +The configuration scan targets `deploy/` in one pass, which covers both charts, +the published Dockerfiles and the raw manifests. The macOS Dockerfiles export a +binary from `FROM scratch` and the CI image is toolchain rather than a release +artifact, so both are skipped. + +Chart coverage additionally depends on value combinations. The chart defaults +render 10 of the chart's 19 templates, while some conditional resources only +render with overrides stored under `deploy/helm/openshell/ci/values-*.yaml`. +The scan exercises each of these CI fixtures to cover resources such as the +high-availability Deployment, Gateway API objects, OpenShift Route, and broader +workspace-mode ClusterRole. These fixtures are test inputs, not a set of +separately supported product profiles. + +Exceptions live in `.trivyignore.yaml`, one justification per entry. Trivy +auto-loads a plain `.trivyignore` but not the YAML variant, so the scripts pass +`--ignorefile` explicitly. An entry qualifies only when the finding is wrong: +the condition it reports is not true of this repository, or it is an artifact of +how the scan renders the chart. Hardening we have not done and risks we have +accepted stay in the report, where they can be seen and argued about, even when +that means the gate fails. + +Four checks report today: `KSV-0014` (`readOnlyRootFilesystem` unset on the +gateway container), `KSV-0041` and `KSV-0056` (RBAC grants the managed workspace +mode needs and that RBAC cannot express more narrowly), and `DS-0002` (the +supervisor image runs as root by design). Resolving or consciously accepting +each of those is what has to happen before `fail-on-findings` is worth turning +on. + +Scans write full-severity reports and never fail on findings, so a report is +always available to upload; a separate `gate` step re-reads them and applies the +threshold. Run them locally with: + +```shell +nix develop --command tasks/scripts/trivy-scan.sh config +nix develop --command tasks/scripts/trivy-scan.sh images ghcr.io/nvidia/openshell/gateway:dev +nix develop --command tasks/scripts/trivy-scan.sh gate +``` + +### Pull-request change gate + +`Trivy Changes` runs directly on pull requests and merge groups. It detects +changes to Helm charts, release Dockerfiles, and the Trivy tooling, then scans +both the base revision and the candidate with the same scanner logic. The check +fails only when the candidate introduces a new `HIGH` or `CRITICAL` +misconfiguration, so existing findings do not block unrelated work. Reports +from both revisions are retained as workflow artifacts. + +This check analyzes Helm and Dockerfile configuration. It does not build +container images, so package and operating-system CVEs remain the responsibility +of the release-artifact image scan. + ## Commit signing copy-pr-bot decides whether to mirror a PR automatically based on whether the author is trusted. For org members and collaborators, "trusted" means **all commits in the PR are cryptographically signed**. Unsigned commits, even from an org member, force the bot to wait for a maintainer's `/ok to test `. @@ -158,14 +235,18 @@ GitHub merge queue is required for `main`. Repository administrators must enable - `OpenShell / E2E` - `OpenShell / GPU E2E` - `OpenShell / Helm Lint` +- `OpenShell / Trivy Changes` -Do not require the underlying workflow job names directly. `Required CI Gates` publishes stable commit statuses for both PR-head mirror commits and merge-group commits. +`Required CI Gates` publishes the stable statuses for mirror-based workflows. +`Trivy Changes` runs directly on pull requests and merge groups and publishes +its own stable result status. Merge-group runs use the `merge_group` event. The event is distinct from `pull_request` and `push`, and GitHub will not report required checks for queued PRs unless the workflows include it. In this repository: - `Branch Checks` runs the standard non-E2E gates on the merge-group SHA. - `Branch E2E Checks` runs core E2E and GPU E2E for merge groups. Kubernetes HA E2E remains optional and label-driven on PRs. - `Helm Lint` runs for merge groups without the PR diff optimization, because the merge-group branch is the final integration state. +- `Trivy Changes` compares the merge-group configuration with its base and rejects new High or Critical findings. - `Required CI Gates` posts the same `OpenShell / ...` statuses to the merge-group SHA and does not require a `pull-request/` mirror for merge-group events. Maintainers should add ready PRs to the queue rather than pressing a direct merge button. GitHub removes a PR from the queue if the merge-group checks fail or time out. @@ -204,6 +285,8 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma | `.github/workflows/dependency-review.yml` | Reports dependency changes when GitHub Dependency Graph is available; otherwise publishes a neutral warning. | | `.github/workflows/codeql.yml` | Runs nightly informational CodeQL analysis on `main` for Rust and the Go, Python, and TypeScript SDKs and retains SARIF artifacts. | | `.github/workflows/codex-security.yml` | Scans the cumulative diff from the previous stable release to each pre-release candidate and publishes train-scoped SARIF on `main`. | +| `.github/workflows/trivy-changes.yml` | Blocks pull requests and merge groups that introduce new High or Critical Helm or Dockerfile misconfigurations. | +| `.github/workflows/trivy-scan.yml` | Reusable scan of published container images and deployment configuration. Findings are informational by default and can be configured to fail the workflow. | ## Release workflows @@ -223,8 +306,13 @@ Require these statuses in the branch ruleset for PR and merge-queue CI: - `OpenShell / E2E` - `OpenShell / GPU E2E` - `OpenShell / Helm Lint` +- `OpenShell / Trivy Changes` -Do not require the underlying workflow jobs directly. PR workflow jobs only appear after copy-pr-bot mirrors trusted code, and merge-group workflow jobs run on temporary queue branches. The stable `OpenShell / ...` contexts prove the expected workflow completed for the commit that GitHub is about to merge. +For mirror-based workflows, require the statuses published by +`Required CI Gates`, not their underlying jobs. `OpenShell / Trivy Changes` is +the stable result job of the direct pull-request workflow. Together these +contexts prove the expected checks completed for the commit GitHub is about to +merge. Do not add the informational Actionlint, Zizmor, Dependency Review, or CodeQL jobs to the required status list while they remain in observation mode. diff --git a/architecture/build.md b/architecture/build.md index d45cc25666..0d6b5402c7 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -439,6 +439,92 @@ tags and gating stable promotion on qualification results are part of [RFC 0014](../rfc/0014-release-stability/release-qualification.md) and are not implemented yet. +## Artifact Scanning + +Trivy runs in two places: a reusable release-artifact workflow and a +pull-request change gate. + +### Release Artifacts + +`.github/workflows/trivy-scan.yml` is reusable and takes OCI references as +input, so it has no knowledge of how a release is assembled and no dependency on +release job ordering. Nix supplies both Trivy and Helm, and the jobs stay on +GitHub-hosted runners like the other scanners. + +Findings are informational during the observation phase: they warn and the run +stays green, while a scanner that cannot run still fails. The `fail-on-findings` +input turns them into failures, which is a prerequisite for wiring the workflow +into a release `needs:` rather than something to do at the same time. + +Two scopes, with deliberately different reporting semantics: + +- **Images.** `HIGH` and `CRITICAL` are reported, and vulnerabilities with no + upstream fix are ignored. Without that exclusion a base-image CVE with no + available patch would be permanent noise, and a gate nobody can act on once + findings start failing. Published tags are multi-arch indexes and Trivy + defaults to the runner's own platform, so each architecture is scanned + separately. +- **Configuration.** The same severity threshold, but the unfixed exclusion does not + apply to misconfigurations. One pass over `deploy/` covers both charts, the + published Dockerfiles and the raw manifests. Coverage then depends on value + combinations: the chart defaults render 10 of the chart's 19 templates, so + CI value fixtures exercise conditional resources such as the high-availability + Deployment, Gateway API objects, OpenShift Route, and broader workspace-mode + ClusterRole. Each fixture is scanned on its own, and the packaged chart is + scanned from its published OCI reference to cover the artifact consumers + actually install. + +Trivy has no OCI artifact target, and `trivy image` rejects the Helm config media +type, so a packaged chart has to be fetched with `helm pull` before it can be +scanned. Trivy reports locations relative to the scanned target, so +`tasks/scripts/trivy-scan.sh` rewrites SARIF URIs to repository-relative paths; +without that, Code Scanning resolves alerts against files that do not exist. That +rewrite and the profile loop are the only repository-specific logic: severity +filtering, the pass/fail decision and the summary table all come from +`trivy convert --exit-code`, so nothing reimplements counting. + +`.trivyignore.yaml` holds exceptions, and the bar for adding one is that the +finding is wrong: the condition it reports is not true of this repository, or it +is an artifact of how the scan renders the chart. Hardening that has not been +done and risks that have been accepted stay in the report instead, so the +scanner keeps describing the real posture rather than a curated one. Trivy +auto-loads a plain `.trivyignore` but not the YAML variant, so the scripts pass +`--ignorefile` explicitly. + +That bar means four checks report today: `KSV-0014`, `KSV-0041`, `KSV-0056` and +`DS-0002`. Reports are written before findings are evaluated, so a warning or a +failure still publishes SARIF and artifacts. Introducing the tooling and settling +its findings are separate changes, in that order. + +### Pull-Request Change Gate + +`.github/workflows/trivy-changes.yml` gates changes rather than releases. It +runs on `pull_request` and `merge_group`; `workflow_dispatch` takes explicit +base and head SHAs for diagnostics. A detection job decides whether the change +touches `deploy/docker/**`, `deploy/helm/**`, or the scanner inputs themselves +(`.trivyignore.yaml`, `flake.nix`, `flake.lock`, `tasks/scripts/trivy-scan.sh`, +and the workflow file). + +When it does, the scan job checks out both the baseline and the candidate and +runs the candidate's `trivy-scan.sh config` over each tree with the candidate's +`.trivyignore.yaml`, so a scanner or ignore-policy change is judged by its own +rules on both sides. `gate-config-diff` then compares semantic finding +identities — rule ID, target, namespace, message, and cause +provider/service/resource — and fails only on identities absent from the +baseline. The four findings above therefore keep reporting without blocking +every pull request, while a newly introduced `HIGH` or `CRITICAL` +misconfiguration fails the check. Both report sets are uploaded as workflow +artifacts. + +The `result` job publishes a stable `OpenShell / Trivy Changes` status that +succeeds when no relevant files changed, so the check can be required +unconditionally. + +This gate scans Helm and Dockerfile configuration only. It builds no image, so +it cannot detect OS or package CVEs in the image a change would produce. +Final-image vulnerability scanning stays with the release-artifact workflow +above. + See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. ## Docs Site diff --git a/flake.nix b/flake.nix index e361fd1597..1302713450 100644 --- a/flake.nix +++ b/flake.nix @@ -57,7 +57,9 @@ pkg-config # Coverage. lcov + kubernetes-helm syft + trivy uv zizmor zstd diff --git a/scripts/agents/gator/skills/gator-gate/SKILL.md b/scripts/agents/gator/skills/gator-gate/SKILL.md index d66d5029ea..7768a38d1e 100644 --- a/scripts/agents/gator/skills/gator-gate/SKILL.md +++ b/scripts/agents/gator/skills/gator-gate/SKILL.md @@ -961,6 +961,7 @@ Required gates include at least: - `OpenShell / Branch Checks` - `OpenShell / Helm Lint` +- `OpenShell / Trivy Changes` - `OpenShell / E2E` when `test:e2e` is applied - `OpenShell / GPU E2E` when `test:e2e-gpu` is applied diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh new file mode 100755 index 0000000000..bb2588d39e --- /dev/null +++ b/tasks/scripts/trivy-scan.sh @@ -0,0 +1,283 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Scan release artifacts with Trivy. +# +# trivy-scan.sh config [--chart-ref ] +# trivy-scan.sh images [...] +# trivy-scan.sh gate +# trivy-scan.sh gate-config-diff +# +# `config` and `images` write full-severity reports and never fail on findings, +# so a report is always available to upload. `gate` then re-reads those reports +# and fails if any finding reaches TRIVY_SEVERITY. +# +# Environment: +# TRIVY_SEVERITY severities that fail `gate` (default HIGH,CRITICAL) +# TRIVY_IGNORE_UNFIXED skip image vulnerabilities with no fix (default true) +# TRIVY_PLATFORMS image platforms (default "linux/amd64 linux/arm64") +# TRIVY_REPORT_DIR output directory (default reports/trivy) +# TRIVY_SOURCE_ROOT source tree to scan (default repository root) +# TRIVY_IGNORE_FILE ignore file to apply (default repository copy) + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SOURCE_ROOT="${TRIVY_SOURCE_ROOT:-${REPO_ROOT}}" +IGNORE_FILE="${TRIVY_IGNORE_FILE:-${REPO_ROOT}/.trivyignore.yaml}" +cd "${SOURCE_ROOT}" + +SEVERITY="${TRIVY_SEVERITY:-HIGH,CRITICAL}" +REPORT_DIR="${TRIVY_REPORT_DIR:-reports/trivy}" +IGNORE_UNFIXED="${TRIVY_IGNORE_UNFIXED:-true}" +PLATFORMS="${TRIVY_PLATFORMS:-linux/amd64 linux/arm64}" + +# Rendering the chart outside a cluster cannot satisfy the Agent Sandbox API +# discovery check, and that template calls `fail`. +PREFLIGHT_OFF=(--helm-set agentSandbox.preflight.enabled=false) + +# These Dockerfiles produce no runnable image: the macOS ones export a binary +# from `FROM scratch`, and the CI image is toolchain, not a release artifact. +SKIP_DOCKERFILES=( + --skip-files 'deploy/docker/Dockerfile.ci' + --skip-files 'deploy/docker/Dockerfile.*-macos' +) + +# Run one scan. Reports keep every severity; `gate` applies the threshold. +# `prefix` is prepended to SARIF locations, which Trivy reports relative to the +# scanned target while Code Scanning resolves them from the repository root. +scan() { + local subcommand=$1 slug=$2 prefix=$3 + shift 3 + + echo "==> ${slug}" + trivy "${subcommand}" --skip-version-check --quiet \ + --ignorefile "${IGNORE_FILE}" \ + --format json --output "${REPORT_DIR}/${slug}.json" "$@" + trivy convert --quiet \ + --format sarif --output "${REPORT_DIR}/${slug}.sarif" \ + "${REPORT_DIR}/${slug}.json" + + if [ -n "${prefix}" ]; then + jq --arg p "${prefix}" ' + (.. | objects | select(has("artifactLocation")) | .artifactLocation.uri) + |= $p + (. | sub("^[^:]*\\.tgz:"; "")) + ' "${REPORT_DIR}/${slug}.sarif" >"${REPORT_DIR}/${slug}.sarif.tmp" + mv "${REPORT_DIR}/${slug}.sarif.tmp" "${REPORT_DIR}/${slug}.sarif" + fi +} + +# Scanning deploy/ in one pass covers both charts, the published Dockerfiles and +# the raw manifests, and keeps every reported path relative to the same root. +scan_config() { + scan config config-defaults deploy/ "${PREFLIGHT_OFF[@]}" \ + "${SKIP_DOCKERFILES[@]}" deploy + + # The chart defaults render 10 of its 19 templates. The high-availability + # Deployment, the Gateway API objects, the OpenShift Route and the wider + # workspace-mode ClusterRole only render under CI value fixtures. + local values fixture + for values in deploy/helm/openshell/ci/values-*.yaml; do + fixture="$(basename "${values}" .yaml | sed 's/^values-//')" + scan config "config-fixture-${fixture}" deploy/ "${PREFLIGHT_OFF[@]}" \ + "${SKIP_DOCKERFILES[@]}" --helm-values "${values}" deploy + done +} + +# Trivy has no OCI artifact target and rejects the Helm config media type, so a +# published chart has to be pulled before it can be scanned. It reads the +# archive directly, and skips secret scanning on packaged charts. +scan_packaged_chart() { + local ref=$1 + if [[ "${ref}" != *:* || "${ref##*/}" != *:* ]]; then + echo "Error: --chart-ref needs a version tag, e.g. oci://host/chart:1.2.3" >&2 + exit 2 + fi + + local dir + dir="$(mktemp -d)" + trap 'rm -rf "${dir}"' RETURN + + helm pull "${ref%:*}" --version "${ref##*:}" --destination "${dir}" + scan config config-packaged-chart deploy/helm/openshell/ \ + "${PREFLIGHT_OFF[@]}" "$(find "${dir}" -name '*.tgz' -print -quit)" +} + +scan_images() { + local extra=() + [ "${IGNORE_UNFIXED}" = "true" ] && extra+=(--ignore-unfixed) + + local image platform slug + for image in "$@"; do + # Published tags are multi-arch indexes and Trivy defaults to the runner's + # own platform, so each architecture needs its own scan. + for platform in ${PLATFORMS}; do + slug="image-$(printf '%s' "${image#*/}-${platform}" | tr -cs 'A-Za-z0-9._-' '-')" + scan image "${slug}" "" --platform "${platform}" --scanners vuln \ + "${extra[@]}" "${image}" + done + done +} + +# Re-read the reports and apply the threshold. The table doubles as the run +# summary, so nothing here reimplements counting. +gate() { + local report result findings=0 + + if ! compgen -G "${REPORT_DIR}/*.json" >/dev/null; then + echo "Error: no reports in ${REPORT_DIR}; run 'config' or 'images' first" >&2 + exit 2 + fi + + for report in "${REPORT_DIR}"/*.json; do + set +e + trivy convert --quiet --exit-code 10 --severity "${SEVERITY}" \ + --format table "${report}" + result=$? + set -e + + case "${result}" in + 0) ;; + 10) findings=1 ;; + *) + echo "Error: Trivy could not evaluate ${report} (exit ${result})" >&2 + return "${result}" + ;; + esac + done + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Trivy (gate: \`${SEVERITY}\`)" + echo '```' + for report in "${REPORT_DIR}"/*.json; do + trivy convert --quiet --severity "${SEVERITY}" --format table "${report}" + done + echo '```' + } >>"${GITHUB_STEP_SUMMARY}" + fi + + [ "${findings}" -eq 0 ] || return 10 +} + +collect_config_findings() { + local report_dir=$1 + + if ! compgen -G "${report_dir}/*.json" >/dev/null; then + echo "Error: no reports in ${report_dir}" >&2 + return 2 + fi + + jq -s --arg severities "${SEVERITY}" ' + [ + .[] + | .Results[]? as $result + | $result.Misconfigurations[]? + | .Severity as $severity + | select(($severities | split(",") | index($severity)) != null) + | { + key: ([ + .ID, + $result.Target, + (.Namespace // ""), + (.Message // ""), + (.CauseMetadata.Provider // ""), + (.CauseMetadata.Service // ""), + (.CauseMetadata.Resource // "") + ] | @json), + severity: .Severity, + id: .ID, + target: $result.Target, + title: .Title + } + ] + | unique_by(.key) + ' "${report_dir}"/*.json +} + +# Compare semantic finding identities instead of line numbers, so unrelated +# edits that move a finding do not make existing debt look newly introduced. +gate_config_diff() ( + set -euo pipefail + + local baseline_dir=$1 candidate_dir=$2 + local inventory_dir baseline candidate new_findings finding_count + inventory_dir="$(mktemp -d)" + trap 'rm -rf "${inventory_dir}"' EXIT + baseline="${inventory_dir}/baseline.json" + candidate="${inventory_dir}/candidate.json" + new_findings="${inventory_dir}/new.json" + + collect_config_findings "${baseline_dir}" >"${baseline}" + collect_config_findings "${candidate_dir}" >"${candidate}" + jq --slurpfile baseline "${baseline}" ' + ($baseline[0] | map(.key)) as $known + | [.[] | select(.key as $key | ($known | index($key)) == null)] + ' "${candidate}" >"${new_findings}" + + finding_count="$(jq 'length' "${new_findings}")" + if [ "${finding_count}" -eq 0 ]; then + echo "No new configuration findings at ${SEVERITY}." + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "No new Trivy configuration findings at \`${SEVERITY}\`." \ + >>"${GITHUB_STEP_SUMMARY}" + fi + exit 0 + fi + + echo "::error::Trivy reported ${finding_count} new configuration finding(s) at ${SEVERITY}." + jq -r '.[] | "::error::[\(.severity)] \(.id) in deploy/\(.target): \(.title)"' \ + "${new_findings}" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### New Trivy configuration findings" + echo + jq -r '.[] | "- **\(.severity)** `\(.id)` in `deploy/\(.target)`: \(.title)"' \ + "${new_findings}" + } >>"${GITHUB_STEP_SUMMARY}" + fi + exit 10 +) + +command -v trivy >/dev/null || { echo "Error: trivy not on PATH; run inside 'nix develop'" >&2; exit 2; } + +case "${1:-}" in + config) + shift + mkdir -p "${REPORT_DIR}" + scan_config + if [ "${1:-}" = "--chart-ref" ]; then + [ -n "${2:-}" ] || { echo "Error: --chart-ref needs a value" >&2; exit 2; } + scan_packaged_chart "$2" + fi + ;; + images) + shift + [ $# -gt 0 ] || { echo "Error: images needs at least one reference" >&2; exit 2; } + mkdir -p "${REPORT_DIR}" + scan_images "$@" + ;; + gate) + gate + ;; + gate-config-diff) + shift + [ $# -eq 2 ] || { + echo "Error: gate-config-diff needs baseline and candidate report directories" >&2 + exit 2 + } + gate_config_diff "$1" "$2" + ;; + *) + cat >&2 <<'USAGE' +Usage: + trivy-scan.sh config [--chart-ref ] + trivy-scan.sh images [...] + trivy-scan.sh gate + trivy-scan.sh gate-config-diff +USAGE + exit 2 + ;; +esac