From e2fa4b57901eb3c02d6718ec553dff960c96f97c Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Wed, 2 Sep 2026 08:17:52 -0500 Subject: [PATCH] feat: add registry artifact integrity check Adds a scheduled check that verifies module tarball responses from registry.coder.com originate from the registry application. Every response from the registry application sets a Server-Version header via serverVersionMiddleware, so a response without one did not come from the registry application. The check probes a rotating sample of tarball URLs and asserts the header is present. Kept separate from check-registry-site-health: that workflow reports availability to the public status page, whereas a failure here warrants engineering triage rather than an automatic public status update. This workflow has no Instatus credentials in scope. Scope is limited to verifying who answered the request, not what was served; content verification against release tags is not covered here. --- .github/scripts/check_registry_integrity.sh | 158 ++++++++++++++++++ .../workflows/check_registry_integrity.yaml | 33 ++++ 2 files changed, 191 insertions(+) create mode 100755 .github/scripts/check_registry_integrity.sh create mode 100644 .github/workflows/check_registry_integrity.yaml diff --git a/.github/scripts/check_registry_integrity.sh b/.github/scripts/check_registry_integrity.sh new file mode 100755 index 000000000..455486108 --- /dev/null +++ b/.github/scripts/check_registry_integrity.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Artifact integrity check for registry.coder.com. +# +# Kept separate from check_registry_site_health.sh. That script answers "is the +# registry reachable?" and reports availability to the public status page. This +# one answers "are responses coming from the registry application?", which +# warrants engineering triage rather than an automatic public status update, so +# this script has no Instatus credentials in scope. +# +# Every response from the registry application passes through +# serverVersionMiddleware and carries a Server-Version header, so a response +# without one did not come from the registry application. +# +# Scope: this verifies who answered the request, not what was served. It does +# not validate tarball contents against their source. +set -o pipefail +set -u + +VERBOSE="${VERBOSE:-0}" +if [[ "${VERBOSE}" -ne "0" ]]; then + set -x +fi + +REGISTRY_BASE_URL="${REGISTRY_BASE_URL:-https://registry.coder.com}" +# Number of artifacts to probe per run. The header is a property of whoever +# answered rather than of the artifact requested, so a modest sample is enough; +# successive runs rotate through the full list so nothing is permanently +# excluded. +SAMPLE_COUNT="${SAMPLE_COUNT:-30}" + +status=0 +# Tracked separately: a missing header and a failed request have different +# causes, and conflating them makes a transient outage look like tampering. +declare -a integrity_failures=() +declare -a request_failures=() +declare -a modules=() + +# Discover modules from the registry's own index so the probe list follows the +# registry rather than being hardcoded here. Entries are "/". +modules_json="$(curl --silent --show-error --fail --max-time 60 \ + "${REGISTRY_BASE_URL}/api/modules")" +if [[ -z "${modules_json}" ]]; then + echo "Error: unable to read the module index from ${REGISTRY_BASE_URL}/api/modules" + exit 1 +fi + +mapfile -t modules < <( + jq -r '.data[] | select(.contributorNamespace and .slug) + | "\(.contributorNamespace)/\(.slug)"' <<< "${modules_json}" \ + | tr -d '\r' | sort +) + +if ((${#modules[@]} == 0)); then + echo "Error: no modules found in the registry index" + exit 1 +fi + +total="${#modules[@]}" +window="${SAMPLE_COUNT}" +((window > total)) && window="${total}" + +# Rotate the window so consecutive runs cover different modules. +offset=$((($(date +%s) / 900 * window) % total)) + +echo "Checking ${window} of ${total} module(s) against ${REGISTRY_BASE_URL}" + +for ((i = 0; i < window; i++)); do + entry="${modules[$(((offset + i) % total))]}" + namespace="${entry%%/*}" + module="${entry##*/}" + + # Ask the registry which versions it publishes and probe the newest, so the + # check follows the registry rather than a stale list. + versions_json="$(curl --silent --show-error --fail --max-time 45 \ + "${REGISTRY_BASE_URL}/terraform_protocol/${namespace}/${module}/coder/versions")" + if [[ -z "${versions_json}" ]]; then + printf '=== Checking %s/%s\n==> COULD NOT LIST VERSIONS\n' "${namespace}" "${module}" + status=1 + request_failures+=("${namespace}/${module} (version listing failed)") + continue + fi + + version="$(jq -r '[.modules[0].versions[].version] + | sort_by(split(".") | map(tonumber? // 0)) + | last // empty' <<< "${versions_json}" | tr -d '\r')" + if [[ -z "${version}" ]]; then + # The module index advertised this module, so the registry should be able + # to list at least one version for it. Treat an empty list as a failure + # rather than skipping, otherwise a run where nothing could be verified + # still reports success. + printf '=== Checking %s/%s\n==> NO VERSIONS PUBLISHED\n' "${namespace}" "${module}" + status=1 + request_failures+=("${namespace}/${module} (no versions published)") + continue + fi + + # A unique query string keeps the CDN from answering, so that we observe the + # origin rather than a cached response. The download handler also requires + # that the client accept gzip. + url="${REGISTRY_BASE_URL}/download/${namespace}/${version}-${module}.tar.gz?integrity=${RANDOM}${RANDOM}" + + printf '=== Checking %s/%s/%s\n' "${namespace}" "${module}" "${version}" + + headers="$(curl --head --silent --show-error --location --max-time 45 \ + --header 'Accept-Encoding: gzip' --retry 2 "${url}")" + curl_status=$? + + if ((curl_status != 0)); then + printf '==> FETCH FAILED (curl exit %s)\n' "${curl_status}" + status=1 + request_failures+=("${namespace}/${module}/${version} (fetch failed)") + continue + fi + + if ! grep -qiE '^HTTP/[0-9.]+ 200' <<< "${headers}"; then + printf '==> UNEXPECTED STATUS\n' + status=1 + request_failures+=("${namespace}/${module}/${version} (unexpected status)") + continue + fi + + if ! grep -qi '^server-version:' <<< "${headers}"; then + printf '==> MISSING Server-Version\n' + status=1 + integrity_failures+=("${namespace}/${module}/${version}") + continue + fi + + printf '==> OK\n' +done + +if ((status == 0)); then + echo "All sampled responses came from the registry application." + exit 0 +fi + +if ((${#request_failures[@]} > 0)); then + echo + echo "Requests that could not be completed:" + for failure in "${request_failures[@]}"; do + echo " - ${failure}" + done + echo "These are usually availability problems. Check whether" + echo "check-registry-site-health is also failing." +fi + +if ((${#integrity_failures[@]} > 0)); then + echo + echo "Responses missing the Server-Version header:" + for failure in "${integrity_failures[@]}"; do + echo " - ${failure}" + done + echo + echo "These responses did not come from the registry application." + echo "Escalate in #registry before taking remediation steps." +fi + +exit "${status}" diff --git a/.github/workflows/check_registry_integrity.yaml b/.github/workflows/check_registry_integrity.yaml new file mode 100644 index 000000000..7b4bff361 --- /dev/null +++ b/.github/workflows/check_registry_integrity.yaml @@ -0,0 +1,33 @@ +# Verify that module tarball responses come from the registry application. +# +# Kept separate from check-registry-site-health because that workflow reports +# availability to the public status page, whereas a failure here warrants +# engineering triage first. +name: check-registry-integrity +on: + schedule: + - cron: "5,20,35,50 * * * *" # Runs every 15 minutes, offset from the health check + workflow_dispatch: # Allows manual triggering of the workflow if needed + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + check-integrity: + runs-on: ubuntu-latest + # Each probe can spend up to ~45s on a timeout, so cap the job well below + # the schedule interval rather than letting a degraded registry queue runs. + timeout-minutes: 10 + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run check_registry_integrity.sh + run: ./.github/scripts/check_registry_integrity.sh