From e155ca6a676288ab8d5db649b7dabafafa4670fb Mon Sep 17 00:00:00 2001 From: Tommy Keswick Date: Tue, 1 Sep 2026 17:01:20 -0700 Subject: [PATCH] Add an S3 publish action Uploads built assets to an S3 bucket and invalidates the matching CloudFront paths, authenticating with OIDC so no long-lived AWS keys exist anywhere. Replaces the per-repository publish_to_s3.bash and invalidate_cdn.bash scripts, which needed an uncommitted media.env to exist on someone's machine. The invalidation stays in this action rather than becoming a separate one. It is cleanup after an upload, not a step anyone would run on its own. Two guards, both covering ways the old scripts could do damage quietly: - It refuses to publish an empty or missing source directory. A build that "succeeds" into nothing would otherwise leave the CDN serving stale objects with no signal that anything went wrong. - The invalidation is scoped to the project's own prefix. The previous script invalidated /*, which charged for and discarded every other project's cached objects in the same distribution. Content types are set explicitly per extension rather than left to the AWS CLI's guess, which omits the charset and has changed between CLI versions. publish_to_s3.bash set text/javascript and text/css with charset=utf-8, and the live objects carry those values; replacing it should not quietly change them. It uses `cp`, not `sync --delete`. The CL-web-components prefix holds a fonts/ directory that exists in no repository -- three woff files that page.tmpl and the footer component load. A delete-enabled sync would remove them and nothing would put them back. --acl public-read is kept and the reason recorded: the media bucket is in legacy ObjectWriter mode with no bucket policy, so public read comes entirely from per-object ACLs. Removing the need for it means adding a bucket policy first, verifying the CDN still serves, then setting BucketOwnerEnforced -- in that order, or every project on the distribution goes down. The calling job must declare `permissions: id-token: write`; a composite action cannot. The role's trust policy must name the calling repository's subject claim, which is repo-specific. Both are documented, because both cost time to diagnose from the error message alone. CI tests the argument handling and the guards but holds no AWS credentials. This repository should not have any. Co-Authored-By: Claude Opus 5 --- .github/actions/publish-to-s3/action.yml | 113 +++++++++++++++++ .github/workflows/ci.yml | 26 ++++ README.md | 38 ++++++ bin/publish-to-s3.sh | 147 +++++++++++++++++++++++ 4 files changed, 324 insertions(+) create mode 100644 .github/actions/publish-to-s3/action.yml create mode 100755 bin/publish-to-s3.sh diff --git a/.github/actions/publish-to-s3/action.yml b/.github/actions/publish-to-s3/action.yml new file mode 100644 index 0000000..b823dac --- /dev/null +++ b/.github/actions/publish-to-s3/action.yml @@ -0,0 +1,113 @@ +name: Publish assets to S3 +description: >- + Authenticate to AWS with OIDC, upload built assets to S3, and invalidate the + matching CloudFront paths so the change is visible. No long-lived credentials + are involved. + + The calling job must declare `permissions: id-token: write`, which a composite + action cannot do on its own. + +inputs: + role-to-assume: + description: >- + ARN of the role to assume via OIDC. The role's trust policy must name the + calling repository's subject claim -- it is repo-specific, so a role that + works for one repository will not work for another without an entry. + required: true + aws-region: + description: AWS region. + required: false + default: us-west-2 + bucket: + description: S3 bucket name. + required: true + prefix: + description: Key prefix within the bucket, without leading or trailing slash. + required: true + sources: + description: >- + Directories to upload, one per line. A directory named `dist` uploads to + the prefix root; anything else uploads to a subdirectory of that name. + required: false + default: | + dist + css + distribution-id: + description: CloudFront distribution to invalidate. Omit to skip invalidation. + required: false + default: "" + acl: + description: >- + Object ACL. Defaults to public-read because the media bucket is in legacy + ObjectWriter mode with no bucket policy, so public read comes entirely + from per-object ACLs. Set to "none" once a bucket policy exists. + required: false + default: public-read + public-base-url: + description: >- + Public URL the prefix is served from, with a trailing slash. When set, + the job summary links every published file. S3 website endpoints have no + directory listing, so naming the files is the only way to link them. + required: false + default: "" + dry-run: + description: Show what would be uploaded and invalidated, and change nothing. + required: false + default: "false" + +runs: + using: composite + steps: + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: ${{ inputs.aws-region }} + + - name: Publish + shell: bash + env: + BUCKET: ${{ inputs.bucket }} + PREFIX: ${{ inputs.prefix }} + SOURCES: ${{ inputs.sources }} + DISTRIBUTION: ${{ inputs.distribution-id }} + ACL: ${{ inputs.acl }} + DRY_RUN: ${{ inputs.dry-run }} + run: | + ARGS=(--bucket "$BUCKET" --prefix "$PREFIX" --acl "$ACL") + ARGS+=(--manifest "$RUNNER_TEMP/published.txt") + [ -n "$DISTRIBUTION" ] && ARGS+=(--distribution "$DISTRIBUTION") + [ "$DRY_RUN" = "true" ] && ARGS+=(--dry-run) + while IFS= read -r line; do + [ -n "$line" ] && ARGS+=(--source "$line") + done <<< "$SOURCES" + "${{ github.action_path }}/../../../bin/publish-to-s3.sh" "${ARGS[@]}" + + - name: Summarize + shell: bash + env: + BUCKET: ${{ inputs.bucket }} + PREFIX: ${{ inputs.prefix }} + BASE_URL: ${{ inputs.public-base-url }} + DRY_RUN: ${{ inputs.dry-run }} + run: | + MANIFEST="$RUNNER_TEMP/published.txt" + COUNT="$(grep -c . "$MANIFEST" 2>/dev/null || echo 0)" + { + echo "### S3 publish${DRY_RUN:+ (dry run)}" + echo + echo "| | |" + echo "|---|---|" + echo "| bucket | \`$BUCKET\` |" + echo "| prefix | \`$PREFIX\` |" + echo "| files | $COUNT |" + echo "| dry run | $DRY_RUN |" + echo + if [ -n "$BASE_URL" ] && [ "$COUNT" -gt 0 ]; then + echo "#### Published files" + echo + sort "$MANIFEST" | head -n 40 | while IFS= read -r key; do + [ -n "$key" ] && echo "- [\`$key\`](${BASE_URL%/}/$key)" + done + [ "$COUNT" -gt 40 ] && echo "- …and $((COUNT - 40)) more" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 046905b..15860ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,6 +191,32 @@ jobs: || { echo "::error::unhelpful error: $(cat /tmp/err)"; exit 1; } echo "ok" + publish-to-s3-guards: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # No AWS credentials here on purpose: this repository should not hold + # any. What is testable without them is the argument handling and the + # guards, which are where the damage would be. + - name: Required arguments are enforced + run: | + fail() { echo "::error::$1"; exit 1; } + bin/publish-to-s3.sh 2>/dev/null && fail "should require --bucket" + bin/publish-to-s3.sh --bucket b 2>/dev/null && fail "should require --prefix" + bin/publish-to-s3.sh --bucket b --prefix p 2>/dev/null && fail "should require --source" + echo "ok" + + - name: An empty or missing source is refused + run: | + fail() { echo "::error::$1"; exit 1; } + mkdir -p /tmp/empty-dist + bin/publish-to-s3.sh --bucket b --prefix p --source /tmp/empty-dist 2>/dev/null \ + && fail "publishing an empty directory should be an error" + bin/publish-to-s3.sh --bucket b --prefix p --source /tmp/nope 2>/dev/null \ + && fail "a missing directory should be an error" + echo "ok" + # deploy-site is the seam every generator meets, so its guards matter more # than any single builder's. deploy-guards: diff --git a/README.md b/README.md index 020f065..bec21e0 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ decision instead of a rewrite. | `build-sphinx` | Sphinx, using the project's own documentation requirements | | `index-site` | Pagefind search index over built HTML — works with any generator | | `deploy-site` | Checks the build produced a real site, uploads it for Pages | +| `publish-to-s3` | Uploads built assets to S3 over OIDC and invalidates CloudFront | Adding a generator means one new `build-*` action; nothing downstream changes. @@ -190,6 +191,43 @@ rather than at `sphinx-build` with a confusing error. Skip this for generators with their own search. +### `publish-to-s3` + +Authenticates with OIDC, so no long-lived AWS keys exist anywhere. The calling +job must declare `permissions: id-token: write` — a composite action cannot. + +The role's trust policy has to name the **calling repository's** subject claim. +That claim is repo-specific, so a role that works for one repository will not +work for another until an entry is added. + +| Input | Default | | +| --- | --- | --- | +| `role-to-assume` | — | ARN of the role to assume via OIDC | +| `aws-region` | `us-west-2` | | +| `bucket` | — | S3 bucket name | +| `prefix` | — | Key prefix within the bucket | +| `sources` | `dist`, `css` | Directories to upload, one per line. `dist` uploads to the prefix root | +| `distribution-id` | — | CloudFront distribution to invalidate; omit to skip | +| `acl` | `public-read` | Set to `none` once the bucket has a policy instead of per-object ACLs | +| `public-base-url` | — | Public URL the prefix is served from. When set, the job summary links every published file | +| `dry-run` | `false` | Show what would happen and change nothing | + +Run it with `dry-run: true` first. It refuses to publish an empty or missing +source directory, because a build that "succeeds" into nothing would otherwise +leave the CDN serving stale objects with no signal. + +Content types are set explicitly per extension rather than left to the CLI's +guess, which omits the charset and can differ between CLI versions — otherwise +what the CDN serves could change for files nobody edited. + +S3 website endpoints serve no directory listing — a URL ending in `/` returns +404 rather than an index — so the summary names each published file instead of +linking a parent directory. Set `public-base-url` to turn those into links. + +It uses `aws s3 cp`, **not `sync --delete`**. Buckets often hold objects the +publishing repository does not manage, and a delete-enabled sync would remove +them. Check what else lives under the prefix before changing that. + ### `deploy-site` | Input | Default | | diff --git a/bin/publish-to-s3.sh b/bin/publish-to-s3.sh new file mode 100755 index 0000000..23fe648 --- /dev/null +++ b/bin/publish-to-s3.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# +# Upload built assets to an S3 bucket behind CloudFront, then invalidate the +# paths that changed. +# +# Credentials are not handled here. The caller is expected to have configured +# them already -- in CI that means OIDC via aws-actions/configure-aws-credentials, +# so no long-lived keys exist anywhere. Locally it means whatever profile you +# normally use. +# +# publish-to-s3.sh --bucket my-bucket --prefix cl-webcomponents \ +# --source dist --source css --distribution E123 --dry-run +# +# Always run --dry-run first. It prints what would be uploaded and what would +# be invalidated, and touches nothing. + +set -euo pipefail + +BUCKET="" +PREFIX="" +DISTRIBUTION="" +DRY_RUN="false" +ACL="public-read" +MANIFEST="" +declare -a SOURCES=() + +usage() { + cat <<'USAGE' +publish-to-s3.sh -- sync built assets to S3 and invalidate CloudFront + +Options: + --bucket NAME S3 bucket (required) + --prefix PATH key prefix within the bucket (required) + --source DIR directory to upload, repeatable (required) + --distribution ID CloudFront distribution to invalidate (optional) + --acl ACL object ACL, or "none" to omit (default: public-read) + --manifest FILE write the published keys, one per line, for a caller + that wants to report or link them + --dry-run show what would happen, change nothing + -h, --help this text +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --bucket) BUCKET="$2"; shift 2 ;; + --prefix) PREFIX="$2"; shift 2 ;; + --source) SOURCES+=("$2"); shift 2 ;; + --distribution) DISTRIBUTION="$2"; shift 2 ;; + --acl) ACL="$2"; shift 2 ;; + --manifest) MANIFEST="$2"; shift 2 ;; + --dry-run) DRY_RUN="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "publish-to-s3: unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +[ -n "$BUCKET" ] || { echo "publish-to-s3: --bucket is required" >&2; exit 2; } +[ -n "$PREFIX" ] || { echo "publish-to-s3: --prefix is required" >&2; exit 2; } +[ ${#SOURCES[@]} -gt 0 ] || { echo "publish-to-s3: at least one --source is required" >&2; exit 2; } +command -v aws >/dev/null || { echo "publish-to-s3: the AWS CLI is not installed" >&2; exit 1; } + +# Refuse to publish nothing. An empty build that "succeeds" would otherwise +# leave the CDN serving whatever was there before, with no signal. +for dir in "${SOURCES[@]}"; do + [ -d "$dir" ] || { echo "publish-to-s3: no such directory: $dir" >&2; exit 1; } + [ -n "$(find "$dir" -type f -print -quit)" ] || { + echo "publish-to-s3: $dir is empty" >&2; exit 1; } +done + +PREFIX="${PREFIX%/}" +declare -a EXTRA=() +[ "$DRY_RUN" = "true" ] && EXTRA+=("--dryrun") +# The bucket is in legacy ObjectWriter mode with no bucket policy: public read +# comes entirely from per-object ACLs. Dropping this makes uploads invisible. +# Removing the need for it means adding a bucket policy first -- see the +# media bucket notes before changing this. +[ "$ACL" != "none" ] && EXTRA+=("--acl" "$ACL") + +# Content types are set explicitly rather than left to the CLI's guess. The +# guess omits the charset and can differ by CLI version, which would silently +# change what the CDN serves for files nobody edited. +content_type_for() { + case "$1" in + js) echo "text/javascript; charset=utf-8" ;; + css) echo "text/css; charset=utf-8" ;; + json) echo "application/json; charset=utf-8" ;; + jsonld) echo "application/ld+json; charset=utf-8" ;; + svg) echo "image/svg+xml; charset=utf-8" ;; + woff) echo "font/woff" ;; + woff2) echo "font/woff2" ;; + map) echo "application/json; charset=utf-8" ;; + *) echo "" ;; + esac +} + +# NOTE: this uses `cp`, not `sync --delete`. The bucket holds objects this +# repository does not manage -- fonts/ among them -- and a delete-enabled sync +# would remove them. Do not "improve" this to a sync without checking what else +# lives under the prefix. +[ -n "$MANIFEST" ] && : > "$MANIFEST" + +for dir in "${SOURCES[@]}"; do + dest="s3://$BUCKET/$PREFIX/$(basename "$dir")/" + # dist/ holds the bundles themselves, which belong at the prefix root + [ "$(basename "$dir")" = "dist" ] && dest="s3://$BUCKET/$PREFIX/" + echo "==> $dir -> $dest" + + # One pass per extension we have a content type for, then a final pass for + # everything else. The final pass is a no-op when nothing is left over. + declare -a SKIP=() + while IFS= read -r ext; do + [ -n "$ext" ] || continue + ct="$(content_type_for "$ext")" + [ -n "$ct" ] || continue + SKIP+=("--exclude" "*.$ext") + aws s3 cp "$dir/" "$dest" --recursive \ + --exclude "*" --include "*.$ext" --content-type "$ct" \ + ${EXTRA[@]+"${EXTRA[@]}"} + done <<< "$(find "$dir" -type f -name '*.*' | sed 's|.*/||; s|.*\.||' | sort -u)" + + aws s3 cp "$dir/" "$dest" --recursive \ + ${SKIP[@]+"${SKIP[@]}"} ${EXTRA[@]+"${EXTRA[@]}"} + + # Keys published, relative to the prefix. + if [ -n "$MANIFEST" ]; then + sub="" + [ "$(basename "$dir")" = "dist" ] || sub="$(basename "$dir")/" + ( cd "$dir" && find . -type f | sed "s|^\./|$sub|" ) >> "$MANIFEST" + fi +done + +if [ -n "$DISTRIBUTION" ]; then + # Scoped to this project's prefix. Invalidating /* would charge for and + # discard every other site's cached objects in the same distribution. + PATHS="/$PREFIX/*" + if [ "$DRY_RUN" = "true" ]; then + echo "==> would invalidate $PATHS on $DISTRIBUTION" + else + echo "==> invalidating $PATHS on $DISTRIBUTION" + aws cloudfront create-invalidation \ + --distribution-id "$DISTRIBUTION" --paths "$PATHS" \ + --query 'Invalidation.{Id:Id,Status:Status}' --output text + fi +fi + +echo "publish-to-s3: ${DRY_RUN:+dry run }done"