diff --git a/.github/actions/publish-to-s3/action.yml b/.github/actions/publish-to-s3/action.yml new file mode 100644 index 0000000..b823dac --- /dev/null +++ b/.github/actions/publish-to-s3/action.yml @@ -0,0 +1,113 @@ +name: Publish assets to S3 +description: >- + Authenticate to AWS with OIDC, upload built assets to S3, and invalidate the + matching CloudFront paths so the change is visible. No long-lived credentials + are involved. + + The calling job must declare `permissions: id-token: write`, which a composite + action cannot do on its own. + +inputs: + role-to-assume: + description: >- + ARN of the role to assume via OIDC. The role's trust policy must name the + calling repository's subject claim -- it is repo-specific, so a role that + works for one repository will not work for another without an entry. + required: true + aws-region: + description: AWS region. + required: false + default: us-west-2 + bucket: + description: S3 bucket name. + required: true + prefix: + description: Key prefix within the bucket, without leading or trailing slash. + required: true + sources: + description: >- + Directories to upload, one per line. A directory named `dist` uploads to + the prefix root; anything else uploads to a subdirectory of that name. + required: false + default: | + dist + css + distribution-id: + description: CloudFront distribution to invalidate. Omit to skip invalidation. + required: false + default: "" + acl: + description: >- + Object ACL. Defaults to public-read because the media bucket is in legacy + ObjectWriter mode with no bucket policy, so public read comes entirely + from per-object ACLs. Set to "none" once a bucket policy exists. + required: false + default: public-read + public-base-url: + description: >- + Public URL the prefix is served from, with a trailing slash. When set, + the job summary links every published file. S3 website endpoints have no + directory listing, so naming the files is the only way to link them. + required: false + default: "" + dry-run: + description: Show what would be uploaded and invalidated, and change nothing. + required: false + default: "false" + +runs: + using: composite + steps: + - uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: ${{ inputs.aws-region }} + + - name: Publish + shell: bash + env: + BUCKET: ${{ inputs.bucket }} + PREFIX: ${{ inputs.prefix }} + SOURCES: ${{ inputs.sources }} + DISTRIBUTION: ${{ inputs.distribution-id }} + ACL: ${{ inputs.acl }} + DRY_RUN: ${{ inputs.dry-run }} + run: | + ARGS=(--bucket "$BUCKET" --prefix "$PREFIX" --acl "$ACL") + ARGS+=(--manifest "$RUNNER_TEMP/published.txt") + [ -n "$DISTRIBUTION" ] && ARGS+=(--distribution "$DISTRIBUTION") + [ "$DRY_RUN" = "true" ] && ARGS+=(--dry-run) + while IFS= read -r line; do + [ -n "$line" ] && ARGS+=(--source "$line") + done <<< "$SOURCES" + "${{ github.action_path }}/../../../bin/publish-to-s3.sh" "${ARGS[@]}" + + - name: Summarize + shell: bash + env: + BUCKET: ${{ inputs.bucket }} + PREFIX: ${{ inputs.prefix }} + BASE_URL: ${{ inputs.public-base-url }} + DRY_RUN: ${{ inputs.dry-run }} + run: | + MANIFEST="$RUNNER_TEMP/published.txt" + COUNT="$(grep -c . "$MANIFEST" 2>/dev/null || echo 0)" + { + echo "### S3 publish${DRY_RUN:+ (dry run)}" + echo + echo "| | |" + echo "|---|---|" + echo "| bucket | \`$BUCKET\` |" + echo "| prefix | \`$PREFIX\` |" + echo "| files | $COUNT |" + echo "| dry run | $DRY_RUN |" + echo + if [ -n "$BASE_URL" ] && [ "$COUNT" -gt 0 ]; then + echo "#### Published files" + echo + sort "$MANIFEST" | head -n 40 | while IFS= read -r key; do + [ -n "$key" ] && echo "- [\`$key\`](${BASE_URL%/}/$key)" + done + [ "$COUNT" -gt 40 ] && echo "- …and $((COUNT - 40)) more" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 046905b..15860ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,6 +191,32 @@ jobs: || { echo "::error::unhelpful error: $(cat /tmp/err)"; exit 1; } echo "ok" + publish-to-s3-guards: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # No AWS credentials here on purpose: this repository should not hold + # any. What is testable without them is the argument handling and the + # guards, which are where the damage would be. + - name: Required arguments are enforced + run: | + fail() { echo "::error::$1"; exit 1; } + bin/publish-to-s3.sh 2>/dev/null && fail "should require --bucket" + bin/publish-to-s3.sh --bucket b 2>/dev/null && fail "should require --prefix" + bin/publish-to-s3.sh --bucket b --prefix p 2>/dev/null && fail "should require --source" + echo "ok" + + - name: An empty or missing source is refused + run: | + fail() { echo "::error::$1"; exit 1; } + mkdir -p /tmp/empty-dist + bin/publish-to-s3.sh --bucket b --prefix p --source /tmp/empty-dist 2>/dev/null \ + && fail "publishing an empty directory should be an error" + bin/publish-to-s3.sh --bucket b --prefix p --source /tmp/nope 2>/dev/null \ + && fail "a missing directory should be an error" + echo "ok" + # deploy-site is the seam every generator meets, so its guards matter more # than any single builder's. deploy-guards: diff --git a/README.md b/README.md index 020f065..bec21e0 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ decision instead of a rewrite. | `build-sphinx` | Sphinx, using the project's own documentation requirements | | `index-site` | Pagefind search index over built HTML — works with any generator | | `deploy-site` | Checks the build produced a real site, uploads it for Pages | +| `publish-to-s3` | Uploads built assets to S3 over OIDC and invalidates CloudFront | Adding a generator means one new `build-*` action; nothing downstream changes. @@ -190,6 +191,43 @@ rather than at `sphinx-build` with a confusing error. Skip this for generators with their own search. +### `publish-to-s3` + +Authenticates with OIDC, so no long-lived AWS keys exist anywhere. The calling +job must declare `permissions: id-token: write` — a composite action cannot. + +The role's trust policy has to name the **calling repository's** subject claim. +That claim is repo-specific, so a role that works for one repository will not +work for another until an entry is added. + +| Input | Default | | +| --- | --- | --- | +| `role-to-assume` | — | ARN of the role to assume via OIDC | +| `aws-region` | `us-west-2` | | +| `bucket` | — | S3 bucket name | +| `prefix` | — | Key prefix within the bucket | +| `sources` | `dist`, `css` | Directories to upload, one per line. `dist` uploads to the prefix root | +| `distribution-id` | — | CloudFront distribution to invalidate; omit to skip | +| `acl` | `public-read` | Set to `none` once the bucket has a policy instead of per-object ACLs | +| `public-base-url` | — | Public URL the prefix is served from. When set, the job summary links every published file | +| `dry-run` | `false` | Show what would happen and change nothing | + +Run it with `dry-run: true` first. It refuses to publish an empty or missing +source directory, because a build that "succeeds" into nothing would otherwise +leave the CDN serving stale objects with no signal. + +Content types are set explicitly per extension rather than left to the CLI's +guess, which omits the charset and can differ between CLI versions — otherwise +what the CDN serves could change for files nobody edited. + +S3 website endpoints serve no directory listing — a URL ending in `/` returns +404 rather than an index — so the summary names each published file instead of +linking a parent directory. Set `public-base-url` to turn those into links. + +It uses `aws s3 cp`, **not `sync --delete`**. Buckets often hold objects the +publishing repository does not manage, and a delete-enabled sync would remove +them. Check what else lives under the prefix before changing that. + ### `deploy-site` | Input | Default | | diff --git a/bin/publish-to-s3.sh b/bin/publish-to-s3.sh new file mode 100755 index 0000000..23fe648 --- /dev/null +++ b/bin/publish-to-s3.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# +# Upload built assets to an S3 bucket behind CloudFront, then invalidate the +# paths that changed. +# +# Credentials are not handled here. The caller is expected to have configured +# them already -- in CI that means OIDC via aws-actions/configure-aws-credentials, +# so no long-lived keys exist anywhere. Locally it means whatever profile you +# normally use. +# +# publish-to-s3.sh --bucket my-bucket --prefix cl-webcomponents \ +# --source dist --source css --distribution E123 --dry-run +# +# Always run --dry-run first. It prints what would be uploaded and what would +# be invalidated, and touches nothing. + +set -euo pipefail + +BUCKET="" +PREFIX="" +DISTRIBUTION="" +DRY_RUN="false" +ACL="public-read" +MANIFEST="" +declare -a SOURCES=() + +usage() { + cat <<'USAGE' +publish-to-s3.sh -- sync built assets to S3 and invalidate CloudFront + +Options: + --bucket NAME S3 bucket (required) + --prefix PATH key prefix within the bucket (required) + --source DIR directory to upload, repeatable (required) + --distribution ID CloudFront distribution to invalidate (optional) + --acl ACL object ACL, or "none" to omit (default: public-read) + --manifest FILE write the published keys, one per line, for a caller + that wants to report or link them + --dry-run show what would happen, change nothing + -h, --help this text +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --bucket) BUCKET="$2"; shift 2 ;; + --prefix) PREFIX="$2"; shift 2 ;; + --source) SOURCES+=("$2"); shift 2 ;; + --distribution) DISTRIBUTION="$2"; shift 2 ;; + --acl) ACL="$2"; shift 2 ;; + --manifest) MANIFEST="$2"; shift 2 ;; + --dry-run) DRY_RUN="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "publish-to-s3: unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +[ -n "$BUCKET" ] || { echo "publish-to-s3: --bucket is required" >&2; exit 2; } +[ -n "$PREFIX" ] || { echo "publish-to-s3: --prefix is required" >&2; exit 2; } +[ ${#SOURCES[@]} -gt 0 ] || { echo "publish-to-s3: at least one --source is required" >&2; exit 2; } +command -v aws >/dev/null || { echo "publish-to-s3: the AWS CLI is not installed" >&2; exit 1; } + +# Refuse to publish nothing. An empty build that "succeeds" would otherwise +# leave the CDN serving whatever was there before, with no signal. +for dir in "${SOURCES[@]}"; do + [ -d "$dir" ] || { echo "publish-to-s3: no such directory: $dir" >&2; exit 1; } + [ -n "$(find "$dir" -type f -print -quit)" ] || { + echo "publish-to-s3: $dir is empty" >&2; exit 1; } +done + +PREFIX="${PREFIX%/}" +declare -a EXTRA=() +[ "$DRY_RUN" = "true" ] && EXTRA+=("--dryrun") +# The bucket is in legacy ObjectWriter mode with no bucket policy: public read +# comes entirely from per-object ACLs. Dropping this makes uploads invisible. +# Removing the need for it means adding a bucket policy first -- see the +# media bucket notes before changing this. +[ "$ACL" != "none" ] && EXTRA+=("--acl" "$ACL") + +# Content types are set explicitly rather than left to the CLI's guess. The +# guess omits the charset and can differ by CLI version, which would silently +# change what the CDN serves for files nobody edited. +content_type_for() { + case "$1" in + js) echo "text/javascript; charset=utf-8" ;; + css) echo "text/css; charset=utf-8" ;; + json) echo "application/json; charset=utf-8" ;; + jsonld) echo "application/ld+json; charset=utf-8" ;; + svg) echo "image/svg+xml; charset=utf-8" ;; + woff) echo "font/woff" ;; + woff2) echo "font/woff2" ;; + map) echo "application/json; charset=utf-8" ;; + *) echo "" ;; + esac +} + +# NOTE: this uses `cp`, not `sync --delete`. The bucket holds objects this +# repository does not manage -- fonts/ among them -- and a delete-enabled sync +# would remove them. Do not "improve" this to a sync without checking what else +# lives under the prefix. +[ -n "$MANIFEST" ] && : > "$MANIFEST" + +for dir in "${SOURCES[@]}"; do + dest="s3://$BUCKET/$PREFIX/$(basename "$dir")/" + # dist/ holds the bundles themselves, which belong at the prefix root + [ "$(basename "$dir")" = "dist" ] && dest="s3://$BUCKET/$PREFIX/" + echo "==> $dir -> $dest" + + # One pass per extension we have a content type for, then a final pass for + # everything else. The final pass is a no-op when nothing is left over. + declare -a SKIP=() + while IFS= read -r ext; do + [ -n "$ext" ] || continue + ct="$(content_type_for "$ext")" + [ -n "$ct" ] || continue + SKIP+=("--exclude" "*.$ext") + aws s3 cp "$dir/" "$dest" --recursive \ + --exclude "*" --include "*.$ext" --content-type "$ct" \ + ${EXTRA[@]+"${EXTRA[@]}"} + done <<< "$(find "$dir" -type f -name '*.*' | sed 's|.*/||; s|.*\.||' | sort -u)" + + aws s3 cp "$dir/" "$dest" --recursive \ + ${SKIP[@]+"${SKIP[@]}"} ${EXTRA[@]+"${EXTRA[@]}"} + + # Keys published, relative to the prefix. + if [ -n "$MANIFEST" ]; then + sub="" + [ "$(basename "$dir")" = "dist" ] || sub="$(basename "$dir")/" + ( cd "$dir" && find . -type f | sed "s|^\./|$sub|" ) >> "$MANIFEST" + fi +done + +if [ -n "$DISTRIBUTION" ]; then + # Scoped to this project's prefix. Invalidating /* would charge for and + # discard every other site's cached objects in the same distribution. + PATHS="/$PREFIX/*" + if [ "$DRY_RUN" = "true" ]; then + echo "==> would invalidate $PATHS on $DISTRIBUTION" + else + echo "==> invalidating $PATHS on $DISTRIBUTION" + aws cloudfront create-invalidation \ + --distribution-id "$DISTRIBUTION" --paths "$PATHS" \ + --query 'Invalidation.{Id:Id,Status:Status}' --output text + fi +fi + +echo "publish-to-s3: ${DRY_RUN:+dry run }done"