deploy-aws-lambda.yml

# Deploy to AWS Lambda
#
# This workflow deploys the CUDly application to AWS Lambda with Function URL.
# It builds a Docker image, pushes to ECR, and deploys via Terraform.
#
# Required GitHub Secrets:
#   - ADMIN_EMAIL: Admin email for notifications
#   - ALERT_EMAILS: Optional, comma-separated addresses that receive monitoring alerts (#714)
#   - FROM_EMAIL: SES-verified FROM address used for outbound approval emails.
#                 Required for purchase-approval emails on deployments that
#                 don't set subdomain_zone_name (bare Lambda Function URL).
#                 Must be verified in the target AWS account's SES identities.
#
# Required GitHub Variables:
#   - AWS_REGION: AWS region (e.g., us-east-1)
#   - AWS_ROLE_TO_ASSUME: IAM role ARN for OIDC keyless auth
#
# Triggered by:
#   - Pushes to main branch
#   - Manual workflow dispatch with environment selection
#   - Release creation (auto-deploy to prod)
#   - Workflow call from other workflows

name: Deploy to AWS Lambda

# Least privilege by default: `id-token: write` is granted per job, only to the
# jobs that actually assume the AWS deploy role, and both of those are bound to
# a deployment environment. Declaring it here would hand it to `prepare` and
# `summary` too, neither of which authenticates and neither of which is bound
# to an environment.
permissions:
  contents: read

# No workflow-level `concurrency` on purpose. What needs protecting is the
# Terraform state object, which is keyed on the environment, and only the
# job-level key can see `needs.prepare.outputs.target_environment` -- this level
# is limited to the `github`, `inputs` and `vars` contexts. Keying it on
# `github.ref` put two runs on different refs that both resolve to the same
# environment into different groups against one state file (#1806). The group
# lives on `build-and-deploy` below.
on:
  push:
    branches: [main]
    paths:
      - 'cmd/**'
      - 'internal/**'
      # Provider code lives in cloud-commitments-go, so a provider change reaches
      # this image through the go.mod pin, not through a path filter here.
      - 'go.mod'
      - 'go.sum'
      - 'frontend/**'
      - 'Dockerfile'
      - 'terraform/modules/compute/aws/lambda/**'
      - '!terraform/modules/compute/aws/lambda/tests/**'
      - 'terraform/modules/database/aws/**'
      - 'terraform/modules/secrets/aws/**'
      - 'terraform/modules/networking/aws/**'
      - 'terraform/environments/aws/**'
      - '.github/workflows/deploy-aws-lambda.yml'
  workflow_dispatch:
    inputs:
      environment:
        description: 'Deployment environment'
        required: true
        type: choice
        options:
          - dev
          - staging
          - prod
  release:
    types: [created]
  workflow_call:
    inputs:
      environment:
        required: true
        type: string
    secrets:
      ADMIN_EMAIL:
        required: true
      ALERT_EMAILS:
        required: false
      DASHBOARD_URL:
        required: false
      FROM_EMAIL:
        required: false
      TF_BACKEND_AWS:
        required: true

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  # Determine deployment environment
  prepare:
    name: Prepare Deployment
    runs-on: ubuntu-latest
    permissions:
      contents: read
    # NOTE: `target_environment` below is an OUTPUT, not an `environment:`
    # binding — this job is deliberately ungated and therefore must never hold
    # `id-token: write`. It was previously named `environment`, which made the
    # `outputs:` block read like a gate at a glance while gating nothing. Do not
    # rename it back.
    outputs:
      target_environment: ${{ steps.set-env.outputs.target_environment }}
      image_tag: ${{ steps.set-tag.outputs.tag }}

    steps:
      - name: Determine environment
        id: set-env
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUT_ENVIRONMENT: ${{ inputs.environment }}
        run: |
          set -euo pipefail
          INPUT_ENVIRONMENT="${INPUT_ENVIRONMENT:-}"

          # Branch on whether an environment was REQUESTED, not on the event
          # name. Inside a reusable workflow `github.event_name` is the CALLER's
          # event and is never "workflow_call", so no event name can identify a
          # workflow_call invocation (issue #1805). A non-empty input is always
          # an explicit request and wins.
          #
          # An empty input means none was supplied, which this workflow's own
          # `release` and `push` triggers are the legitimate causes of; every
          # other event stops the run. Note that `release` is inherited by a
          # workflow_call from a release-triggered caller, so a caller passing
          # an EMPTY environment would resolve to prod here rather than
          # failing. deploy-all.yml is the only caller and allowlists the value
          # it passes, so that cannot happen. Telling the two apart would need
          # `github.job_workflow_ref`, which is not worth the machinery for an
          # unreachable path.
          if [ -n "$INPUT_ENVIRONMENT" ]; then
            TARGET_ENVIRONMENT="$INPUT_ENVIRONMENT"
          elif [ "$EVENT_NAME" = release ]; then
            TARGET_ENVIRONMENT=prod
          elif [ "$EVENT_NAME" = push ]; then
            TARGET_ENVIRONMENT=dev
          else
            echo "::error::No environment supplied on a '$EVENT_NAME'-triggered run; refusing to guess a deployment target."
            exit 1
          fi

          # workflow_dispatch constrains this to the declared `choice` options,
          # but the workflow_call input is typed as a free-form string, so the
          # allowlist is enforced here rather than assumed.
          case "$TARGET_ENVIRONMENT" in
            dev|staging|prod) ;;
            *)
              echo "::error::Refusing unknown environment: '$TARGET_ENVIRONMENT'"
              exit 1
              ;;
          esac

          echo "target_environment=$TARGET_ENVIRONMENT" >> "$GITHUB_OUTPUT"

      - name: Set image tag
        id: set-tag
        env:
          EVENT_NAME: ${{ github.event_name }}
          RELEASE_TAG: ${{ github.event.release.tag_name }}
          COMMIT_SHA: ${{ github.sha }}
        run: |
          set -euo pipefail

          if [ "$EVENT_NAME" = "release" ]; then
            TAG="${RELEASE_TAG:-}"
          else
            TAG="$COMMIT_SHA"
          fi

          # Shell metacharacters in a tag (';', '$', '`', '|') are already inert:
          # the value arrives via `env:` and is only ever referenced quoted, so
          # it is data, not code. What a charset filter must actually stop is a
          # NEWLINE, because this value is written to $GITHUB_OUTPUT as a single
          # `tag=<value>` line — a newline would let a crafted tag append extra
          # attacker-chosen output keys, and those outputs are consumed by the
          # two credentialed downstream jobs.
          #
          # So reject control characters and reject empty, and allow the rest of
          # the git ref charset. A stricter OCI-grammar filter would reject
          # `release/1.0` and `v1.2.3+build.5` — both legitimate git tags, and
          # this repo already uses slash-namespaced ones — hard-failing a
          # production deploy for no security gain.
          #
          # NOTE: this deliberately does NOT enforce the OCI tag grammar,
          # because the value is cosmetic today: it reaches only
          # deployment-info.json (via `jq --arg`) and the step summary.
          # Terraform derives the real image tag from the git commit in
          # modules/build. If this is ever wired to `custom_image_tag`, add an
          # OCI-grammar check HERE at the same time.
          if [ -z "$TAG" ]; then
            echo "::error::Refusing empty image tag"
            exit 1
          fi
          if [[ "$TAG" =~ [[:cntrl:]] ]]; then
            echo "::error::Refusing image tag containing control characters or newlines"
            exit 1
          fi

          echo "tag=$TAG" >> "$GITHUB_OUTPUT"

  # Deploy infrastructure with Terraform (includes Docker build via build module)
  build-and-deploy:
    name: Build & Deploy
    runs-on: ubuntu-24.04-arm  # ARM runner: Docker build targets linux/arm64 (Lambda/Fargate Graviton2)
    needs: prepare
    permissions:
      # Assumes the AWS deploy role. The `environment:` binding below is what
      # scopes this job's OIDC `sub` to `repo:<org/repo>:environment:<name>`,
      # which is the subject the trust policy matches. NOTE it is not by itself
      # a reviewer gate: an Environment only blocks a job once required-reviewer
      # protection rules are configured on it in repo settings, and as of this
      # change none of this repo's Environments have any. See #1648.
      id-token: write
      contents: read
    # Every run that writes s3://<bucket>/github-<environment>/terraform.tfstate
    # serializes here, whatever ref or workflow it came from (#1806). The suffix
    # is the exact value the backend key below is built from, so the group and
    # the state object cannot drift apart, and `prepare` fails the run on
    # anything outside dev|staging|prod so it can never be empty.
    #
    # `cancel-in-progress: false` is stated rather than left to the default:
    # cancelling mid-`terraform apply` is how you get a half-applied stack and a
    # lock nobody releases.
    concurrency:
      group: aws-tfstate-${{ needs.prepare.outputs.target_environment }}
      cancel-in-progress: false
    # Bind to the named GitHub Environment matching the target so
    # secrets.* resolve to environment-scoped values when defined,
    # falling back to repo-scoped secrets otherwise. Without this,
    # DASHBOARD_URL (and any other per-env secret like FROM_EMAIL,
    # ADMIN_EMAIL) would be repo-wide and dev/staging/prod would
    # all share one value — producing wrong email links in customer
    # mailboxes. Per CR review on PR #368.
    environment: ${{ needs.prepare.outputs.target_environment }}
    # Set at job level, not workflow level: workflow-level `env:` is resolved
    # before this job's `environment:` is in scope, so an environment-scoped
    # AWS_REGION variable would be invisible there. Matches cleanup-staging.yml.
    env:
      AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
    outputs:
      function_url: ${{ steps.outputs.outputs.function_url }}
      function_name: ${{ steps.outputs.outputs.function_name }}
      log_group: ${{ steps.outputs.outputs.log_group }}

    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
        with:
          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Read Terraform version
        # The one pin lives in .terraform-version (issue #349); the script
        # validates it, because an empty terraform_version means `latest`.
        id: terraform-version
        run: bash scripts/terraform-version.sh

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
        with:
          terraform_version: ${{ steps.terraform-version.outputs.version }}

      - name: Terraform Init
        env:
          TF_BACKEND: ${{ secrets.TF_BACKEND_AWS }}
          ENVIRONMENT: ${{ needs.prepare.outputs.target_environment }}
        run: |
          printf '%s\nkey = "github-%s/terraform.tfstate"\n' "$TF_BACKEND" "$ENVIRONMENT" > /tmp/backend.tfbackend
          cd terraform/environments/aws
          terraform init -backend-config=/tmp/backend.tfbackend

      # The Function URL's host is server-assigned, so its origin is read from
      # the previous apply instead of being committed to github-*.tfvars, where
      # it went stale whenever the URL was reissued (#324). Empty on the first
      # deploy; "Check CORS covers the Function URL" below flags that case.
      - name: Read deployed Function URL
        id: deployed-url
        run: |
          set -euo pipefail
          url=$(scripts/read-lambda-function-url.sh terraform/environments/aws)
          echo "url=${url}" >> "$GITHUB_OUTPUT"

      - name: Terraform Plan
        id: plan
        env:
          TF_VAR_admin_email: ${{ secrets.ADMIN_EMAIL }}
          TF_VAR_alert_emails: ${{ secrets.ALERT_EMAILS }}
          TF_VAR_from_email: ${{ secrets.FROM_EMAIL }}
          # DASHBOARD_URL is the customer-facing dashboard origin used in
          # email links (invite, password reset, welcome). For Lambda-
          # Function-URL deployments (no custom domain), set the repo
          # secret DASHBOARD_URL to the function URL — see #363 +
          # github-dev.tfvars for the bootstrap pattern. Empty string is
          # accepted by the var (the Terraform local then falls back to
          # frontend_domain_names[0]).
          TF_VAR_dashboard_url: ${{ secrets.DASHBOARD_URL }}
          TF_VAR_lambda_deployed_function_url: ${{ steps.deployed-url.outputs.url }}
          ENVIRONMENT: ${{ needs.prepare.outputs.target_environment }}
        run: |
          cd terraform/environments/aws
          terraform plan \
            -var-file="github-${ENVIRONMENT}.tfvars" \
            -var="compute_platform=lambda" \
            -out=tfplan

      - name: Terraform Apply
        env:
          TF_VAR_admin_email: ${{ secrets.ADMIN_EMAIL }}
          TF_VAR_alert_emails: ${{ secrets.ALERT_EMAILS }}
          TF_VAR_from_email: ${{ secrets.FROM_EMAIL }}
          # DASHBOARD_URL is the customer-facing dashboard origin used in
          # email links (invite, password reset, welcome). For Lambda-
          # Function-URL deployments (no custom domain), set the repo
          # secret DASHBOARD_URL to the function URL — see #363 +
          # github-dev.tfvars for the bootstrap pattern. Empty string is
          # accepted by the var (the Terraform local then falls back to
          # frontend_domain_names[0]).
          TF_VAR_dashboard_url: ${{ secrets.DASHBOARD_URL }}
        run: |
          cd terraform/environments/aws
          terraform apply -auto-approve tfplan

      # No automatic state-lock release on failure. The step that used to live
      # here ran on `failure() || cancelled()` with no age check and no check
      # that the lock was this run's, so it deleted whatever lock object was
      # there. The `cancelled()` half is the decisive one: GitHub runs those
      # steps while `terraform apply` is still shutting down and may still be
      # writing state, so it destroyed a lock out from under an active writer.
      # A run that failed *because* it could not acquire the lock would also
      # have deleted the lock held by the run still applying. Removed with
      # #1806, matching the shape deploy-aws-fargate.yml already uses.
      # Terraform releases its own lock on a clean apply error; a lock that
      # survives a run means the run died abnormally, which needs operator
      # confirmation that the owning run is dead. Recovery is
      # `terraform force-unlock <ID>` -- see runbooks/terraform-stuck-lock.md.

      - name: Get Terraform outputs
        id: outputs
        run: |
          cd terraform/environments/aws
          {
            echo "function_url=$(terraform output -raw lambda_function_url 2>/dev/null | grep -v '::' || echo "")"
            echo "function_name=$(terraform output -raw lambda_function_name 2>/dev/null | grep -v '::' || echo "")"
            echo "log_group=$(terraform output -raw lambda_log_group_name 2>/dev/null | grep -v '::' || echo "")"
          } >> "$GITHUB_OUTPUT"

      # Second, idempotent migration runner (#31): the lazy ensureDB path stays
      # on and the golang-migrate advisory lock serializes the two. Same job as
      # the apply, so it shares the aws-tfstate-<env> serialization and the
      # job's credentials. `--cli-read-timeout 0` keeps the CLI from retrying an
      # invoke that is still running. `aws lambda invoke` exits 0 on a function
      # error, so the script reads FunctionError and the payload status. Prod is
      # excluded until the owner confirms it may migrate inside the deploy job
      # (plan decision D1); it migrates on demand through the same invoke.
      - name: Run database migrations
        if: needs.prepare.outputs.target_environment != 'prod'
        timeout-minutes: 20
        env:
          FUNCTION_NAME: ${{ steps.outputs.outputs.function_name }}
        run: |
          set -euo pipefail
          if [ -z "$FUNCTION_NAME" ]; then
            echo "::error::Terraform did not output a Lambda function name; cannot run migrations."
            exit 1
          fi
          out_dir="$(mktemp -d)"
          aws lambda invoke \
            --function-name "$FUNCTION_NAME" \
            --cli-read-timeout 0 \
            --cli-connect-timeout 60 \
            --cli-binary-format raw-in-base64-out \
            --payload '{"action":"migrate"}' \
            "$out_dir/payload.json" >"$out_dir/invoke.json"
          summary="$(scripts/check-lambda-migrate-result.sh "$out_dir/invoke.json" "$out_dir/payload.json")"
          echo "$summary"
          echo "$summary" >> "$GITHUB_STEP_SUMMARY"

      # The allowlist this apply wrote holds the origin read before the plan.
      # If the apply created or reissued the Function URL, the live origin is
      # not in it until the next deploy, which reads the new URL back.
      - name: Check CORS covers the Function URL
        env:
          PLANNED_URL: ${{ steps.deployed-url.outputs.url }}
        run: |
          set -euo pipefail
          live=$(scripts/read-lambda-function-url.sh terraform/environments/aws)
          if [ -n "$live" ] && [ "$live" != "$PLANNED_URL" ]; then
            echo "::warning::The Function URL is now ${live}, but this apply's CORS allowlist was built from '${PLANNED_URL}'. Re-run this workflow so the next apply adds the new origin."
          fi

      - name: Save deployment info
        env:
          ENVIRONMENT: ${{ needs.prepare.outputs.target_environment }}
          IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
          FUNCTION_URL: ${{ steps.outputs.outputs.function_url }}
          FUNCTION_NAME: ${{ steps.outputs.outputs.function_name }}
          DEPLOYED_BY: ${{ github.actor }}
          COMMIT: ${{ github.sha }}
        run: |
          set -euo pipefail

          # Built with jq rather than an unquoted `cat <<EOF`, which would both
          # break the JSON on a quote and evaluate $(...) in any interpolated
          # value as shell. `image_tag` is charset-validated upstream in
          # `prepare`, so this is defence in depth rather than the only guard —
          # but this job holds `id-token: write`, so the shape is not one to
          # leave lying around.
          jq -n \
            --arg environment "$ENVIRONMENT" \
            --arg image_tag "$IMAGE_TAG" \
            --arg function_url "$FUNCTION_URL" \
            --arg function_name "$FUNCTION_NAME" \
            --arg deployed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --arg deployed_by "$DEPLOYED_BY" \
            --arg commit "$COMMIT" \
            '$ARGS.named' > deployment-info.json

          cat deployment-info.json

      - name: Upload deployment info
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: deployment-info-lambda-${{ needs.prepare.outputs.target_environment }}
          path: deployment-info.json
          retention-days: 90

  # Test the deployment
  test-deployment:
    name: Test Deployment
    runs-on: ubuntu-latest
    needs: [prepare, build-and-deploy]
    if: always() && needs.build-and-deploy.result == 'success'
    permissions:
      # Assumes the AWS deploy role. The `environment:` binding below is what
      # scopes this job's OIDC `sub` to `repo:<org/repo>:environment:<name>`,
      # which is the subject the trust policy matches. NOTE it is not by itself
      # a reviewer gate: an Environment only blocks a job once required-reviewer
      # protection rules are configured on it in repo settings, and as of this
      # change none of this repo's Environments have any. See #1648.
      id-token: write
      contents: read
    # Bind to the same named environment as build-and-deploy so that
    # vars.AWS_ROLE_TO_ASSUME resolves to the per-environment scoped value.
    environment: ${{ needs.prepare.outputs.target_environment }}
    # Set at job level, not workflow level: workflow-level `env:` is resolved
    # before this job's `environment:` is in scope, so an environment-scoped
    # AWS_REGION variable would be invisible there. Matches cleanup-staging.yml.
    env:
      AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}

    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd  # v6.1.1 — pinned per SC review
        with:
          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Get Function URL
        id: get-url
        env:
          FUNCTION_NAME: ${{ needs.build-and-deploy.outputs.function_name }}
        run: |
          if [ -z "$FUNCTION_NAME" ]; then
            echo "Failed to get function name from deploy outputs"
            exit 1
          fi
          FUNCTION_URL=$(aws lambda get-function-url-config \
            --function-name "$FUNCTION_NAME" \
            --query 'FunctionUrl' \
            --output text)
          if [ -z "$FUNCTION_URL" ] || [ "$FUNCTION_URL" = "None" ]; then
            echo "Failed to get function URL from AWS Lambda API for $FUNCTION_NAME"
            exit 1
          fi
          echo "url=$FUNCTION_URL" >> "$GITHUB_OUTPUT"

      - name: Wait for Lambda to be ready
        run: |
          echo "Waiting 30 seconds for Lambda to be fully ready..."
          sleep 30

      - name: Test health endpoint
        env:
          FUNCTION_URL_RAW: ${{ steps.get-url.outputs.url }}
        run: |
          URL="${FUNCTION_URL_RAW%/}"
          echo "Testing health endpoint: $URL/health"

          for i in {1..5}; do
            # /health returns HTTP 200 even when degraded; the body is the verdict.
            if RESPONSE=$(curl -fsS --max-time 10 "$URL/health"); then
              echo "Response: $RESPONSE"
              if jq -e '.status == "healthy"' > /dev/null <<< "$RESPONSE"; then
                echo "Health check passed!"
                exit 0
              fi
            fi
            echo "Attempt $i failed, retrying in 10 seconds..."
            sleep 10
          done

          echo "Health check failed after 5 attempts"
          exit 1

      - name: Run smoke tests
        env:
          FUNCTION_URL_RAW: ${{ steps.get-url.outputs.url }}
        run: |
          URL="${FUNCTION_URL_RAW%/}"

          echo "Running basic smoke tests..."

          for i in {1..3}; do
            if RESPONSE=$(curl -fsS --max-time 10 "$URL/health") &&
              jq -e '.status == "healthy"' > /dev/null <<< "$RESPONSE"; then
              echo "Health check $i: passed"
            else
              echo "Health check $i: failed: ${RESPONSE:-no response}"
              exit 1
            fi
            sleep 2
          done

          echo "All smoke tests passed!"

  # Post deployment summary
  summary:
    name: Deployment Summary
    runs-on: ubuntu-latest
    needs: [prepare, build-and-deploy, test-deployment]
    if: always()
    permissions:
      contents: read

    steps:
      - name: Download deployment info
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: deployment-info-lambda-${{ needs.prepare.outputs.target_environment }}
        continue-on-error: true

      # Every value arrives via `env:`, including the ones whose safety is
      # currently guaranteed by a check in a DIFFERENT job. Relying on "an
      # upstream job validated this" makes the rule "raw interpolation is fine
      # when someone else checked" — an implicit invariant that breaks silently
      # the moment the upstream guard moves. One uniform rule instead.
      - name: Post summary
        env:
          TARGET_ENVIRONMENT: ${{ needs.prepare.outputs.target_environment }}
          IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
          DEPLOY_RESULT: ${{ needs.build-and-deploy.result }}
          TEST_RESULT: ${{ needs.test-deployment.result }}
        run: |
          set -euo pipefail

          {
            echo "## AWS Lambda Deployment Summary"
            echo ""
            echo "**Environment:** $TARGET_ENVIRONMENT"
            echo "**Image Tag:** $IMAGE_TAG"
            echo "**Status:** $DEPLOY_RESULT"
            echo ""

            if [ -f deployment-info.json ]; then
              echo "### Deployment Details"
              echo '```json'
              cat deployment-info.json
              echo '```'
            fi

            echo ""
            echo "### Job Results"
            echo "- Deploy: $DEPLOY_RESULT"
            echo "- Test: $TEST_RESULT"
            echo ""

            if [ "$DEPLOY_RESULT" = "success" ] && [ "$TEST_RESULT" = "success" ]; then
              echo "**Deployment successful!**"
            else
              echo "**Deployment failed. Check logs for details.**"
            fi
          } >> "$GITHUB_STEP_SUMMARY"