verify-e2e.yml

name: Expo E2E

on:
  pull_request:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: verify-e2e-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  e2e-tests:
    name: Device tests (${{ matrix.platform }})
    if: ${{ github.event_name == 'workflow_dispatch' || (github.head_ref == 'changeset-release/main' && github.event.pull_request.head.repo.full_name == github.repository) }}
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 75
    permissions:
      contents: read
      actions: read
      pull-requests: write
    strategy:
      fail-fast: false
      matrix:
        include:
          - platform: ios
            runner: ${{ vars.VERIFY_CI_RUNNER_IOS || 'blacksmith-6vcpu-macos-26' }}
          - platform: android
            runner: ${{ vars.VERIFY_CI_RUNNER_ANDROID || 'blacksmith-8vcpu-ubuntu-2204' }}
    env:
      PACKAGE: integration/expo-native
      CLI: integration/expo-native/bin/control-clerk-expo
      PLATFORM: ${{ matrix.platform }}
      VERIFY_LOCAL_BUILD: standalone
      VERIFY_NATIVE_CACHE: integration/expo-native/.verify/native
      VERIFY_THROWAWAY_HOURS: '2'
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4.4.0
        with:
          persist-credentials: false

      - uses: pnpm/action-setup@v4.3.0

      - uses: actions/setup-node@v4.4.0
        with:
          node-version: 24.15.0
          cache: pnpm

      - name: Set up JDK 21
        if: matrix.platform == 'android'
        uses: actions/setup-java@v4.9.1
        with:
          distribution: temurin
          java-version: 21

      - name: Enable KVM
        if: matrix.platform == 'android'
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      - name: Prepare the device image
        id: device-image
        background: true
        timeout-minutes: 15
        env:
          IOS_RUNTIME: com.apple.CoreSimulator.SimRuntime.iOS-26-5
        run: |
          set -euo pipefail
          if [ "$PLATFORM" = "ios" ]; then
            if ! xcrun simctl list runtimes available -j | jq -e --arg id "$IOS_RUNTIME" '.runtimes[] | select(.identifier == $id)' > /dev/null; then
              echo "::error::This runner image has no $IOS_RUNTIME. It has: $(xcrun simctl list runtimes available -j | jq -r '[.runtimes[].identifier] | join(", ")')"
              exit 1
            fi
            udid="$(xcrun simctl create "Clerk Verify Template iOS" "iPhone 17 Pro" "$IOS_RUNTIME")"
            xcrun simctl boot "$udid"
            xcrun simctl bootstatus "$udid" -b
            CLERK_TEST_DEVICES="$udid" integration/expo-native/bin/configure-ios-simulators.sh
            xcrun simctl shutdown "$udid"
          else
            IMAGE="system-images;android-36;google_apis;x86_64"
            for attempt in 1 2 3; do
              if "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install "$IMAGE" emulator platform-tools < /dev/null > "$RUNNER_TEMP/sdkmanager.log" 2>&1; then
                break
              fi
              echo "::warning::sdkmanager could not install $IMAGE on attempt $attempt"
              grep -v '^\[' "$RUNNER_TEMP/sdkmanager.log" | tail -20 || true
              if [ "$attempt" = "3" ]; then
                exit 1
              fi
            done
            echo "$ANDROID_HOME/platform-tools" >> "$GITHUB_PATH"
          fi

      - name: Install monorepo dependencies
        run: pnpm install --frozen-lockfile

      - name: Install the test package
        id: install
        run: npm ci --prefix "$PACKAGE" --no-audit --no-fund

      - name: Name the native build that this commit needs
        id: native
        run: node "$PACKAGE/src/fixture.ts" native "$PLATFORM" >> "$GITHUB_OUTPUT"

      - name: Fetch a native build that an earlier run made
        id: kept
        timeout-minutes: 5
        env:
          GH_TOKEN: ${{ github.token }}
          BRANCH: ${{ github.head_ref || github.ref_name }}
          NATIVE_ID: ${{ steps.native.outputs.id }}
          NATIVE_DIR: ${{ steps.native.outputs.dir }}
          NATIVE_APP: ${{ steps.native.outputs.app }}
        run: integration/expo-native/bin/fetch-native-build.sh

      - name: Wait for the device image
        wait: device-image

      - name: Install simslim
        if: matrix.platform == 'ios'
        run: |
          set -euo pipefail
          archive="$RUNNER_TEMP/simslim.tar.gz"
          curl --fail --silent --show-error --location --retry 3 --output "$archive" \
            https://github.com/MobAI-App/simslim/releases/download/v0.12.2/simslim-v0.12.2-macos-arm64.tar.gz
          echo "b47302da6fb83b8b4c21f4fdd61507469770c07b0ff07fd1f48207bda4270887  $archive" | shasum -a 256 -c -
          mkdir -p "$RUNNER_TEMP/bin"
          tar -xzf "$archive" -C "$RUNNER_TEMP/bin" simslim
          echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"

      - name: Build the fixture and boot a device
        timeout-minutes: 30
        env:
          CLERK_PLATFORM_API_KEY: ${{ secrets.MOBILE_VERIFICATION_PLATFORM_API_KEY }}
          VERIFY_SIMSLIM: ${{ matrix.platform == 'ios' && '1' || '' }}
          SIMSLIM_BOOT_TIMEOUT: 15m
          SIMSLIM_SPAWN_TIMEOUT: 5m
        run: |
          "$CLI" up --platform "$PLATFORM" --backend local

      - name: Keep the native build for later runs
        if: ${{ !cancelled() && steps.kept.outputs.found == 'none' }}
        continue-on-error: true
        uses: actions/upload-artifact@v4.6.2
        with:
          name: verify-expo-native-${{ steps.native.outputs.id }}
          path: ${{ steps.native.outputs.dir }}/${{ steps.native.outputs.app }}
          include-hidden-files: true
          retention-days: 7
          if-no-files-found: warn

      - name: Cache the XCTest runner
        if: matrix.platform == 'ios'
        uses: actions/cache@v4.3.0
        with:
          path: ~/.agent-device/apple-runner
          key: verify-agent-device-${{ runner.os }}-${{ hashFiles('integration/expo-native/package-lock.json') }}

      - name: Compile the XCTest runner before the first spec launches
        if: matrix.platform == 'ios'
        timeout-minutes: 20
        run: |
          set -euo pipefail
          udid="$(node -p 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).deviceId' "$PACKAGE/.verify/leases/ios.json")"
          AGENT_DEVICE_STATE_DIR="$GITHUB_WORKSPACE/$PACKAGE/.verify/agent-device" \
            "$PACKAGE/node_modules/.bin/agent-device" prepare ios-runner --platform ios --udid "$udid" --timeout 900000 \
            || echo "::warning::prepare ios-runner failed; the run starts its own agent-device daemon, and the first test waits for the XCTest runner"

      - name: Launch the app once before the first spec
        if: matrix.platform == 'ios'
        timeout-minutes: 5
        run: |
          "$CLI" screen --platform ios > /dev/null || echo "::warning::the first launch failed here; the first spec pays for the cold start"

      - name: Run the end-to-end tests
        timeout-minutes: 30
        env:
          CLERK_PLATFORM_API_KEY: ${{ secrets.MOBILE_VERIFICATION_PLATFORM_API_KEY }}
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          "$CLI" run --all --platform "$PLATFORM" --backend local --retries 1 --max-failures 5 --github-report

      - name: Release the device and delete the Clerk application
        if: always() && steps.install.outcome == 'success'
        env:
          CLERK_PLATFORM_API_KEY: ${{ secrets.MOBILE_VERIFICATION_PLATFORM_API_KEY }}
        run: |
          "$CLI" down

      - name: Check that every run is sealed
        id: evidence
        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
        run: |
          node -e '
            const fs = require("fs");
            const path = require("path");
            const runs = path.join(process.env.PACKAGE, ".verify", "runs");
            const ids = fs.existsSync(runs) ? fs.readdirSync(runs).filter((id) => !id.startsWith(".")) : [];
            let upload = ids.length > 0;
            for (const id of ids) {
              let record = null;
              try { record = JSON.parse(fs.readFileSync(path.join(runs, id, "run.json"), "utf8")); } catch {}
              if (record === null || record.sealed !== true || record.tainted.length > 0) {
                upload = false;
                console.log("::warning::Run " + id + " was not sealed, or a file in it holds a secret value. No evidence is uploaded.");
              }
            }
            fs.appendFileSync(process.env.GITHUB_OUTPUT, "upload=" + upload + "\n");
          '

      - name: Upload the evidence
        if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
        uses: actions/upload-artifact@v4.6.2
        with:
          name: verify-e2e-${{ matrix.platform }}
          overwrite: true
          include-hidden-files: true
          path: |
            ${{ env.PACKAGE }}/.verify/runs/*/run.json
            ${{ env.PACKAGE }}/.verify/runs/*/app.log
            ${{ env.PACKAGE }}/.verify/runs/*/driver
            ${{ env.PACKAGE }}/.verify/runs/*/video.mp4
            ${{ env.PACKAGE }}/.verify/runs/*/screenshots
            ${{ env.PACKAGE }}/.verify/runs/*/e2e*/report.json
            ${{ env.PACKAGE }}/.verify/runs/*/e2e*/junit.xml
            ${{ env.PACKAGE }}/.verify/runs/*/e2e*/summary.md
            ${{ env.PACKAGE }}/.verify/runs/*/e2e*/results
          retention-days: 3
          if-no-files-found: warn

      - name: Record the flaky tests
        if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
        continue-on-error: true
        run: node scripts/verify-flaky.mjs record "$PACKAGE/.verify/runs" "$RUNNER_TEMP/verify-flaky.json"

      - name: Keep the flaky tests for the Slack post
        if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
        continue-on-error: true
        uses: actions/upload-artifact@v4.6.2
        with:
          name: verify-flaky-${{ matrix.platform }}
          path: ${{ runner.temp }}/verify-flaky.json
          retention-days: 1
          overwrite: true
          if-no-files-found: ignore

  report-flaky:
    name: Post the flaky tests to Slack
    needs: e2e-tests
    if: ${{ !cancelled() && needs.e2e-tests.result != 'skipped' && github.head_ref == 'changeset-release/main' }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      contents: read
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4.4.0
        with:
          persist-credentials: false
          sparse-checkout: scripts/verify-flaky.mjs
          sparse-checkout-cone-mode: false

      - uses: actions/setup-node@v4.4.0
        with:
          node-version: 24.15.0

      - name: Download the flaky tests of both platforms
        continue-on-error: true
        uses: actions/download-artifact@v4.3.0
        with:
          pattern: verify-flaky-*
          path: ${{ runner.temp }}/flaky

      - name: Generate notification payload
        id: notification
        run: |
          mkdir -p "$RUNNER_TEMP/flaky"
          payload="$(node scripts/verify-flaky.mjs payload "$RUNNER_TEMP/flaky")"
          if [ -z "$payload" ]; then
            echo "No flaky tests in this run."
            exit 0
          fi
          echo "$payload"
          {
            echo 'payload<<__NOTIFY_EOF__'
            echo "$payload"
            echo '__NOTIFY_EOF__'
          } >> "$GITHUB_OUTPUT"

      - name: Send the flaky tests to Slack
        if: ${{ steps.notification.outputs.payload != '' }}
        uses: slackapi/slack-github-action@v1.27.1
        with:
          payload: ${{ steps.notification.outputs.payload }}
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

  verify-e2e:
    name: Expo E2E
    needs: e2e-tests
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Pass when the end-to-end tests passed or did not need to run
        env:
          RESULT: ${{ needs.e2e-tests.result }}
        run: |
          echo "The end-to-end tests ended with $RESULT."
          [ "$RESULT" = "success" ] || [ "$RESULT" = "skipped" ]