diff --git a/.github/workflow-scripts/upload-test-results.sh b/.github/workflow-scripts/upload-test-results.sh new file mode 100755 index 000000000000..45fa2bf20367 --- /dev/null +++ b/.github/workflow-scripts/upload-test-results.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Copies Test All's test results for one run attempt to GCS, for +# upload-test-results.yml. Artifacts, including those from forks, are treated +# as inert data: only each one's results.json is copied, after validation, and +# nothing in them is executed. +# +# upload-test-results.sh list Stage artifacts.json, and output whether +# this attempt should be recorded +# upload-test-results.sh collect Stage each downloaded results.json +# upload-test-results.sh describe Stage run.json from $WORKFLOW_RUN +# upload-test-results.sh upload Copy $DEST to $PREFIX, run.json last + +set -euo pipefail + +MAX_RESULTS_BYTES=1048576 +STATE_ARTIFACT='^e2e_[a-z]+_[a-z]+_state_[A-Za-z0-9_-]+$' +# The artifacts the workflow's e2e_*_state_* download pattern matches. +DOWNLOAD_PATTERN='^e2e_.*_state_' + +retry() { + for attempt in 1 2 3; do + "$@" && return 0 + echo "Attempt $attempt failed" + sleep $((attempt * 10)) + done + return 1 +} + +list_artifacts() { + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts?per_page=100" --jq '.artifacts[]' > /tmp/artifacts.jsonl +} + +list() { + mkdir -p "$DEST" + retry list_artifacts + # A rerun replaces the per-flow state of the jobs it reruns, so once a later + # attempt has replaced any, this attempt's results are gone and it isn't + # recorded. + if jq -e --slurp --arg pattern "$DOWNLOAD_PATTERN" '[.[] | select(.created_at > env.RUN_UPDATED_AT and (.name | test($pattern)))] | length > 0' /tmp/artifacts.jsonl > /dev/null; then + echo "A later attempt replaced this attempt's test state, so it isn't recorded" + echo "record=false" >> "$GITHUB_OUTPUT" + return + fi + # Every artifact from this attempt, so recordings and logs can be linked. + jq --slurp --arg base "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" '[.[] + | select(.created_at <= env.RUN_UPDATED_AT) + | {id, name, sizeInBytes: .size_in_bytes, createdAt: .created_at, expiresAt: .expires_at, expired, url: "\($base)/\(.id)"}]' \ + /tmp/artifacts.jsonl > "$DEST/artifacts.json" + echo "record=true" >> "$GITHUB_OUTPUT" +} + +# Stages one artifact's results.json, if it's a regular file of valid JSON +# within the size limit. +copy_results() { + local name=$1 file=$2 + if [[ "$name" =~ $STATE_ARTIFACT && -f "$file" && ! -L "$file" ]] && + (( $(wc -c < "$file") <= MAX_RESULTS_BYTES )) && jq empty "$file" 2>/dev/null; then + mkdir -p "$DEST/artifacts/$name" + cp "$file" "$DEST/artifacts/$name/results.json" + else + echo "Skipping ${name:-artifact}: no results.json, too large or not JSON" + fi +} + +collect() { + mkdir -p "$DEST/artifacts" + # download-artifact extracts a single matching artifact without a directory + # of its own. + if [[ -e "$DOWNLOADS/results.json" ]]; then + copy_results "$(jq -r --arg pattern "$DOWNLOAD_PATTERN" '[.[].name | select(test($pattern))] | if length == 1 then .[0] else "" end' "$DEST/artifacts.json")" "$DOWNLOADS/results.json" + else + for dir in "$DOWNLOADS"/*/; do + if [[ -d "$dir" ]]; then + copy_results "$(basename "$dir")" "$dir/results.json" + fi + done + fi + find "$DEST" -type f +} + +describe() { + # pull_requests omits fork PRs and can list unrelated PRs on a push, so only + # PRs into this repository are kept, and only for pull request runs. The head + # repository and branch identify a fork PR. + jq --argjson repoId "$REPO_ID" '{ + repository: .repository.full_name, + runId: .id, + runAttempt: .run_attempt, + event, + conclusion, + headSha: .head_sha, + headBranch: .head_branch, + headRepository: .head_repository.full_name, + pullRequests: (if .event == "pull_request" + then [.pull_requests[] | select(.base.repo.id == $repoId) | .number] + else [] end), + createdAt: .created_at, + runStartedAt: .run_started_at, + updatedAt: .updated_at, + url: .html_url + }' <<< "$WORKFLOW_RUN" > "$DEST/run.json" + cat "$DEST/run.json" +} + +upload() { + if [[ -n "$(ls -A "$DEST/artifacts")" ]]; then + retry gcloud storage cp --recursive --no-clobber "$DEST/artifacts" "$PREFIX/" + fi + retry gcloud storage cp --no-clobber "$DEST/artifacts.json" "$PREFIX/artifacts.json" + # A run is complete once its run.json exists, so it goes last. + retry gcloud storage cp --no-clobber "$DEST/run.json" "$PREFIX/run.json" +} + +case "${1:-}" in + list | collect | describe | upload) "$1" ;; + *) + echo "Usage: $0 list|collect|describe|upload" >&2 + exit 1 + ;; +esac diff --git a/.github/workflows/upload-test-results.yml b/.github/workflows/upload-test-results.yml new file mode 100644 index 000000000000..7d3291f57303 --- /dev/null +++ b/.github/workflows/upload-test-results.yml @@ -0,0 +1,72 @@ +# Copies Test All's test result artifacts, plus a description of the run, to +# GCS for test history. Runs from the default branch after every Test All run, +# including pull requests from forks. The artifacts are copied as inert data: +# nothing in them is executed, and they are validated where they're read. +name: Upload Test Results + +on: + workflow_run: + workflows: [Test All] + types: [completed] + +permissions: {} + +jobs: + upload: + # Nothing runs until the repository has GCP credentials configured. + if: | + vars.VISUAL_GCP_PROJECT != '' && + vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER != '' && + vars.VISUAL_BUCKET != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + id-token: write + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_UPDATED_AT: ${{ github.event.workflow_run.updated_at }} + DOWNLOADS: /tmp/test-artifacts + DEST: /tmp/test-results/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + # Every key under this prefix is written once, so a rerun of this + # workflow skips what an earlier attempt already uploaded. + PREFIX: gs://${{ vars.VISUAL_BUCKET }}/verify/incoming/${{ github.repository }}/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + steps: + # The default branch's own scripts, never the pull request's. + - name: Checkout default branch + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: List artifacts + id: list + run: .github/workflow-scripts/upload-test-results.sh list + - name: Download test results + if: steps.list.outputs.record == 'true' + uses: ./.github/actions/download-artifact + with: + pattern: e2e_*_state_* + path: ${{ env.DOWNLOADS }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Collect test results + if: steps.list.outputs.record == 'true' + run: .github/workflow-scripts/upload-test-results.sh collect + - name: Describe the run + if: steps.list.outputs.record == 'true' + env: + WORKFLOW_RUN: ${{ toJSON(github.event.workflow_run) }} + REPO_ID: ${{ github.event.repository.id }} + run: .github/workflow-scripts/upload-test-results.sh describe + - name: Authenticate to Google Cloud + if: steps.list.outputs.record == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.list.outputs.record == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Upload + if: steps.list.outputs.record == 'true' + run: .github/workflow-scripts/upload-test-results.sh upload