From 7b75010806a843ad4d02f696643f74fc7a645cbb Mon Sep 17 00:00:00 2001 From: Rob Hogan <2590098+robhogan@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:18:22 +0000 Subject: [PATCH 1/2] Upload Test All results to GCS for test history After every Test All run, including PRs from forks, a trusted `workflow_run` workflow on the default branch copies each E2E job's per-flow `results.json`, an `artifacts.json` listing every artifact in the run with its GitHub link (so recordings and logs can be linked), and a `run.json` describing the run to `verify/incoming//-/` in the visual testing bucket. It reuses the visual testing GCP variables and skips when they are unset. Artifacts are copied as data: only `results.json` is extracted from each archive, size-limited, and nothing is executed. An attempt whose test state a later rerun has replaced isn't recorded, so a late upload of an earlier attempt can't store incomplete or later results. GitHub API calls are retried. Objects are write-once, with `run.json` uploaded last to mark a run complete. Changelog: [Internal] --- .github/workflows/upload-test-results.yml | 135 ++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 .github/workflows/upload-test-results.yml diff --git a/.github/workflows/upload-test-results.yml b/.github/workflows/upload-test-results.yml new file mode 100644 index 000000000000..6534b3e8f9ca --- /dev/null +++ b/.github/workflows/upload-test-results.yml @@ -0,0 +1,135 @@ +# Copies Test All's test result artifacts, plus a description of the run, to +# GCS for test history. Runs from the default branch after every Test All run, +# including pull requests from forks. The artifacts are copied as inert data: +# nothing in them is executed, and they are validated where they're read. +name: Upload Test Results + +on: + workflow_run: + workflows: [Test All] + types: [completed] + +permissions: {} + +jobs: + upload: + # Nothing runs until the repository has GCP credentials configured. + if: | + vars.VISUAL_GCP_PROJECT != '' && + vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER != '' && + vars.VISUAL_BUCKET != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: read + id-token: write + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_UPDATED_AT: ${{ github.event.workflow_run.updated_at }} + DEST: /tmp/test-results/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + # Every key under this prefix is written once, so a rerun of this + # workflow skips what an earlier attempt already uploaded. + PREFIX: gs://${{ vars.VISUAL_BUCKET }}/verify/incoming/${{ github.repository }}/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + steps: + - name: Download test results + run: | + set -euo pipefail + retry() { + for attempt in 1 2 3; do + "$@" && return 0 + echo "Attempt $attempt failed" + sleep $((attempt * 10)) + done + return 1 + } + fetch() { + gh api "$@" > "$OUT" + } + mkdir -p "$DEST/artifacts" + OUT=/tmp/artifacts.jsonl retry fetch --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts?per_page=100" --jq '.artifacts[]' + # A rerun replaces the per-flow state of the jobs it reruns, so once a + # later attempt has replaced any, this attempt's results are gone and + # it isn't recorded. + if jq -e --slurp '[.[] | select(.created_at > env.RUN_UPDATED_AT and (.name | test("^e2e_[a-z]+_[a-z]+_state_")))] | length > 0' /tmp/artifacts.jsonl > /dev/null; then + echo "A later attempt replaced this attempt's test state, so it isn't recorded" + touch "$DEST/replaced" + exit 0 + fi + # Every artifact from this attempt, so recordings and logs can be + # linked. + jq --slurp --arg base "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" '[.[] + | select(.created_at <= env.RUN_UPDATED_AT) + | {id, name, sizeInBytes: .size_in_bytes, createdAt: .created_at, expiresAt: .expires_at, expired, url: "\($base)/\(.id)"}]' \ + /tmp/artifacts.jsonl > "$DEST/artifacts.json" + # Only small per-flow E2E state artifacts are downloaded. + jq -r '.[] + | select((.expired | not) and .sizeInBytes <= 1048576) + | select(.name | test("^e2e_[a-z]+_[a-z]+_state_[A-Za-z0-9_-]+$")) + | "\(.id) \(.name)"' "$DEST/artifacts.json" | + while read -r ID NAME; do + OUT="/tmp/artifact-$ID.zip" retry fetch "repos/$GITHUB_REPOSITORY/actions/artifacts/$ID/zip" < /dev/null + # Extract only results.json, so paths inside the archive are never used. + if unzip -p "/tmp/artifact-$ID.zip" results.json 2>/dev/null | head -c 1048577 > "/tmp/results-$ID.json" && + [[ -s "/tmp/results-$ID.json" ]] && (( $(wc -c < "/tmp/results-$ID.json") <= 1048576 )) && + jq empty "/tmp/results-$ID.json" 2>/dev/null; then + mkdir -p "$DEST/artifacts/$NAME" + mv "/tmp/results-$ID.json" "$DEST/artifacts/$NAME/results.json" + else + echo "Skipping $NAME: no results.json, too large or not JSON" + fi + done + find "$DEST" -type f + - name: Describe the run + env: + WORKFLOW_RUN: ${{ toJSON(github.event.workflow_run) }} + REPO_ID: ${{ github.event.repository.id }} + run: | + # pull_requests omits fork PRs and can list unrelated PRs on a push, + # so only PRs into this repository are kept, and only for pull + # request runs. The head repository and branch identify a fork PR. + jq --argjson repoId "$REPO_ID" '{ + repository: .repository.full_name, + runId: .id, + runAttempt: .run_attempt, + event, + conclusion, + headSha: .head_sha, + headBranch: .head_branch, + headRepository: .head_repository.full_name, + pullRequests: (if .event == "pull_request" + then [.pull_requests[] | select(.base.repo.id == $repoId) | .number] + else [] end), + createdAt: .created_at, + runStartedAt: .run_started_at, + updatedAt: .updated_at, + url: .html_url + }' <<< "$WORKFLOW_RUN" > "$DEST/run.json" + cat "$DEST/run.json" + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Upload + run: | + set -euo pipefail + if [[ -e "$DEST/replaced" ]]; then + exit 0 + fi + retry() { + for attempt in 1 2 3; do + "$@" && return 0 + echo "Attempt $attempt failed" + sleep $((attempt * 10)) + done + return 1 + } + if [[ -n "$(ls -A "$DEST/artifacts")" ]]; then + retry gcloud storage cp --recursive --no-clobber "$DEST/artifacts" "$PREFIX/" + fi + retry gcloud storage cp --no-clobber "$DEST/artifacts.json" "$PREFIX/artifacts.json" + # A run is complete once its run.json exists, so it goes last. + retry gcloud storage cp --no-clobber "$DEST/run.json" "$PREFIX/run.json" From 06844c826889ef4cfd8479c9b31acceb136a57ff Mon Sep 17 00:00:00 2001 From: Rob Hogan <2590098+robhogan@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:39:05 +0000 Subject: [PATCH 2/2] Move test result upload steps into a script and download with download-artifact --- .../workflow-scripts/upload-test-results.sh | 125 ++++++++++++++++++ .github/workflows/upload-test-results.yml | 113 ++++------------ 2 files changed, 150 insertions(+), 88 deletions(-) create mode 100755 .github/workflow-scripts/upload-test-results.sh diff --git a/.github/workflow-scripts/upload-test-results.sh b/.github/workflow-scripts/upload-test-results.sh new file mode 100755 index 000000000000..45fa2bf20367 --- /dev/null +++ b/.github/workflow-scripts/upload-test-results.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Copies Test All's test results for one run attempt to GCS, for +# upload-test-results.yml. Artifacts, including those from forks, are treated +# as inert data: only each one's results.json is copied, after validation, and +# nothing in them is executed. +# +# upload-test-results.sh list Stage artifacts.json, and output whether +# this attempt should be recorded +# upload-test-results.sh collect Stage each downloaded results.json +# upload-test-results.sh describe Stage run.json from $WORKFLOW_RUN +# upload-test-results.sh upload Copy $DEST to $PREFIX, run.json last + +set -euo pipefail + +MAX_RESULTS_BYTES=1048576 +STATE_ARTIFACT='^e2e_[a-z]+_[a-z]+_state_[A-Za-z0-9_-]+$' +# The artifacts the workflow's e2e_*_state_* download pattern matches. +DOWNLOAD_PATTERN='^e2e_.*_state_' + +retry() { + for attempt in 1 2 3; do + "$@" && return 0 + echo "Attempt $attempt failed" + sleep $((attempt * 10)) + done + return 1 +} + +list_artifacts() { + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts?per_page=100" --jq '.artifacts[]' > /tmp/artifacts.jsonl +} + +list() { + mkdir -p "$DEST" + retry list_artifacts + # A rerun replaces the per-flow state of the jobs it reruns, so once a later + # attempt has replaced any, this attempt's results are gone and it isn't + # recorded. + if jq -e --slurp --arg pattern "$DOWNLOAD_PATTERN" '[.[] | select(.created_at > env.RUN_UPDATED_AT and (.name | test($pattern)))] | length > 0' /tmp/artifacts.jsonl > /dev/null; then + echo "A later attempt replaced this attempt's test state, so it isn't recorded" + echo "record=false" >> "$GITHUB_OUTPUT" + return + fi + # Every artifact from this attempt, so recordings and logs can be linked. + jq --slurp --arg base "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" '[.[] + | select(.created_at <= env.RUN_UPDATED_AT) + | {id, name, sizeInBytes: .size_in_bytes, createdAt: .created_at, expiresAt: .expires_at, expired, url: "\($base)/\(.id)"}]' \ + /tmp/artifacts.jsonl > "$DEST/artifacts.json" + echo "record=true" >> "$GITHUB_OUTPUT" +} + +# Stages one artifact's results.json, if it's a regular file of valid JSON +# within the size limit. +copy_results() { + local name=$1 file=$2 + if [[ "$name" =~ $STATE_ARTIFACT && -f "$file" && ! -L "$file" ]] && + (( $(wc -c < "$file") <= MAX_RESULTS_BYTES )) && jq empty "$file" 2>/dev/null; then + mkdir -p "$DEST/artifacts/$name" + cp "$file" "$DEST/artifacts/$name/results.json" + else + echo "Skipping ${name:-artifact}: no results.json, too large or not JSON" + fi +} + +collect() { + mkdir -p "$DEST/artifacts" + # download-artifact extracts a single matching artifact without a directory + # of its own. + if [[ -e "$DOWNLOADS/results.json" ]]; then + copy_results "$(jq -r --arg pattern "$DOWNLOAD_PATTERN" '[.[].name | select(test($pattern))] | if length == 1 then .[0] else "" end' "$DEST/artifacts.json")" "$DOWNLOADS/results.json" + else + for dir in "$DOWNLOADS"/*/; do + if [[ -d "$dir" ]]; then + copy_results "$(basename "$dir")" "$dir/results.json" + fi + done + fi + find "$DEST" -type f +} + +describe() { + # pull_requests omits fork PRs and can list unrelated PRs on a push, so only + # PRs into this repository are kept, and only for pull request runs. The head + # repository and branch identify a fork PR. + jq --argjson repoId "$REPO_ID" '{ + repository: .repository.full_name, + runId: .id, + runAttempt: .run_attempt, + event, + conclusion, + headSha: .head_sha, + headBranch: .head_branch, + headRepository: .head_repository.full_name, + pullRequests: (if .event == "pull_request" + then [.pull_requests[] | select(.base.repo.id == $repoId) | .number] + else [] end), + createdAt: .created_at, + runStartedAt: .run_started_at, + updatedAt: .updated_at, + url: .html_url + }' <<< "$WORKFLOW_RUN" > "$DEST/run.json" + cat "$DEST/run.json" +} + +upload() { + if [[ -n "$(ls -A "$DEST/artifacts")" ]]; then + retry gcloud storage cp --recursive --no-clobber "$DEST/artifacts" "$PREFIX/" + fi + retry gcloud storage cp --no-clobber "$DEST/artifacts.json" "$PREFIX/artifacts.json" + # A run is complete once its run.json exists, so it goes last. + retry gcloud storage cp --no-clobber "$DEST/run.json" "$PREFIX/run.json" +} + +case "${1:-}" in + list | collect | describe | upload) "$1" ;; + *) + echo "Usage: $0 list|collect|describe|upload" >&2 + exit 1 + ;; +esac diff --git a/.github/workflows/upload-test-results.yml b/.github/workflows/upload-test-results.yml index 6534b3e8f9ca..7d3291f57303 100644 --- a/.github/workflows/upload-test-results.yml +++ b/.github/workflows/upload-test-results.yml @@ -27,109 +27,46 @@ jobs: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ github.event.workflow_run.id }} RUN_UPDATED_AT: ${{ github.event.workflow_run.updated_at }} + DOWNLOADS: /tmp/test-artifacts DEST: /tmp/test-results/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} # Every key under this prefix is written once, so a rerun of this # workflow skips what an earlier attempt already uploaded. PREFIX: gs://${{ vars.VISUAL_BUCKET }}/verify/incoming/${{ github.repository }}/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} steps: + # The default branch's own scripts, never the pull request's. + - name: Checkout default branch + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: List artifacts + id: list + run: .github/workflow-scripts/upload-test-results.sh list - name: Download test results - run: | - set -euo pipefail - retry() { - for attempt in 1 2 3; do - "$@" && return 0 - echo "Attempt $attempt failed" - sleep $((attempt * 10)) - done - return 1 - } - fetch() { - gh api "$@" > "$OUT" - } - mkdir -p "$DEST/artifacts" - OUT=/tmp/artifacts.jsonl retry fetch --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts?per_page=100" --jq '.artifacts[]' - # A rerun replaces the per-flow state of the jobs it reruns, so once a - # later attempt has replaced any, this attempt's results are gone and - # it isn't recorded. - if jq -e --slurp '[.[] | select(.created_at > env.RUN_UPDATED_AT and (.name | test("^e2e_[a-z]+_[a-z]+_state_")))] | length > 0' /tmp/artifacts.jsonl > /dev/null; then - echo "A later attempt replaced this attempt's test state, so it isn't recorded" - touch "$DEST/replaced" - exit 0 - fi - # Every artifact from this attempt, so recordings and logs can be - # linked. - jq --slurp --arg base "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" '[.[] - | select(.created_at <= env.RUN_UPDATED_AT) - | {id, name, sizeInBytes: .size_in_bytes, createdAt: .created_at, expiresAt: .expires_at, expired, url: "\($base)/\(.id)"}]' \ - /tmp/artifacts.jsonl > "$DEST/artifacts.json" - # Only small per-flow E2E state artifacts are downloaded. - jq -r '.[] - | select((.expired | not) and .sizeInBytes <= 1048576) - | select(.name | test("^e2e_[a-z]+_[a-z]+_state_[A-Za-z0-9_-]+$")) - | "\(.id) \(.name)"' "$DEST/artifacts.json" | - while read -r ID NAME; do - OUT="/tmp/artifact-$ID.zip" retry fetch "repos/$GITHUB_REPOSITORY/actions/artifacts/$ID/zip" < /dev/null - # Extract only results.json, so paths inside the archive are never used. - if unzip -p "/tmp/artifact-$ID.zip" results.json 2>/dev/null | head -c 1048577 > "/tmp/results-$ID.json" && - [[ -s "/tmp/results-$ID.json" ]] && (( $(wc -c < "/tmp/results-$ID.json") <= 1048576 )) && - jq empty "/tmp/results-$ID.json" 2>/dev/null; then - mkdir -p "$DEST/artifacts/$NAME" - mv "/tmp/results-$ID.json" "$DEST/artifacts/$NAME/results.json" - else - echo "Skipping $NAME: no results.json, too large or not JSON" - fi - done - find "$DEST" -type f + if: steps.list.outputs.record == 'true' + uses: ./.github/actions/download-artifact + with: + pattern: e2e_*_state_* + path: ${{ env.DOWNLOADS }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Collect test results + if: steps.list.outputs.record == 'true' + run: .github/workflow-scripts/upload-test-results.sh collect - name: Describe the run + if: steps.list.outputs.record == 'true' env: WORKFLOW_RUN: ${{ toJSON(github.event.workflow_run) }} REPO_ID: ${{ github.event.repository.id }} - run: | - # pull_requests omits fork PRs and can list unrelated PRs on a push, - # so only PRs into this repository are kept, and only for pull - # request runs. The head repository and branch identify a fork PR. - jq --argjson repoId "$REPO_ID" '{ - repository: .repository.full_name, - runId: .id, - runAttempt: .run_attempt, - event, - conclusion, - headSha: .head_sha, - headBranch: .head_branch, - headRepository: .head_repository.full_name, - pullRequests: (if .event == "pull_request" - then [.pull_requests[] | select(.base.repo.id == $repoId) | .number] - else [] end), - createdAt: .created_at, - runStartedAt: .run_started_at, - updatedAt: .updated_at, - url: .html_url - }' <<< "$WORKFLOW_RUN" > "$DEST/run.json" - cat "$DEST/run.json" + run: .github/workflow-scripts/upload-test-results.sh describe - name: Authenticate to Google Cloud + if: steps.list.outputs.record == 'true' uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 with: project_id: ${{ vars.VISUAL_GCP_PROJECT }} workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} - name: Setup gcloud + if: steps.list.outputs.record == 'true' uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - name: Upload - run: | - set -euo pipefail - if [[ -e "$DEST/replaced" ]]; then - exit 0 - fi - retry() { - for attempt in 1 2 3; do - "$@" && return 0 - echo "Attempt $attempt failed" - sleep $((attempt * 10)) - done - return 1 - } - if [[ -n "$(ls -A "$DEST/artifacts")" ]]; then - retry gcloud storage cp --recursive --no-clobber "$DEST/artifacts" "$PREFIX/" - fi - retry gcloud storage cp --no-clobber "$DEST/artifacts.json" "$PREFIX/artifacts.json" - # A run is complete once its run.json exists, so it goes last. - retry gcloud storage cp --no-clobber "$DEST/run.json" "$PREFIX/run.json" + if: steps.list.outputs.record == 'true' + run: .github/workflow-scripts/upload-test-results.sh upload