From 5757c4bf6967fe8fea92ffc0cfcc83df9e61f3bd Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 17:07:22 +1000 Subject: [PATCH 01/10] [#2863] Added opt-in for visual regression on non-PR deployments. Added the 'VR_DIFFY_NON_PR_BRANCHES' repository variable so deployments without a pull request environment - release branches, integration branches, permanent environments - can run a Diffy comparison when the deployed branch matches. Empty by default, preserving the current skip. Extracted the glob matching into a 'ref_matches' helper shared by the PR head-branch bypass and the new branch opt-in, reported the result to the workflow run summary for runs that have no pull request to comment on, and skipped instead of failing when a resolved PR number cannot be read. --- .github/workflows/test-vr.yml | 92 +++++++++++++------ .../content/development/visual-regression.mdx | 65 +++++++++++-- 2 files changed, 122 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test-vr.yml b/.github/workflows/test-vr.yml index 0c3ca0884..bdf38a29c 100644 --- a/.github/workflows/test-vr.yml +++ b/.github/workflows/test-vr.yml @@ -1,9 +1,9 @@ # GitHub Actions visual regression testing workflow. # # Runs a Diffy visual regression comparison after a deployment lands. -# Triggered automatically by 'notify-diffy' (via repository_dispatch) -# when the deployed PR has the configured label, or manually via -# 'workflow_dispatch' against any URL. +# Triggered automatically by 'notify-diffy' (via repository_dispatch) when +# the deployed PR has the configured label or the deployed branch is opted +# in, or manually via 'workflow_dispatch' against any URL. name: Test - Visual regression on: @@ -44,9 +44,11 @@ env: DIFFY_MAX_WAIT: ${{ vars.VR_DIFFY_MAX_WAIT || '2700' }} DIFFY_PR_LABEL: ${{ vars.VR_DIFFY_PR_LABEL || 'VR' }} DIFFY_AUTO_BRANCHES: ${{ vars.VR_DIFFY_AUTO_BRANCHES || 'deps/*' }} + DIFFY_NON_PR_BRANCHES: ${{ vars.VR_DIFFY_NON_PR_BRANCHES }} DIFFY_POLL_INTERVAL: ${{ vars.VR_DIFFY_POLL_INTERVAL || '30' }} SOURCE_ENV: ${{ github.event.client_payload.source_env || inputs.source_env || 'production' }} TARGET_URL: ${{ github.event.client_payload.target_url || inputs.target_url }} + BRANCH: ${{ github.event.client_payload.branch }} LABEL: ${{ github.event.client_payload.label || inputs.label || 'manual' }} jobs: @@ -74,18 +76,18 @@ jobs: # Hosting providers expose PR environments via URLs that contain # a 'pr-' segment (e.g. 'app.pr-123.example.lagoon.cloud'). - # Extract the number from the URL; if no match, this is not a PR - # deployment and visual regression should not run. + # Extract the number from the URL; a URL without one is not a PR + # deployment and is gated on the deployed branch instead. pr_number="$(printf '%s' "${TARGET_URL}" | sed -n 's|.*pr-\([0-9]\{1,\}\).*|\1|p')" if [ -n "${pr_number}" ]; then echo "PR #${pr_number} resolved from target URL ${TARGET_URL}." else - echo "::notice::No PR pattern found in target URL ${TARGET_URL}. Visual regression will not run." + echo "No PR pattern found in target URL ${TARGET_URL}." fi echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT" - - name: Gate run on PR label (or skip for non-PR contexts on dispatch) + - name: Gate run on PR label or opted-in branch id: gate env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -95,6 +97,39 @@ jobs: run: | set -euo pipefail + # Matches a git ref against a comma-separated list of glob + # patterns. Returns 0 on the first match; 1 when the ref or the + # list is empty, or when nothing matches. + ref_matches() { + local ref="${1}" + local list="${2}" + local pattern + local patterns + + if [ -z "${ref}" ] || [ -z "${list}" ]; then + return 1 + fi + + IFS=',' read -ra patterns <<<"${list}" + + for pattern in "${patterns[@]}"; do + # Trim surrounding whitespace. + pattern="${pattern# }" + pattern="${pattern% }" + + if [ -z "${pattern}" ]; then + continue + fi + + # shellcheck disable=SC2254 + case "${ref}" in + ${pattern}) return 0 ;; + esac + done + + return 1 + } + # Manual workflow_dispatch always proceeds - the operator # explicitly asked for a comparison. if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then @@ -103,36 +138,39 @@ jobs: exit 0 fi - # repository_dispatch with no PR resolved - nothing to gate on. + # No PR resolved - the deployment targets a permanent environment + # such as a release branch, 'develop', 'dev' or 'stage'. Those run + # only when the deployed branch is explicitly opted in. if [ -z "${PR_NUMBER}" ]; then - echo "::notice::No PR associated with this deployment, skipping visual regression." + if ref_matches "${BRANCH}" "${DIFFY_NON_PR_BRANCHES}"; then + echo "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES, proceeding." + echo "skipped=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "::notice::No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." + echo "skipped=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # PR resolved - fetch labels and head branch in one call. A failed + # lookup means the 'pr-' segment in the URL did not come + # from a pull request in this repository. + if ! pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"; then + echo "::notice::PR #${PR_NUMBER} could not be read, skipping visual regression." echo "skipped=true" >> "$GITHUB_OUTPUT" exit 0 fi - # PR resolved - fetch labels and head branch in one call. - pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)" head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')" labels_lower="$(printf '%s' "${pr_data}" | jq -r '.labels[].name' | tr '[:upper:]' '[:lower:]')" # Auto-run bypass: PRs from configured branches (e.g. Renovate's # `deps/*`) skip the label check. - if [ -n "${DIFFY_AUTO_BRANCHES}" ]; then - IFS=',' read -ra patterns <<<"${DIFFY_AUTO_BRANCHES}" - for pattern in "${patterns[@]}"; do - # Trim surrounding whitespace. - pattern="${pattern# }" - pattern="${pattern% }" - [ -z "${pattern}" ] && continue - # shellcheck disable=SC2254 - case "${head_ref}" in - ${pattern}) - echo "PR #${PR_NUMBER} head branch '${head_ref}' matches auto-run pattern '${pattern}', proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 - ;; - esac - done + if ref_matches "${head_ref}" "${DIFFY_AUTO_BRANCHES}"; then + echo "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES, proceeding." + echo "skipped=false" >> "$GITHUB_OUTPUT" + exit 0 fi # Label check (case-insensitive). diff --git a/.vortex/docs/content/development/visual-regression.mdx b/.vortex/docs/content/development/visual-regression.mdx index 800c73cdd..11bfda669 100644 --- a/.vortex/docs/content/development/visual-regression.mdx +++ b/.vortex/docs/content/development/visual-regression.mdx @@ -7,7 +7,8 @@ sidebar_label: Visual regression **Vortex** ships an optional visual regression workflow powered by [Diffy](../tools/diffy.mdx). It compares the just-deployed environment against a baseline (typically `production`) and posts the result back to -the related pull request. +the related pull request, or to the workflow run summary when the +deployment has no pull request. ## Account setup @@ -75,6 +76,7 @@ the site. In Cloudflare, add an IP Access Rule for that address with the | `VR_DIFFY_POLL_INTERVAL` | `30` | Polling interval in seconds | | `VR_DIFFY_PR_LABEL` | `VR` | PR label that opts a deployment into visual regression. Case-insensitive | | `VR_DIFFY_AUTO_BRANCHES` | `deps/*` | Comma-separated glob list of PR head branches that bypass the label gate (matches Renovate's `branchPrefix`). Set to empty to require the label on every PR. | +| `VR_DIFFY_NON_PR_BRANCHES` | (empty) | Comma-separated glob list of deployed branches that run visual regression when the deployment is *not* a PR environment (for example `release/*`). Empty means non-PR deployments never run. | | `VR_DIFFY_SOURCE_ENV` | `production` | Default Diffy source environment for comparisons | Add these under *Settings > Secrets and variables > Actions > Repository @@ -158,8 +160,9 @@ commit SHA**. The workflow itself extracts the PR number from the deployed environment URL by matching the `pr-` segment (e.g. `https://pr-123.example.com/` resolves to PR #123) and verifies the `VR` label is present (case-insensitive). If the target URL has no -`pr-` segment, the deployment is not a PR environment and the -run is skipped. +`pr-` segment, the deployment is not a PR environment and the run +is skipped unless the deployed branch is opted in - see [Release and other +non-PR deployments](#release-and-other-non-pr-deployments). This means the PR lookup works uniformly across hosting providers - the host only needs to expose the deployed environment URL, which all of @@ -251,6 +254,45 @@ Consumers using Dependabot just append: `VR_DIFFY_AUTO_BRANCHES=deps/*,dependabot/*`. Consumers who want the label as the only gate (no auto-bypass) set the variable to empty. +## Release and other non-PR deployments + +A deployment that is not a pull request environment - a release branch, an +integration branch such as `develop`, or a permanent `dev`/`stage` +environment - has no pull request to carry the `VR` label, so it is skipped +by default. + +Set `VR_DIFFY_NON_PR_BRANCHES` to a comma-separated glob list of deployed +branch names to compare them automatically: + +```ini +VR_DIFFY_NON_PR_BRANCHES=release/* +``` + +| Deployment | `VR_DIFFY_NON_PR_BRANCHES` | Runs? | +|---|---|---| +| PR environment (`pr-123` in the URL) | any value | gated by the `VR` label and `VR_DIFFY_AUTO_BRANCHES` as usual | +| Branch `release/1.2.3` | (empty) | no | +| Branch `release/1.2.3` | `release/*` | **yes** | +| Branch `develop` | `release/*` | no | +| Branch `develop` | `release/*,develop` | **yes** | +| Any branch | `*` | **yes** (every non-PR deployment) | + +The deployed branch travels in the dispatch payload, so no GitHub API call +is needed to resolve it. + +:::note + +Patterns match the **git branch name**, not the hostname. Hosting providers +sanitize branch names for URLs - Lagoon deploys `release/26.7.5` to a host +containing `release-26-7-5` - but the gate compares against the original +`release/26.7.5`. + +::: + +There is no pull request to comment on for these runs, so the result +appears in the workflow run summary and in the Diffy UI instead of as a PR +comment. + ## How the PR is resolved The workflow extracts the PR number from the deployed environment URL by @@ -272,7 +314,8 @@ This means: If the target URL has no `pr-` segment (for example, a deploy to a named environment like `dev`/`test`/`prod`), the workflow treats it as -"not a PR deployment" and exits without running. +"not a PR deployment" and runs only when the deployed branch matches +`VR_DIFFY_NON_PR_BRANCHES`. ## Missed-window behavior @@ -288,8 +331,9 @@ entry point. `vr-compare`: -1. Resolves the PR number from the target URL and gates on the label - (or auto-branch pattern for dependency PRs). +1. Resolves the PR number from the target URL and gates on the label (or + the auto-branch pattern for dependency PRs, or the deployed-branch + opt-in for non-PR deployments). 2. Installs the pinned Diffy CLI. 3. Calls `diffy project:compare` with the target URL and a label. 4. Polls the comparison status, printing progress to the job log every @@ -323,10 +367,15 @@ be posted, which is not a merge-blocker. ## Costs and quotas Diffy bills per screenshot set. **Vortex**'s default gating (`VR` label on -PRs only, plus the `deps/*` auto-branch list) keeps the run rate low. -Adjust `VR_DIFFY_PR_LABEL`, `VR_DIFFY_AUTO_BRANCHES`, and +PRs only, plus the `deps/*` auto-branch list, with non-PR deployments off +entirely) keeps the run rate low. Adjust `VR_DIFFY_PR_LABEL`, +`VR_DIFFY_AUTO_BRANCHES`, `VR_DIFFY_NON_PR_BRANCHES`, and `VORTEX_NOTIFY_DIFFY_BRANCHES` to suit the team's review rhythm. +Scope `VR_DIFFY_NON_PR_BRANCHES` as narrowly as the workflow allows: `*` +compares on every deployment to every permanent environment, which on a +busy integration branch consumes quota quickly. + ## Disabling Remove `diffy` from `VORTEX_NOTIFY_CHANNELS` and delete the From 0aeea6183a15648656ee30c7f6d128eba5e7156f Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 17:09:59 +1000 Subject: [PATCH 02/10] Updated snapshots. --- .../.github/workflows/test-vr.yml | 106 +++++++++++++----- 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml index 113b96b03..dbedc5515 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml @@ -1,9 +1,9 @@ # GitHub Actions visual regression testing workflow. # # Runs a Diffy visual regression comparison after a deployment lands. -# Triggered automatically by 'notify-diffy' (via repository_dispatch) -# when the deployed PR has the configured label, or manually via -# 'workflow_dispatch' against any URL. +# Triggered automatically by 'notify-diffy' (via repository_dispatch) when +# the deployed PR has the configured label or the deployed branch is opted +# in, or manually via 'workflow_dispatch' against any URL. name: Test - Visual regression on: @@ -44,9 +44,11 @@ env: DIFFY_MAX_WAIT: ${{ vars.VR_DIFFY_MAX_WAIT || '2700' }} DIFFY_PR_LABEL: ${{ vars.VR_DIFFY_PR_LABEL || 'VR' }} DIFFY_AUTO_BRANCHES: ${{ vars.VR_DIFFY_AUTO_BRANCHES || 'deps/*' }} + DIFFY_NON_PR_BRANCHES: ${{ vars.VR_DIFFY_NON_PR_BRANCHES }} DIFFY_POLL_INTERVAL: ${{ vars.VR_DIFFY_POLL_INTERVAL || '30' }} SOURCE_ENV: ${{ github.event.client_payload.source_env || inputs.source_env || 'production' }} TARGET_URL: ${{ github.event.client_payload.target_url || inputs.target_url }} + BRANCH: ${{ github.event.client_payload.branch }} LABEL: ${{ github.event.client_payload.label || inputs.label || 'manual' }} jobs: @@ -74,18 +76,18 @@ jobs: # Hosting providers expose PR environments via URLs that contain # a 'pr-' segment (e.g. 'app.pr-123.example.lagoon.cloud'). - # Extract the number from the URL; if no match, this is not a PR - # deployment and visual regression should not run. + # Extract the number from the URL; a URL without one is not a PR + # deployment and is gated on the deployed branch instead. pr_number="$(printf '%s' "${TARGET_URL}" | sed -n 's|.*pr-\([0-9]\{1,\}\).*|\1|p')" if [ -n "${pr_number}" ]; then echo "PR #${pr_number} resolved from target URL ${TARGET_URL}." else - echo "::notice::No PR pattern found in target URL ${TARGET_URL}. Visual regression will not run." + echo "No PR pattern found in target URL ${TARGET_URL}." fi echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT" - - name: Gate run on PR label (or skip for non-PR contexts on dispatch) + - name: Gate run on PR label or opted-in branch id: gate env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -95,6 +97,39 @@ jobs: run: | set -euo pipefail + # Matches a git ref against a comma-separated list of glob + # patterns. Returns 0 on the first match; 1 when the ref or the + # list is empty, or when nothing matches. + ref_matches() { + local ref="${1}" + local list="${2}" + local pattern + local patterns + + if [ -z "${ref}" ] || [ -z "${list}" ]; then + return 1 + fi + + IFS=',' read -ra patterns <<<"${list}" + + for pattern in "${patterns[@]}"; do + # Trim surrounding whitespace. + pattern="${pattern# }" + pattern="${pattern% }" + + if [ -z "${pattern}" ]; then + continue + fi + + # shellcheck disable=SC2254 + case "${ref}" in + ${pattern}) return 0 ;; + esac + done + + return 1 + } + # Manual workflow_dispatch always proceeds - the operator # explicitly asked for a comparison. if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then @@ -103,36 +138,39 @@ jobs: exit 0 fi - # repository_dispatch with no PR resolved - nothing to gate on. + # No PR resolved - the deployment targets a permanent environment + # such as a release branch, 'develop', 'dev' or 'stage'. Those run + # only when the deployed branch is explicitly opted in. if [ -z "${PR_NUMBER}" ]; then - echo "::notice::No PR associated with this deployment, skipping visual regression." + if ref_matches "${BRANCH}" "${DIFFY_NON_PR_BRANCHES}"; then + echo "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES, proceeding." + echo "skipped=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "::notice::No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." + echo "skipped=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # PR resolved - fetch labels and head branch in one call. A failed + # lookup means the 'pr-' segment in the URL did not come + # from a pull request in this repository. + if ! pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"; then + echo "::notice::PR #${PR_NUMBER} could not be read, skipping visual regression." echo "skipped=true" >> "$GITHUB_OUTPUT" exit 0 fi - # PR resolved - fetch labels and head branch in one call. - pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)" head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')" labels_lower="$(printf '%s' "${pr_data}" | jq -r '.labels[].name' | tr '[:upper:]' '[:lower:]')" # Auto-run bypass: PRs from configured branches (e.g. Renovate's # `deps/*`) skip the label check. - if [ -n "${DIFFY_AUTO_BRANCHES}" ]; then - IFS=',' read -ra patterns <<<"${DIFFY_AUTO_BRANCHES}" - for pattern in "${patterns[@]}"; do - # Trim surrounding whitespace. - pattern="${pattern# }" - pattern="${pattern% }" - [ -z "${pattern}" ] && continue - # shellcheck disable=SC2254 - case "${head_ref}" in - ${pattern}) - echo "PR #${PR_NUMBER} head branch '${head_ref}' matches auto-run pattern '${pattern}', proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 - ;; - esac - done + if ref_matches "${head_ref}" "${DIFFY_AUTO_BRANCHES}"; then + echo "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES, proceeding." + echo "skipped=false" >> "$GITHUB_OUTPUT" + exit 0 fi # Label check (case-insensitive). @@ -266,6 +304,20 @@ jobs: echo "Diff summary: ${pages_changed} of ${pages_total} pages changed (${changes_percent}% total)." echo "Report URL: ${shared_url}" + + # Runs without a PR - opted-in branches and manual dispatches - + # have nowhere to post a comment, so the run summary is the only + # place the result surfaces inside GitHub. + { + echo "### Visual regression report" + echo + echo "- **Pages changed**: ${pages_changed} of ${pages_total}" + echo "- **Overall difference**: ${changes_percent}%" + echo "- **Target environment**: ${TARGET_URL}" + echo "- **Source environment**: ${SOURCE_ENV}" + echo + echo "[View full Diffy report](${shared_url})" + } >> "$GITHUB_STEP_SUMMARY" env: STEPS_COMPARE_OUTPUTS_DIFF_ID: ${{ steps.compare.outputs.diff_id }} From 28addc8d38832a3b3741f1628fac774d3e834907 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 17:12:30 +1000 Subject: [PATCH 03/10] [#2863] Trimmed all surrounding whitespace from branch patterns. A pattern list written as 'release/*, develop' left the extra spaces in place, so the pattern never matched. --- .github/workflows/test-vr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-vr.yml b/.github/workflows/test-vr.yml index bdf38a29c..e4c8086e9 100644 --- a/.github/workflows/test-vr.yml +++ b/.github/workflows/test-vr.yml @@ -114,8 +114,8 @@ jobs: for pattern in "${patterns[@]}"; do # Trim surrounding whitespace. - pattern="${pattern# }" - pattern="${pattern% }" + pattern="${pattern#"${pattern%%[![:space:]]*}"}" + pattern="${pattern%"${pattern##*[![:space:]]}"}" if [ -z "${pattern}" ]; then continue From dd8b8c69e5ff7e44a164a9dbf505c282e813f4a8 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 17:14:56 +1000 Subject: [PATCH 04/10] [#2863] Documented where runs without a pull request report their result. The flow diagrams showed every run ending in a PR comment, and the manual entry point showed a 'vr-report' job that never runs without a resolved PR. --- .../content/development/visual-regression.mdx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.vortex/docs/content/development/visual-regression.mdx b/.vortex/docs/content/development/visual-regression.mdx index 11bfda669..bbb4828d7 100644 --- a/.vortex/docs/content/development/visual-regression.mdx +++ b/.vortex/docs/content/development/visual-regression.mdx @@ -108,7 +108,7 @@ GitHub branch protection rules. ```text ┌─ GitHub ────────────────────────────────────┐ │ │ -│ PR opened with `VR` label │ +│ PR with `VR` label, or opted-in branch │ │ │ │ │ ▼ │ │ build-test-deploy.yml │ @@ -142,11 +142,20 @@ GitHub branch protection rules. │ ▼ │ │ vr-compare: parse PR from URL, gate │ │ │ │ +│ ┌────────┴────────┐ │ +│ ▼ ▼ │ +│ PR deployment no PR resolved │ +│ │ │ │ +│ ▼ ▼ │ +│ VR label or branch in │ +│ auto-branch non-PR list │ +│ │ │ │ +│ └────────┬────────┘ │ │ ▼ │ │ Report in workflow run summary │ │ │ │ │ ▼ │ -│ vr-report │ +│ vr-report (PR only) │ │ │ │ │ ▼ │ │ PR comment + Diffy report link │ @@ -347,7 +356,8 @@ entry point. 1. Posts the same report as a sticky comment on the PR, with a link to the Diffy report. Re-deploys edit the same comment rather than - stacking. + stacking. The job runs only when a PR was resolved - manual runs and + opted-in branch deployments report through the run summary alone. ## Making it blocking From 2c1b11f6e500a0c75ff1d4b4e13022033f8602cd Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 17:18:30 +1000 Subject: [PATCH 05/10] Updated snapshots. --- .../visual_regression_enabled/.github/workflows/test-vr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml index dbedc5515..58950dcf8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml @@ -114,8 +114,8 @@ jobs: for pattern in "${patterns[@]}"; do # Trim surrounding whitespace. - pattern="${pattern# }" - pattern="${pattern% }" + pattern="${pattern#"${pattern%%[![:space:]]*}"}" + pattern="${pattern%"${pattern##*[![:space:]]}"}" if [ -z "${pattern}" ]; then continue From 5c04b62bfec4d8e8891b2820aff424739cde7cc5 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 17:22:22 +1000 Subject: [PATCH 06/10] [#2863] Placed both gate decisions at their evaluation point in the flow diagram. The entry box implied the branch opt-in was evaluated before the deployment, when the workflow evaluates it after the dispatch. --- .vortex/docs/content/development/visual-regression.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vortex/docs/content/development/visual-regression.mdx b/.vortex/docs/content/development/visual-regression.mdx index bbb4828d7..c8a8bdec9 100644 --- a/.vortex/docs/content/development/visual-regression.mdx +++ b/.vortex/docs/content/development/visual-regression.mdx @@ -108,7 +108,7 @@ GitHub branch protection rules. ```text ┌─ GitHub ────────────────────────────────────┐ │ │ -│ PR with `VR` label, or opted-in branch │ +│ PR opened or branch pushed │ │ │ │ │ ▼ │ │ build-test-deploy.yml │ From 5c4118fbf2d5e12b0094fb2d24dbdc4f3ad3eaa0 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 19:11:12 +1000 Subject: [PATCH 07/10] [#2863] Extracted the gate decision into named 'proceed' and 'skip' helpers. Recording the decision and ending the step was written out at all seven exit points, so a change to the step's output contract had to be repeated seven times. --- .github/workflows/test-vr.yml | 42 ++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-vr.yml b/.github/workflows/test-vr.yml index e4c8086e9..88b5cae82 100644 --- a/.github/workflows/test-vr.yml +++ b/.github/workflows/test-vr.yml @@ -130,12 +130,23 @@ jobs: return 1 } + # Records the gate decision and ends the step. + proceed() { + echo "${1}" + echo "skipped=false" >> "$GITHUB_OUTPUT" + exit 0 + } + + skip() { + echo "::notice::${1}" + echo "skipped=true" >> "$GITHUB_OUTPUT" + exit 0 + } + # Manual workflow_dispatch always proceeds - the operator # explicitly asked for a comparison. if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then - echo "Manual dispatch, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 + proceed "Manual dispatch." fi # No PR resolved - the deployment targets a permanent environment @@ -143,23 +154,17 @@ jobs: # only when the deployed branch is explicitly opted in. if [ -z "${PR_NUMBER}" ]; then if ref_matches "${BRANCH}" "${DIFFY_NON_PR_BRANCHES}"; then - echo "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 + proceed "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES." fi - echo "::notice::No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 + skip "No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." fi # PR resolved - fetch labels and head branch in one call. A failed # lookup means the 'pr-' segment in the URL did not come # from a pull request in this repository. if ! pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"; then - echo "::notice::PR #${PR_NUMBER} could not be read, skipping visual regression." - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 + skip "PR #${PR_NUMBER} could not be read, skipping visual regression." fi head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')" @@ -168,21 +173,18 @@ jobs: # Auto-run bypass: PRs from configured branches (e.g. Renovate's # `deps/*`) skip the label check. if ref_matches "${head_ref}" "${DIFFY_AUTO_BRANCHES}"; then - echo "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 + proceed "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES." fi # Label check (case-insensitive). needle="$(printf '%s' "${DIFFY_PR_LABEL}" | tr '[:upper:]' '[:lower:]')" + if echo "${labels_lower}" | grep -qx "${needle}"; then - echo "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - else - echo "::notice::PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression." - echo "skipped=true" >> "$GITHUB_OUTPUT" + proceed "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label." fi + skip "PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression." + - name: Validate target URL if: steps.gate.outputs.skipped != 'true' run: | From 7c08efd30a072a26a4157d650c7415c3b5e0fb2f Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 19:13:30 +1000 Subject: [PATCH 08/10] Updated snapshots. --- .../.github/workflows/test-vr.yml | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml index 58950dcf8..a0873221e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml @@ -130,12 +130,23 @@ jobs: return 1 } + # Records the gate decision and ends the step. + proceed() { + echo "${1}" + echo "skipped=false" >> "$GITHUB_OUTPUT" + exit 0 + } + + skip() { + echo "::notice::${1}" + echo "skipped=true" >> "$GITHUB_OUTPUT" + exit 0 + } + # Manual workflow_dispatch always proceeds - the operator # explicitly asked for a comparison. if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then - echo "Manual dispatch, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 + proceed "Manual dispatch." fi # No PR resolved - the deployment targets a permanent environment @@ -143,23 +154,17 @@ jobs: # only when the deployed branch is explicitly opted in. if [ -z "${PR_NUMBER}" ]; then if ref_matches "${BRANCH}" "${DIFFY_NON_PR_BRANCHES}"; then - echo "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 + proceed "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES." fi - echo "::notice::No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 + skip "No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." fi # PR resolved - fetch labels and head branch in one call. A failed # lookup means the 'pr-' segment in the URL did not come # from a pull request in this repository. if ! pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"; then - echo "::notice::PR #${PR_NUMBER} could not be read, skipping visual regression." - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 + skip "PR #${PR_NUMBER} could not be read, skipping visual regression." fi head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')" @@ -168,21 +173,18 @@ jobs: # Auto-run bypass: PRs from configured branches (e.g. Renovate's # `deps/*`) skip the label check. if ref_matches "${head_ref}" "${DIFFY_AUTO_BRANCHES}"; then - echo "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 + proceed "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES." fi # Label check (case-insensitive). needle="$(printf '%s' "${DIFFY_PR_LABEL}" | tr '[:upper:]' '[:lower:]')" + if echo "${labels_lower}" | grep -qx "${needle}"; then - echo "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label, proceeding." - echo "skipped=false" >> "$GITHUB_OUTPUT" - else - echo "::notice::PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression." - echo "skipped=true" >> "$GITHUB_OUTPUT" + proceed "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label." fi + skip "PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression." + - name: Validate target URL if: steps.gate.outputs.skipped != 'true' run: | @@ -304,20 +306,6 @@ jobs: echo "Diff summary: ${pages_changed} of ${pages_total} pages changed (${changes_percent}% total)." echo "Report URL: ${shared_url}" - - # Runs without a PR - opted-in branches and manual dispatches - - # have nowhere to post a comment, so the run summary is the only - # place the result surfaces inside GitHub. - { - echo "### Visual regression report" - echo - echo "- **Pages changed**: ${pages_changed} of ${pages_total}" - echo "- **Overall difference**: ${changes_percent}%" - echo "- **Target environment**: ${TARGET_URL}" - echo "- **Source environment**: ${SOURCE_ENV}" - echo - echo "[View full Diffy report](${shared_url})" - } >> "$GITHUB_STEP_SUMMARY" env: STEPS_COMPARE_OUTPUTS_DIFF_ID: ${{ steps.compare.outputs.diff_id }} From 02b37d3dd0d76564ebafa82e2de5b20a43a9b116 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 20:11:40 +1000 Subject: [PATCH 09/10] [#2863] Renamed the branch gate variables and compacted the gate step. Named the new variable 'VR_DIFFY_BRANCHES' rather than describing it by what it is not, and renamed 'VR_DIFFY_AUTO_BRANCHES' to 'VR_DIFFY_PR_SKIP_BRANCHES' so each name states which gate it controls. Collapsed the gate to roughly half its height: single-line helpers, guard clauses as AND lists, splitting the pattern list on whitespace as well as commas instead of trimming each pattern by hand, and a case-insensitive fixed-string 'grep' in place of lowercasing both sides of the label comparison. 'VR_DIFFY_AUTO_BRANCHES' is an existing repository variable, so a consumer that has set it falls back to the 'deps/*' default until it is renamed. --- .github/workflows/test-vr.yml | 95 +++++-------------- .../content/development/visual-regression.mdx | 37 ++++---- 2 files changed, 44 insertions(+), 88 deletions(-) diff --git a/.github/workflows/test-vr.yml b/.github/workflows/test-vr.yml index 88b5cae82..e1d91eccf 100644 --- a/.github/workflows/test-vr.yml +++ b/.github/workflows/test-vr.yml @@ -43,8 +43,8 @@ env: DIFFY_CLI_VERSION: ${{ vars.VR_DIFFY_CLI_VERSION || '0.1.53' }} DIFFY_MAX_WAIT: ${{ vars.VR_DIFFY_MAX_WAIT || '2700' }} DIFFY_PR_LABEL: ${{ vars.VR_DIFFY_PR_LABEL || 'VR' }} - DIFFY_AUTO_BRANCHES: ${{ vars.VR_DIFFY_AUTO_BRANCHES || 'deps/*' }} - DIFFY_NON_PR_BRANCHES: ${{ vars.VR_DIFFY_NON_PR_BRANCHES }} + DIFFY_PR_SKIP_BRANCHES: ${{ vars.VR_DIFFY_PR_SKIP_BRANCHES || 'deps/*' }} + DIFFY_BRANCHES: ${{ vars.VR_DIFFY_BRANCHES }} DIFFY_POLL_INTERVAL: ${{ vars.VR_DIFFY_POLL_INTERVAL || '30' }} SOURCE_ENV: ${{ github.event.client_payload.source_env || inputs.source_env || 'production' }} TARGET_URL: ${{ github.event.client_payload.target_url || inputs.target_url }} @@ -97,93 +97,50 @@ jobs: run: | set -euo pipefail - # Matches a git ref against a comma-separated list of glob - # patterns. Returns 0 on the first match; 1 when the ref or the - # list is empty, or when nothing matches. + # Matches a git ref against a comma-separated glob list. Splitting + # on whitespace as well as commas absorbs padding around each + # pattern. ref_matches() { - local ref="${1}" - local list="${2}" - local pattern - local patterns - - if [ -z "${ref}" ] || [ -z "${list}" ]; then - return 1 - fi - - IFS=',' read -ra patterns <<<"${list}" - + local ref="${1}" pattern patterns + [ -n "${ref}" ] || return 1 + [ -n "${2}" ] || return 1 + IFS=$', \t' read -ra patterns <<<"${2}" for pattern in "${patterns[@]}"; do - # Trim surrounding whitespace. - pattern="${pattern#"${pattern%%[![:space:]]*}"}" - pattern="${pattern%"${pattern##*[![:space:]]}"}" - - if [ -z "${pattern}" ]; then - continue - fi - # shellcheck disable=SC2254 - case "${ref}" in - ${pattern}) return 0 ;; - esac + case "${ref}" in ${pattern}) return 0 ;; esac done - return 1 } - # Records the gate decision and ends the step. - proceed() { - echo "${1}" - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 - } + # Record the gate decision and end the step. + proceed() { echo "${1}"; echo "skipped=false" >> "$GITHUB_OUTPUT"; exit 0; } + skip() { echo "::notice::${1}"; echo "skipped=true" >> "$GITHUB_OUTPUT"; exit 0; } - skip() { - echo "::notice::${1}" - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 - } - - # Manual workflow_dispatch always proceeds - the operator - # explicitly asked for a comparison. - if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then - proceed "Manual dispatch." - fi + # The operator explicitly asked for a comparison. + [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && proceed "Manual dispatch." # No PR resolved - the deployment targets a permanent environment # such as a release branch, 'develop', 'dev' or 'stage'. Those run # only when the deployed branch is explicitly opted in. if [ -z "${PR_NUMBER}" ]; then - if ref_matches "${BRANCH}" "${DIFFY_NON_PR_BRANCHES}"; then - proceed "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES." - fi - - skip "No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." + ref_matches "${BRANCH}" "${DIFFY_BRANCHES}" && proceed "Deployed branch '${BRANCH}' matches VR_DIFFY_BRANCHES." + skip "No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_BRANCHES, skipping visual regression." fi - # PR resolved - fetch labels and head branch in one call. A failed - # lookup means the 'pr-' segment in the URL did not come - # from a pull request in this repository. - if ! pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"; then - skip "PR #${PR_NUMBER} could not be read, skipping visual regression." - fi + # A failed lookup means the 'pr-' segment in the URL did + # not come from a pull request in this repository. + pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)" || skip "PR #${PR_NUMBER} could not be read, skipping visual regression." head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')" - labels_lower="$(printf '%s' "${pr_data}" | jq -r '.labels[].name' | tr '[:upper:]' '[:lower:]')" - - # Auto-run bypass: PRs from configured branches (e.g. Renovate's - # `deps/*`) skip the label check. - if ref_matches "${head_ref}" "${DIFFY_AUTO_BRANCHES}"; then - proceed "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES." - fi + labels="$(printf '%s' "${pr_data}" | jq -r '.labels[].name')" - # Label check (case-insensitive). - needle="$(printf '%s' "${DIFFY_PR_LABEL}" | tr '[:upper:]' '[:lower:]')" + # PRs from configured branches (e.g. Renovate's `deps/*`) skip the + # label check. + ref_matches "${head_ref}" "${DIFFY_PR_SKIP_BRANCHES}" && proceed "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_PR_SKIP_BRANCHES." - if echo "${labels_lower}" | grep -qx "${needle}"; then - proceed "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label." - fi + echo "${labels}" | grep -qixF "${DIFFY_PR_LABEL}" && proceed "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label." - skip "PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression." + skip "PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_PR_SKIP_BRANCHES, skipping visual regression." - name: Validate target URL if: steps.gate.outputs.skipped != 'true' diff --git a/.vortex/docs/content/development/visual-regression.mdx b/.vortex/docs/content/development/visual-regression.mdx index c8a8bdec9..4593b2c68 100644 --- a/.vortex/docs/content/development/visual-regression.mdx +++ b/.vortex/docs/content/development/visual-regression.mdx @@ -75,8 +75,8 @@ the site. In Cloudflare, add an IP Access Rule for that address with the | `VR_DIFFY_MAX_WAIT` | `2700` | Maximum seconds to wait for a comparison to complete (45 minutes) | | `VR_DIFFY_POLL_INTERVAL` | `30` | Polling interval in seconds | | `VR_DIFFY_PR_LABEL` | `VR` | PR label that opts a deployment into visual regression. Case-insensitive | -| `VR_DIFFY_AUTO_BRANCHES` | `deps/*` | Comma-separated glob list of PR head branches that bypass the label gate (matches Renovate's `branchPrefix`). Set to empty to require the label on every PR. | -| `VR_DIFFY_NON_PR_BRANCHES` | (empty) | Comma-separated glob list of deployed branches that run visual regression when the deployment is *not* a PR environment (for example `release/*`). Empty means non-PR deployments never run. | +| `VR_DIFFY_PR_SKIP_BRANCHES` | `deps/*` | Comma-separated glob list of PR head branches that skip the label gate (matches Renovate's `branchPrefix`). Set to empty to require the label on every PR. | +| `VR_DIFFY_BRANCHES` | (empty) | Comma-separated glob list of deployed branches that run visual regression when the deployment has no pull request (for example `release/*`). Empty means such deployments never run. | | `VR_DIFFY_SOURCE_ENV` | `production` | Default Diffy source environment for comparisons | Add these under *Settings > Secrets and variables > Actions > Repository @@ -148,7 +148,7 @@ GitHub branch protection rules. │ │ │ │ │ ▼ ▼ │ │ VR label or branch in │ -│ auto-branch non-PR list │ +│ PR skip list branch list │ │ │ │ │ │ └────────┬────────┘ │ │ ▼ │ @@ -230,8 +230,8 @@ be the sole gate. PRs raised by Renovate (or any other automated dependency-update bot) typically do not carry the `VR` label - they carry their own bot label (e.g. `Dependencies`). To still run visual regression on them, the -workflow consults `VR_DIFFY_AUTO_BRANCHES`: a comma-separated glob list -of PR head branches that bypass the `VR` label gate. +workflow consults `VR_DIFFY_PR_SKIP_BRANCHES`: a comma-separated glob +list of PR head branches that skip the `VR` label gate. The default value is `deps/*`, matching **Vortex**'s Renovate `branchPrefix` configuration. Other common values: @@ -243,13 +243,12 @@ configuration. Other common values: | Dependabot | `dependabot/` | `dependabot/*` | Multiple patterns can be combined with commas: -`VR_DIFFY_AUTO_BRANCHES=deps/*,dependabot/*`. Set the variable to empty -to disable the bypass entirely (every PR, including bot PRs, then needs -the label). +`VR_DIFFY_PR_SKIP_BRANCHES=deps/*,dependabot/*`. Set the variable to +empty to require the label on every PR, including bot PRs. ### Default behavior out of the box -With `VR_DIFFY_AUTO_BRANCHES=deps/*` (default) and +With `VR_DIFFY_PR_SKIP_BRANCHES=deps/*` (default) and `VR_DIFFY_PR_LABEL=VR` (default): | PR head branch | Has `VR` label? | Runs? | @@ -260,8 +259,8 @@ With `VR_DIFFY_AUTO_BRANCHES=deps/*` (default) and | `deps/drupal-core-11.2` | yes | yes (matches `deps/*`, label irrelevant) | Consumers using Dependabot just append: -`VR_DIFFY_AUTO_BRANCHES=deps/*,dependabot/*`. Consumers who want the -label as the only gate (no auto-bypass) set the variable to empty. +`VR_DIFFY_PR_SKIP_BRANCHES=deps/*,dependabot/*`. Consumers who want the +label as the only gate set the variable to empty. ## Release and other non-PR deployments @@ -270,16 +269,16 @@ integration branch such as `develop`, or a permanent `dev`/`stage` environment - has no pull request to carry the `VR` label, so it is skipped by default. -Set `VR_DIFFY_NON_PR_BRANCHES` to a comma-separated glob list of deployed -branch names to compare them automatically: +Set `VR_DIFFY_BRANCHES` to a comma-separated glob list of deployed branch +names to compare them automatically: ```ini -VR_DIFFY_NON_PR_BRANCHES=release/* +VR_DIFFY_BRANCHES=release/* ``` -| Deployment | `VR_DIFFY_NON_PR_BRANCHES` | Runs? | +| Deployment | `VR_DIFFY_BRANCHES` | Runs? | |---|---|---| -| PR environment (`pr-123` in the URL) | any value | gated by the `VR` label and `VR_DIFFY_AUTO_BRANCHES` as usual | +| PR environment (`pr-123` in the URL) | any value | gated by the `VR` label and `VR_DIFFY_PR_SKIP_BRANCHES` as usual | | Branch `release/1.2.3` | (empty) | no | | Branch `release/1.2.3` | `release/*` | **yes** | | Branch `develop` | `release/*` | no | @@ -324,7 +323,7 @@ This means: If the target URL has no `pr-` segment (for example, a deploy to a named environment like `dev`/`test`/`prod`), the workflow treats it as "not a PR deployment" and runs only when the deployed branch matches -`VR_DIFFY_NON_PR_BRANCHES`. +`VR_DIFFY_BRANCHES`. ## Missed-window behavior @@ -379,10 +378,10 @@ be posted, which is not a merge-blocker. Diffy bills per screenshot set. **Vortex**'s default gating (`VR` label on PRs only, plus the `deps/*` auto-branch list, with non-PR deployments off entirely) keeps the run rate low. Adjust `VR_DIFFY_PR_LABEL`, -`VR_DIFFY_AUTO_BRANCHES`, `VR_DIFFY_NON_PR_BRANCHES`, and +`VR_DIFFY_PR_SKIP_BRANCHES`, `VR_DIFFY_BRANCHES`, and `VORTEX_NOTIFY_DIFFY_BRANCHES` to suit the team's review rhythm. -Scope `VR_DIFFY_NON_PR_BRANCHES` as narrowly as the workflow allows: `*` +Scope `VR_DIFFY_BRANCHES` as narrowly as the workflow allows: `*` compares on every deployment to every permanent environment, which on a busy integration branch consumes quota quickly. From eea96515276ca3832c85a14f2faa51a5013f62ec Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 29 Jul 2026 20:13:00 +1000 Subject: [PATCH 10/10] Updated snapshots. --- .../.github/workflows/test-vr.yml | 95 +++++-------------- 1 file changed, 26 insertions(+), 69 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml index a0873221e..a48ce4a88 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/visual_regression_enabled/.github/workflows/test-vr.yml @@ -43,8 +43,8 @@ env: DIFFY_CLI_VERSION: ${{ vars.VR_DIFFY_CLI_VERSION || '__VERSION__' }} DIFFY_MAX_WAIT: ${{ vars.VR_DIFFY_MAX_WAIT || '2700' }} DIFFY_PR_LABEL: ${{ vars.VR_DIFFY_PR_LABEL || 'VR' }} - DIFFY_AUTO_BRANCHES: ${{ vars.VR_DIFFY_AUTO_BRANCHES || 'deps/*' }} - DIFFY_NON_PR_BRANCHES: ${{ vars.VR_DIFFY_NON_PR_BRANCHES }} + DIFFY_PR_SKIP_BRANCHES: ${{ vars.VR_DIFFY_PR_SKIP_BRANCHES || 'deps/*' }} + DIFFY_BRANCHES: ${{ vars.VR_DIFFY_BRANCHES }} DIFFY_POLL_INTERVAL: ${{ vars.VR_DIFFY_POLL_INTERVAL || '30' }} SOURCE_ENV: ${{ github.event.client_payload.source_env || inputs.source_env || 'production' }} TARGET_URL: ${{ github.event.client_payload.target_url || inputs.target_url }} @@ -97,93 +97,50 @@ jobs: run: | set -euo pipefail - # Matches a git ref against a comma-separated list of glob - # patterns. Returns 0 on the first match; 1 when the ref or the - # list is empty, or when nothing matches. + # Matches a git ref against a comma-separated glob list. Splitting + # on whitespace as well as commas absorbs padding around each + # pattern. ref_matches() { - local ref="${1}" - local list="${2}" - local pattern - local patterns - - if [ -z "${ref}" ] || [ -z "${list}" ]; then - return 1 - fi - - IFS=',' read -ra patterns <<<"${list}" - + local ref="${1}" pattern patterns + [ -n "${ref}" ] || return 1 + [ -n "${2}" ] || return 1 + IFS=$', \t' read -ra patterns <<<"${2}" for pattern in "${patterns[@]}"; do - # Trim surrounding whitespace. - pattern="${pattern#"${pattern%%[![:space:]]*}"}" - pattern="${pattern%"${pattern##*[![:space:]]}"}" - - if [ -z "${pattern}" ]; then - continue - fi - # shellcheck disable=SC2254 - case "${ref}" in - ${pattern}) return 0 ;; - esac + case "${ref}" in ${pattern}) return 0 ;; esac done - return 1 } - # Records the gate decision and ends the step. - proceed() { - echo "${1}" - echo "skipped=false" >> "$GITHUB_OUTPUT" - exit 0 - } + # Record the gate decision and end the step. + proceed() { echo "${1}"; echo "skipped=false" >> "$GITHUB_OUTPUT"; exit 0; } + skip() { echo "::notice::${1}"; echo "skipped=true" >> "$GITHUB_OUTPUT"; exit 0; } - skip() { - echo "::notice::${1}" - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 - } - - # Manual workflow_dispatch always proceeds - the operator - # explicitly asked for a comparison. - if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then - proceed "Manual dispatch." - fi + # The operator explicitly asked for a comparison. + [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && proceed "Manual dispatch." # No PR resolved - the deployment targets a permanent environment # such as a release branch, 'develop', 'dev' or 'stage'. Those run # only when the deployed branch is explicitly opted in. if [ -z "${PR_NUMBER}" ]; then - if ref_matches "${BRANCH}" "${DIFFY_NON_PR_BRANCHES}"; then - proceed "Deployed branch '${BRANCH}' matches VR_DIFFY_NON_PR_BRANCHES." - fi - - skip "No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_NON_PR_BRANCHES, skipping visual regression." + ref_matches "${BRANCH}" "${DIFFY_BRANCHES}" && proceed "Deployed branch '${BRANCH}' matches VR_DIFFY_BRANCHES." + skip "No PR associated with this deployment and branch '${BRANCH}' does not match VR_DIFFY_BRANCHES, skipping visual regression." fi - # PR resolved - fetch labels and head branch in one call. A failed - # lookup means the 'pr-' segment in the URL did not come - # from a pull request in this repository. - if ! pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"; then - skip "PR #${PR_NUMBER} could not be read, skipping visual regression." - fi + # A failed lookup means the 'pr-' segment in the URL did + # not come from a pull request in this repository. + pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)" || skip "PR #${PR_NUMBER} could not be read, skipping visual regression." head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')" - labels_lower="$(printf '%s' "${pr_data}" | jq -r '.labels[].name' | tr '[:upper:]' '[:lower:]')" - - # Auto-run bypass: PRs from configured branches (e.g. Renovate's - # `deps/*`) skip the label check. - if ref_matches "${head_ref}" "${DIFFY_AUTO_BRANCHES}"; then - proceed "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_AUTO_BRANCHES." - fi + labels="$(printf '%s' "${pr_data}" | jq -r '.labels[].name')" - # Label check (case-insensitive). - needle="$(printf '%s' "${DIFFY_PR_LABEL}" | tr '[:upper:]' '[:lower:]')" + # PRs from configured branches (e.g. Renovate's `deps/*`) skip the + # label check. + ref_matches "${head_ref}" "${DIFFY_PR_SKIP_BRANCHES}" && proceed "PR #${PR_NUMBER} head branch '${head_ref}' matches VR_DIFFY_PR_SKIP_BRANCHES." - if echo "${labels_lower}" | grep -qx "${needle}"; then - proceed "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label." - fi + echo "${labels}" | grep -qixF "${DIFFY_PR_LABEL}" && proceed "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label." - skip "PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression." + skip "PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_PR_SKIP_BRANCHES, skipping visual regression." - name: Validate target URL if: steps.gate.outputs.skipped != 'true'