acc: Add -regr-main and -regr-release to compare changed tests against main and latest release

.github/workflows/regression-report.yml

@@ -0,0 +1,105 @@
+name: Regression report
+
+# Runs the acceptance tests a PR changed against main and the latest release, and posts the
+# result as a PR comment (see `./task regr`). A new commit updates the same comment.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+# A newer commit supersedes the run in progress, so an old report never overwrites a new one.
+concurrency:
+  group: regression-report-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  regression-report:
+    # Fork PRs get a read-only token and cannot comment.
+    if: >-
+      !github.event.pull_request.head.repo.fork &&
+      github.actor != 'dependabot[bot]'
+
+    runs-on:
+      group: databricks-deco-testing-runner-group
+      labels: ubuntu-latest-deco
+
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: regression-report
+
+      # `git diff --merge-base origin/main` needs origin/main and enough history. This must run
+      # AFTER setup-build-environment, whose own checkout would re-shallow the repo.
+      - name: Fetch origin/main
+        run: |
+          git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
+          if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then
+            git fetch --no-tags --unshallow origin
+          fi
+
+      - name: Run regression tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: go tool -modfile=tools/task/go.mod task regr
+
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: regression-report
+          path: regression_report.md
+          if-no-files-found: ignore
+          retention-days: 7
+
+      # No report means no acceptance tests are changed (anymore), so there is nothing to
+      # comment: drop the comment of an earlier commit instead of leaving it stale.
+      - name: Post, update or delete PR comment
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |-
+            const fs = require('fs');
+            const path = process.env.GITHUB_WORKSPACE + '/regression_report.md';
+
+            // Other comments by the same bot are left alone: ours is found by this marker.
+            const MARKER = '<!-- regression-test-report -->';
+            const { owner, repo } = context.repo;
+            const issue_number = context.issue.number;
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner, repo, issue_number,
+            });
+            const existing = comments.find(c =>
+              c.user.login === 'github-actions[bot]' && c.body && c.body.startsWith(MARKER));
+
+            if (!fs.existsSync(path)) {
+              if (existing) {
+                await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });
+              }
+              return;
+            }
+
+            const sha = context.payload.pull_request.head.sha;
+            const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+            const commitUrl = `${context.serverUrl}/${owner}/${repo}/commit/${sha}`;
+
+            // Comments are limited to 65536 characters; the artifact holds the full report.
+            const MAX_REPORT = 60000;
+            let report = fs.readFileSync(path, 'utf8');
+            if (report.length > MAX_REPORT) {
+              report = report.slice(0, MAX_REPORT) + `\n\n_Report truncated, see the full report in the [run](${runUrl})._\n`;
+            }
+
+            const body = `${MARKER}\nCommit: [\`${sha.slice(0, 9)}\`](${commitUrl}) | [Workflow run](${runUrl})\n\n${report}`;
+
+            if (existing) {
+              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
+            } else {
+              await github.rest.issues.createComment({ owner, repo, issue_number, body });
+            }

.gitignore

@@ -78,3 +78,4 @@ tools/gofumpt
 .claude/worktrees/
 .worktrees/
 .isaac/
+/regression_report.md

Taskfile.yml

@@ -528,6 +528,22 @@ tasks:
     cmds:
       - "DATABRICKS_TEST_SELECT_CHANGED=${DATABRICKS_TEST_SELECT_CHANGED:-50} go test ./acceptance -run '^TestAccept$' -v -timeout=${LOCAL_TIMEOUT:-60m}"
 
+  regr:
+    desc: Run acceptance tests changed in this branch against main and the latest release, write regression_report.md
+    # No sources/generates: like local-select, the selection depends on the git diff.
+    cmds:
+      - "go test ./acceptance -run '^TestAccept$' -v -regr-main -regr-release -timeout=${LOCAL_TIMEOUT:-60m}"
+
+  regr-main:
+    desc: Run acceptance tests changed in this branch against a CLI built from main, write regression_report.md
+    cmds:
+      - "go test ./acceptance -run '^TestAccept$' -v -regr-main -timeout=${LOCAL_TIMEOUT:-60m}"
+
+  regr-release:
+    desc: Run acceptance tests changed in this branch against the latest release, write regression_report.md
+    cmds:
+      - "go test ./acceptance -run '^TestAccept$' -v -regr-release -timeout=${LOCAL_TIMEOUT:-60m}"
+
   test-update:
     desc: Update acceptance test output (local)
     # Excludes out* because the task rewrites them; keeping them in sources would

acceptance/acceptance_test.go

@@ -152,6 +152,10 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
+	if RegrMain || RegrRelease {
+		runRegression(t)
+		return
+	}
 	testAccept(t, InprocessMode, nil, false)
 }