From 3057b0b2c6b17949b1dda60f1a37d55e310c72a8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 14:56:40 +0200 Subject: [PATCH 01/10] acc: add -regr-main and -regr-release to compare changed tests against main and the latest release Co-authored-by: Isaac --- .github/workflows/regression-report.yml | 97 ++++++ .gitignore | 1 + Taskfile.yml | 16 + acceptance/acceptance_test.go | 4 + acceptance/regression_test.go | 407 ++++++++++++++++++++++++ acceptance/regression_unit_test.go | 68 ++++ 6 files changed, 593 insertions(+) create mode 100644 .github/workflows/regression-report.yml create mode 100644 acceptance/regression_test.go create mode 100644 acceptance/regression_unit_test.go diff --git a/.github/workflows/regression-report.yml b/.github/workflows/regression-report.yml new file mode 100644 index 00000000000..8032ef2b585 --- /dev/null +++ b/.github/workflows/regression-report.yml @@ -0,0 +1,97 @@ +name: Regression report + +# Runs the acceptance tests a PR changed against main and the latest release, and posts the +# result as a PR comment (see `./task regr`). A new commit updates the same comment. +on: + pull_request: + types: [opened, synchronize, reopened] + +# A newer commit supersedes the run in progress, so an old report never overwrites a new one. +concurrency: + group: regression-report-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + regression-report: + # Fork PRs get a read-only token and cannot comment. + if: >- + !github.event.pull_request.head.repo.fork && + github.actor != 'dependabot[bot]' + + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + + permissions: + id-token: write + contents: read + pull-requests: write + + steps: + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: regression-report + + # `git diff --merge-base origin/main` needs origin/main and enough history. This must run + # AFTER setup-build-environment, whose own checkout would re-shallow the repo. + - name: Fetch origin/main + run: | + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + git fetch --no-tags --unshallow origin + fi + + - name: Run regression tests + run: go tool -modfile=tools/task/go.mod task regr + + - name: Upload report + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: regression-report + path: regression_report.md + if-no-files-found: ignore + retention-days: 7 + + # No report means no acceptance tests were changed, which needs no comment. + - name: Post or update PR comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('fs'); + const path = process.env.GITHUB_WORKSPACE + '/regression_report.md'; + if (!fs.existsSync(path)) { + core.info('No regression report, nothing to post'); + return; + } + + // Other comments by the same bot are left alone: ours is found by this marker. + const MARKER = ''; + const { owner, repo } = context.repo; + const sha = context.payload.pull_request.head.sha; + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + const commitUrl = `${context.serverUrl}/${owner}/${repo}/commit/${sha}`; + + // Comments are limited to 65536 characters; the artifact holds the full report. + const MAX_REPORT = 60000; + let report = fs.readFileSync(path, 'utf8'); + if (report.length > MAX_REPORT) { + report = report.slice(0, MAX_REPORT) + `\n\n_Report truncated, see the full report in the [run](${runUrl})._\n`; + } + + const body = `${MARKER}\nCommit: [\`${sha.slice(0, 9)}\`](${commitUrl}) | [Workflow run](${runUrl})\n\n${report}`; + + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: context.issue.number, + }); + const existing = comments.find(c => + c.user.login === 'github-actions[bot]' && c.body && c.body.startsWith(MARKER)); + + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ + owner, repo, issue_number: context.issue.number, body, + }); + } diff --git a/.gitignore b/.gitignore index 86cef760657..fbf05b54389 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ tools/gofumpt .claude/worktrees/ .worktrees/ .isaac/ +/regression_report.md diff --git a/Taskfile.yml b/Taskfile.yml index be1c0e72146..7d1c26da294 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -528,6 +528,22 @@ tasks: cmds: - "DATABRICKS_TEST_SELECT_CHANGED=${DATABRICKS_TEST_SELECT_CHANGED:-50} go test ./acceptance -run '^TestAccept$' -v -timeout=${LOCAL_TIMEOUT:-60m}" + regr: + desc: Run acceptance tests changed in this branch against main and the latest release, write regression_report.md + # No sources/generates: like local-select, the selection depends on the git diff. + cmds: + - "go test ./acceptance -run '^TestAccept$' -v -regr-main -regr-release -timeout=${LOCAL_TIMEOUT:-60m}" + + regr-main: + desc: Run acceptance tests changed in this branch against a CLI built from main, write regression_report.md + cmds: + - "go test ./acceptance -run '^TestAccept$' -v -regr-main -timeout=${LOCAL_TIMEOUT:-60m}" + + regr-release: + desc: Run acceptance tests changed in this branch against the latest release, write regression_report.md + cmds: + - "go test ./acceptance -run '^TestAccept$' -v -regr-release -timeout=${LOCAL_TIMEOUT:-60m}" + test-update: desc: Update acceptance test output (local) # Excludes out* because the task rewrites them; keeping them in sources would diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index d9783615d64..c8c891a1090 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -152,6 +152,10 @@ var Scripts = map[string]bool{ } func TestAccept(t *testing.T) { + if RegrMain || RegrRelease { + runRegression(t) + return + } testAccept(t, InprocessMode, nil, false) } diff --git a/acceptance/regression_test.go b/acceptance/regression_test.go new file mode 100644 index 00000000000..32ec07d702d --- /dev/null +++ b/acceptance/regression_test.go @@ -0,0 +1,407 @@ +package acceptance_test + +import ( + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "slices" + "strconv" + "strings" + "testing" + + "github.com/databricks/cli/acceptance/internal/selection" + "github.com/stretchr/testify/require" +) + +// Regression mode runs the acceptance tests this branch changed against other CLI +// builds, to tell tests that pin new behavior (fail on main) from tests that cover +// existing behavior (pass everywhere). The tests and the test harness always come from +// this branch; only the CLI binary under test is swapped (see -clipath, -useversion). +var ( + RegrMain bool + RegrRelease bool + RegrOutput string +) + +func init() { + flag.BoolVar(&RegrMain, "regr-main", false, "Run the tests changed on this branch against a CLI built from the merge base with origin/main and report the difference. Use with -v to see the summary") + flag.BoolVar(&RegrRelease, "regr-release", false, "Run the tests changed on this branch against the latest released CLI and report the difference. Use with -v to see the summary") + flag.StringVar(&RegrOutput, "regr-output", "../regression_report.md", "Where -regr-main and -regr-release write the full markdown report") +} + +const ( + // regrDefaultLimit caps how many changed tests are compared unless selection.EnvVar says otherwise. + regrDefaultLimit = 20 + + // regrDetailLines caps the output of one failed run in the report. + regrDetailLines = 200 + + prerequisitesTest = "TestAccept/prerequisites" +) + +const ( + statusPass = "pass" + statusFail = "fail" + statusSkip = "skip" +) + +// Categories a test falls into, see classifyRegression. +const ( + // catRegression passes on this branch and fails on main: it pins behavior this branch introduced. + catRegression = "regression" + + // catUnreleased passes on this branch and main and fails on the latest release: it covers + // behavior that is merged but not shipped yet. + catUnreleased = "unreleased" + + // catCoverage passes everywhere. + catCoverage = "coverage" + + // catFailing fails on this branch. + catFailing = "failing" +) + +// regrCategories lists the categories in the order reports show them. +var regrCategories = []string{catRegression, catUnreleased, catCoverage, catFailing} + +// testResult is the outcome of one test in a child run, with the log lines it produced. +type testResult struct { + Status string + Output []string +} + +// regrColumn holds the results of running the selected tests with one CLI. +type regrColumn struct { + // Title is the column header in the report. + Title string + + // Category is what a failure in this column means for the test, empty for the branch column. + Category string + + Results map[string]*testResult +} + +func (c regrColumn) status(test string) string { + if r, ok := c.Results[test]; ok { + return r.Status + } + return "" +} + +var ( + // "=== RUN TestAccept/foo", also CONT, NAME and PAUSE: following log lines belong to that test. + markerLineRE = regexp.MustCompile(`^=== (?:RUN|CONT|NAME|PAUSE)\s+(\S+)`) + + // " --- FAIL: TestAccept/foo (0.10s)" + resultLineRE = regexp.MustCompile(`^\s*--- (PASS|FAIL|SKIP): (\S+) `) +) + +// parseTestOutput parses the output of a test binary run with -test.v into results by test name. +func parseTestOutput(output string) map[string]*testResult { + results := map[string]*testResult{} + current := "" + for line := range strings.Lines(output) { + line = strings.TrimRight(line, "\r\n") + if m := markerLineRE.FindStringSubmatch(line); m != nil { + current = m[1] + if _, ok := results[current]; !ok { + results[current] = &testResult{} + } + continue + } + if m := resultLineRE.FindStringSubmatch(line); m != nil { + if _, ok := results[m[2]]; !ok { + results[m[2]] = &testResult{} + } + results[m[2]].Status = strings.ToLower(m[1]) + continue + } + if current != "" { + results[current].Output = append(results[current].Output, line) + } + } + return results +} + +// leafTests returns the tests that ran a test script, sorted. A test with EnvMatrix variants +// is represented by its variants (named KEY=value) rather than by itself. +func leafTests(results map[string]*testResult) []string { + parents := map[string]bool{} + for name := range results { + for i, c := range name { + if c != '/' { + continue + } + next, _, _ := strings.Cut(name[i+1:], "/") + if strings.Contains(next, "=") { + parents[name[:i]] = true + } + } + } + + var leaves []string + for name, r := range results { + if parents[name] || name == "TestAccept" || name == prerequisitesTest || r.Status == "" { + continue + } + leaves = append(leaves, name) + } + slices.Sort(leaves) + return leaves +} + +func isOK(status string) bool { + return status == statusPass || status == statusSkip +} + +// classifyRegression says what the status of one test on every CLI means. refs are the +// non-branch columns in the order they are checked. +func classifyRegression(branch string, refs []string, refCategories []string) string { + if !isOK(branch) { + return catFailing + } + for i, status := range refs { + if !isOK(status) { + return refCategories[i] + } + } + return catCoverage +} + +func categorize(test string, branch regrColumn, refs []regrColumn) string { + statuses := make([]string, len(refs)) + categories := make([]string, len(refs)) + for i, ref := range refs { + statuses[i] = ref.status(test) + categories[i] = ref.Category + } + return classifyRegression(branch.status(test), statuses, categories) +} + +func statusMark(status string) string { + switch status { + case statusPass: + return "✅" + case statusFail: + return "❌" + case statusSkip: + return "⏭️" + default: + return "➖" + } +} + +// countByCategory counts the tests in each category. +func countByCategory(tests []string, branch regrColumn, refs []regrColumn) map[string]int { + counts := map[string]int{} + for _, test := range tests { + counts[categorize(test, branch, refs)]++ + } + return counts +} + +func formatCounts(counts map[string]int) string { + var parts []string + for _, cat := range regrCategories { + parts = append(parts, fmt.Sprintf("%s=%d", cat, counts[cat])) + } + return strings.Join(parts, " ") +} + +// renderRegressionReport returns the markdown report. columns[0] is the branch, the rest are +// the CLIs it is compared against. +func renderRegressionReport(selected string, columns []regrColumn) string { + branch, refs := columns[0], columns[1:] + tests := leafTests(branch.Results) + + var b strings.Builder + b.WriteString("## Regression test report\n\n") + fmt.Fprintf(&b, "%s\n\n", selected) + fmt.Fprintf(&b, "%d tests: %s\n\n", len(tests), formatCounts(countByCategory(tests, branch, refs))) + + b.WriteString("| test | " + strings.Join(columnTitles(columns), " | ") + " |\n") + b.WriteString("|---|" + strings.Repeat("---|", len(columns)) + "\n") + for _, test := range tests { + marks := make([]string, len(columns)) + for i, c := range columns { + marks[i] = statusMark(c.status(test)) + } + fmt.Fprintf(&b, "| %s | %s |\n", test, strings.Join(marks, " | ")) + } + b.WriteString("\n") + + for _, test := range tests { + renderDetails(&b, test, columns) + } + return b.String() +} + +func columnTitles(columns []regrColumn) []string { + titles := make([]string, len(columns)) + for i, c := range columns { + titles[i] = c.Title + } + return titles +} + +// renderDetails adds a collapsible section with the output of every failed run of the test. +func renderDetails(b *strings.Builder, test string, columns []regrColumn) { + var failed []regrColumn + for _, c := range columns { + if c.status(test) == statusFail { + failed = append(failed, c) + } + } + if len(failed) == 0 { + return + } + + marks := make([]string, len(columns)) + for i, c := range columns { + marks[i] = c.Title + " " + statusMark(c.status(test)) + } + fmt.Fprintf(b, "
\n%s: %s\n\n", test, strings.Join(marks, " | ")) + for _, c := range failed { + output := truncateLines(strings.Join(c.Results[test].Output, "\n"), regrDetailLines) + // The longer fence keeps backticks in the output from ending the block. + fmt.Fprintf(b, "**%s:**\n````\n%s\n````\n\n", c.Title, strings.TrimSpace(output)) + } + b.WriteString("
\n\n") +} + +// printRegressionSummary prints the short form of the report: counts, and the names of the +// tests that need attention. +func printRegressionSummary(columns []regrColumn, reportPath string) { + branch, refs := columns[0], columns[1:] + tests := leafTests(branch.Results) + + fmt.Printf("\nRegression summary (%d tests): %s\n", len(tests), formatCounts(countByCategory(tests, branch, refs))) + for _, cat := range []string{catRegression, catFailing} { + for _, test := range tests { + if categorize(test, branch, refs) == cat { + fmt.Printf(" %s: %s\n", cat, test) + } + } + } + fmt.Printf("Full report: %s\n", reportPath) +} + +// runRegression runs the tests this branch changed with the branch CLI and with each CLI +// requested by -regr-main and -regr-release, then reports the difference. +// +// Each run is a child process of this test binary, because the harness builds the CLI and +// starts the servers once per process. The children select the changed tests themselves +// (selection.EnvVar), so all of them run the same tests. +func runRegression(t *testing.T) { + testDirs, err := selection.FindTestDirs(".") + require.NoError(t, err) + testDirsSet := make(map[string]bool, len(testDirs)) + for _, dir := range testDirs { + testDirsSet[dir] = true + } + + limit, err := selection.ParseLimit(os.Getenv(selection.EnvVar)) + require.NoError(t, err) + if limit == 0 { + limit = regrDefaultLimit + } + + result, err := selection.FromGit(".", testDirsSet, limit) + require.NoError(t, err) + fmt.Println(result.Counts()) + if len(result.Selected) == 0 { + fmt.Println("No acceptance tests were changed on this branch, nothing to compare") + return + } + + cwd, err := os.Getwd() + require.NoError(t, err) + + fmt.Println("Running the selected tests with the CLI of this branch") + columns := []regrColumn{{Title: "branch", Results: runAcceptanceChild(t, limit)}} + + if RegrMain { + mergeBase := gitOutput(t, "merge-base", "HEAD", "origin/main") + short := gitOutput(t, "rev-parse", "--short", mergeBase) + fmt.Printf("Running the selected tests with the CLI of main (%s)\n", short) + cli := buildCLIAtRef(t, cwd, mergeBase) + columns = append(columns, regrColumn{ + Title: "main (" + short + ")", + Category: catRegression, + Results: runAcceptanceChild(t, limit, "-clipath="+cli), + }) + } + + if RegrRelease { + version := resolveLatestVersion(t, getBuildDir(t, cwd, runtime.GOOS, runtime.GOARCH)) + fmt.Printf("Running the selected tests with the latest release (v%s)\n", version) + columns = append(columns, regrColumn{ + Title: "latest (v" + version + ")", + Category: catUnreleased, + Results: runAcceptanceChild(t, limit, "-useversion="+version), + }) + } + + report := renderRegressionReport(result.Counts(), columns) + require.NoError(t, os.WriteFile(RegrOutput, []byte(report), 0o644)) + printRegressionSummary(columns, filepath.ToSlash(RegrOutput)) +} + +// runAcceptanceChild runs TestAccept in a child process of this test binary and returns what it +// reports. Failing tests are expected and are not an error, a run without results is. +func runAcceptanceChild(t *testing.T, limit int, args ...string) map[string]*testResult { + args = append([]string{ + "-test.run=^TestAccept$", + "-test.v", + "-test.count=1", + "-test.timeout=" + flag.Lookup("test.timeout").Value.String(), + }, args...) + + cmd := exec.Command(os.Args[0], args...) + cmd.Env = append(os.Environ(), selection.EnvVar+"="+strconv.Itoa(limit)) + out, err := cmd.CombinedOutput() + if _, ok := errors.AsType[*exec.ExitError](err); err != nil && !ok { + require.NoError(t, err) + } + + results := parseTestOutput(string(out)) + if len(leafTests(results)) == 0 { + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + require.Fail(t, "no test results", "%s failed:\n%s", cmd.Args, strings.Join(lines[max(0, len(lines)-50):], "\n")) + } + return results +} + +// buildCLIAtRef builds the CLI from the source tree at ref, exported with git archive so the +// working tree is not touched, and returns the path of the binary. +func buildCLIAtRef(t *testing.T, cwd, ref string) string { + src := t.TempDir() + + archive := exec.Command("git", "archive", "--format=tar", ref) + // git archive exports the subtree of its working directory, and the CLI is the whole repo. + archive.Dir = ".." + extract := exec.Command("tar", "-x", "-C", src) + pipe, err := archive.StdoutPipe() + require.NoError(t, err) + extract.Stdin = pipe + require.NoError(t, extract.Start()) + archiveErr := archive.Run() + require.NoError(t, extract.Wait()) + require.NoError(t, archiveErr) + + cli := filepath.Join(src, "databricks"+exeSuffix) + // The exported tree has no .git, so VCS stamping would fail. + RunCommand(t, []string{"go", "build", "-buildvcs=false", "-o", cli, "."}, src, []string{"GOFIPS140=" + readGOFIPS140(t, cwd)}) + return cli +} + +func gitOutput(t *testing.T, args ...string) string { + out, err := exec.Command("git", args...).Output() + require.NoError(t, err, "git %s", strings.Join(args, " ")) + return strings.TrimSpace(string(out)) +} diff --git a/acceptance/regression_unit_test.go b/acceptance/regression_unit_test.go new file mode 100644 index 00000000000..e4d121e4708 --- /dev/null +++ b/acceptance/regression_unit_test.go @@ -0,0 +1,68 @@ +package acceptance_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const sampleOutput = `=== RUN TestAccept +=== RUN TestAccept/prerequisites +--- PASS: TestAccept/prerequisites (0.10s) +=== RUN TestAccept/bundle/a +=== PAUSE TestAccept/bundle/a +=== RUN TestAccept/bundle/inv +=== RUN TestAccept/bundle/inv/DMS=true/INPUT_CONFIG=x.yml.tmpl +=== CONT TestAccept/bundle/a + acceptance_test.go:1: boom +=== CONT TestAccept/bundle/inv/DMS=true/INPUT_CONFIG=x.yml.tmpl +--- FAIL: TestAccept (1.00s) + --- PASS: TestAccept/prerequisites (0.10s) + --- FAIL: TestAccept/bundle/a (0.50s) + --- SKIP: TestAccept/bundle/inv (0.00s) + --- PASS: TestAccept/bundle/inv/DMS=true/INPUT_CONFIG=x.yml.tmpl (0.00s) +` + +func TestParseTestOutput(t *testing.T) { + results := parseTestOutput(sampleOutput) + assert.Equal(t, statusFail, results["TestAccept/bundle/a"].Status) + assert.Equal(t, []string{" acceptance_test.go:1: boom"}, results["TestAccept/bundle/a"].Output) + assert.Equal(t, statusPass, results["TestAccept/bundle/inv/DMS=true/INPUT_CONFIG=x.yml.tmpl"].Status) +} + +func TestLeafTests(t *testing.T) { + assert.Equal(t, + []string{"TestAccept/bundle/a", "TestAccept/bundle/inv/DMS=true/INPUT_CONFIG=x.yml.tmpl"}, + leafTests(parseTestOutput(sampleOutput))) +} + +func TestClassifyRegression(t *testing.T) { + refCats := []string{catRegression, catUnreleased} + tests := []struct { + branch string + refs []string + want string + }{ + {statusFail, []string{statusPass, statusPass}, catFailing}, + {statusPass, []string{statusFail, statusFail}, catRegression}, + {statusPass, []string{statusPass, statusFail}, catUnreleased}, + {statusPass, []string{statusSkip, statusSkip}, catCoverage}, + {statusPass, []string{"", statusPass}, catRegression}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, classifyRegression(tt.branch, tt.refs, refCats), "%v", tt) + } +} + +func TestRenderRegressionReport(t *testing.T) { + branch := regrColumn{Title: "branch", Results: parseTestOutput(sampleOutput)} + main := regrColumn{Title: "main (abc)", Category: catRegression, Results: map[string]*testResult{ + "TestAccept/bundle/a": {Status: statusFail, Output: []string{"main failure"}}, + }} + + report := renderRegressionReport("Selected 2", []regrColumn{branch, main}) + assert.Contains(t, report, "| test | branch | main (abc) |") + assert.Contains(t, report, "| TestAccept/bundle/a | ❌ | ❌ |") + assert.Contains(t, report, "**main (abc):**\n````\nmain failure\n````") + assert.Contains(t, report, "failing=1") +} From 3c3ef9616a2464dad6c54f635921beb46d9d86f4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 14:57:10 +0200 Subject: [PATCH 02/10] tmp: break selftest/basic to trigger regression report Co-authored-by: Isaac --- acceptance/selftest/basic/output.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/selftest/basic/output.txt b/acceptance/selftest/basic/output.txt index c43d7529752..cacf82fa334 100644 --- a/acceptance/selftest/basic/output.txt +++ b/acceptance/selftest/basic/output.txt @@ -42,3 +42,4 @@ CUSTOM_NUMBER_REGEX === Testing --version >>> [CLI] --version Databricks CLI v[CLI_VERSION] +BREAK 1 From 0e7dd4a07dbc8ee0f3952a46b1c925af6851b375 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 14:59:21 +0200 Subject: [PATCH 03/10] ci: checkout before using the local setup action in regression-report Co-authored-by: Isaac --- .github/workflows/regression-report.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/regression-report.yml b/.github/workflows/regression-report.yml index 8032ef2b585..fe47c876fe0 100644 --- a/.github/workflows/regression-report.yml +++ b/.github/workflows/regression-report.yml @@ -28,6 +28,9 @@ jobs: pull-requests: write steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup build environment uses: ./.github/actions/setup-build-environment with: From a5857d395815f27a80c11aeab296598de83590ab Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 15:09:08 +0200 Subject: [PATCH 04/10] acc: authenticate the latest release lookup when GITHUB_TOKEN is set Co-authored-by: Isaac --- .github/workflows/regression-report.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/regression-report.yml b/.github/workflows/regression-report.yml index fe47c876fe0..59be54daf11 100644 --- a/.github/workflows/regression-report.yml +++ b/.github/workflows/regression-report.yml @@ -46,6 +46,8 @@ jobs: fi - name: Run regression tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: go tool -modfile=tools/task/go.mod task regr - name: Upload report From 88bd803deb0f300deda0a3e9c46c89d0f0e849a5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 15:19:19 +0200 Subject: [PATCH 05/10] tmp: break selftest/basic differently to trigger a comment update Co-authored-by: Isaac --- acceptance/selftest/basic/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/selftest/basic/output.txt b/acceptance/selftest/basic/output.txt index cacf82fa334..965c52ba042 100644 --- a/acceptance/selftest/basic/output.txt +++ b/acceptance/selftest/basic/output.txt @@ -42,4 +42,4 @@ CUSTOM_NUMBER_REGEX === Testing --version >>> [CLI] --version Databricks CLI v[CLI_VERSION] -BREAK 1 +BREAK 2 From e3eb55d0dbb950a3adab3b95962018e04e4bf775 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 15:29:29 +0200 Subject: [PATCH 06/10] Revert "tmp: break selftest/basic differently to trigger a comment update" This reverts commit 437c0e9b74b4c61cad7d5fe12f644d31a28bc4e1. --- acceptance/selftest/basic/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/selftest/basic/output.txt b/acceptance/selftest/basic/output.txt index 965c52ba042..cacf82fa334 100644 --- a/acceptance/selftest/basic/output.txt +++ b/acceptance/selftest/basic/output.txt @@ -42,4 +42,4 @@ CUSTOM_NUMBER_REGEX === Testing --version >>> [CLI] --version Databricks CLI v[CLI_VERSION] -BREAK 2 +BREAK 1 From 07fa0cfe4929c1da23c88448d23fa1e996e4b8de Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 15:29:29 +0200 Subject: [PATCH 07/10] Revert "tmp: break selftest/basic to trigger regression report" This reverts commit e9d29da1caa96e1208d17447bad8a61ddcc65583. --- acceptance/selftest/basic/output.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/selftest/basic/output.txt b/acceptance/selftest/basic/output.txt index cacf82fa334..c43d7529752 100644 --- a/acceptance/selftest/basic/output.txt +++ b/acceptance/selftest/basic/output.txt @@ -42,4 +42,3 @@ CUSTOM_NUMBER_REGEX === Testing --version >>> [CLI] --version Databricks CLI v[CLI_VERSION] -BREAK 1 From 234d686bc8d9d5c3756ef39480a0056e633c226e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 15:38:55 +0200 Subject: [PATCH 08/10] acc: group parameters in classifyRegression (gofumpt) Co-authored-by: Isaac --- acceptance/regression_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/regression_test.go b/acceptance/regression_test.go index 32ec07d702d..69523dbd4b1 100644 --- a/acceptance/regression_test.go +++ b/acceptance/regression_test.go @@ -161,7 +161,7 @@ func isOK(status string) bool { // classifyRegression says what the status of one test on every CLI means. refs are the // non-branch columns in the order they are checked. -func classifyRegression(branch string, refs []string, refCategories []string) string { +func classifyRegression(branch string, refs, refCategories []string) string { if !isOK(branch) { return catFailing } From 62fc4bd821de75a2e8cdef122e399eaba35980b2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 15:48:14 +0200 Subject: [PATCH 09/10] ci: apply yamlfmt to regression-report workflow Co-authored-by: Isaac --- .github/workflows/regression-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression-report.yml b/.github/workflows/regression-report.yml index 59be54daf11..e3abdbec103 100644 --- a/.github/workflows/regression-report.yml +++ b/.github/workflows/regression-report.yml @@ -63,7 +63,7 @@ jobs: - name: Post or update PR comment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: | + script: |- const fs = require('fs'); const path = process.env.GITHUB_WORKSPACE + '/regression_report.md'; if (!fs.existsSync(path)) { From b064dbd50b2bfb0f73ba6349bed7227122170f1d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 19 Oct 2026 16:23:31 +0200 Subject: [PATCH 10/10] ci: delete the regression report comment when no tests are selected anymore Co-authored-by: Isaac --- .github/workflows/regression-report.yml | 33 ++++++++++++++----------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/workflows/regression-report.yml b/.github/workflows/regression-report.yml index e3abdbec103..e61733ab75e 100644 --- a/.github/workflows/regression-report.yml +++ b/.github/workflows/regression-report.yml @@ -59,21 +59,32 @@ jobs: if-no-files-found: ignore retention-days: 7 - # No report means no acceptance tests were changed, which needs no comment. - - name: Post or update PR comment + # No report means no acceptance tests are changed (anymore), so there is nothing to + # comment: drop the comment of an earlier commit instead of leaving it stale. + - name: Post, update or delete PR comment uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: |- const fs = require('fs'); const path = process.env.GITHUB_WORKSPACE + '/regression_report.md'; - if (!fs.existsSync(path)) { - core.info('No regression report, nothing to post'); - return; - } // Other comments by the same bot are left alone: ours is found by this marker. const MARKER = ''; const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number, + }); + const existing = comments.find(c => + c.user.login === 'github-actions[bot]' && c.body && c.body.startsWith(MARKER)); + + if (!fs.existsSync(path)) { + if (existing) { + await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); + } + return; + } + const sha = context.payload.pull_request.head.sha; const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; const commitUrl = `${context.serverUrl}/${owner}/${repo}/commit/${sha}`; @@ -87,16 +98,8 @@ jobs: const body = `${MARKER}\nCommit: [\`${sha.slice(0, 9)}\`](${commitUrl}) | [Workflow run](${runUrl})\n\n${report}`; - const comments = await github.paginate(github.rest.issues.listComments, { - owner, repo, issue_number: context.issue.number, - }); - const existing = comments.find(c => - c.user.login === 'github-actions[bot]' && c.body && c.body.startsWith(MARKER)); - if (existing) { await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); } else { - await github.rest.issues.createComment({ - owner, repo, issue_number: context.issue.number, body, - }); + await github.rest.issues.createComment({ owner, repo, issue_number, body }); }