diff --git a/acceptance/pipelines/test/arguments/out.flag-error.xml b/acceptance/pipelines/test/arguments/out.flag-error.xml new file mode 100644 index 00000000000..475a397f5fe --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.flag-error.xml @@ -0,0 +1,8 @@ + + + + + unsupported pytest argument "-x": only -k and -m are supported + + + diff --git a/acceptance/pipelines/test/arguments/out.gate-off.xml b/acceptance/pipelines/test/arguments/out.gate-off.xml new file mode 100644 index 00000000000..efe909a6fa0 --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.gate-off.xml @@ -0,0 +1,8 @@ + + + + + pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it + + + diff --git a/acceptance/pipelines/test/arguments/out.no-wait.xml b/acceptance/pipelines/test/arguments/out.no-wait.xml new file mode 100644 index 00000000000..5f025801a52 --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.no-wait.xml @@ -0,0 +1,8 @@ + + + + + --junit-xml cannot be used with --no-wait because no results are available when the command returns + + + diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index b64826fccbd..d03f8618d22 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -31,10 +31,17 @@ update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume to read the results later. To stop the update, run 'pipelines stop'. +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause. The report replaces the file on every exit except a flag that +cannot be parsed. With workspace.sync_mode immutable, test names keep the path +of the deployed snapshot, which changes with each deployment. + Usage: databricks pipelines test [flags] [KEY] [TEST_SELECTION...] Pipeline Test Flags: + --junit-xml string Write the results as a JUnit XML report to this file, replacing it. --no-wait Don't wait for the tests to complete. --pipeline-id string ID of the pipeline to test, instead of a KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. @@ -55,6 +62,10 @@ Global Flags: >>> musterr [CLI] pipelines test --pipeline-id 1234 Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it +=== With the command off, an older report is still replaced +>>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml +Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it + === pytest arguments must be passed with --pytest-arg >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' @@ -95,3 +106,11 @@ Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that p >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume Error: --resume "" is not an update ID; pass the update ID that pipelines test printed + +=== --junit-xml needs the results +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml +Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns + +=== An invalid flag replaces an older report +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml +Error: unsupported pytest argument "-x": only -k and -m are supported diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index 62ed32b4a1b..ceb54ec432e 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -3,6 +3,10 @@ trace $CLI pipelines test --help title "The command is experimental and off by default" (unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234) +title "With the command off, an older report is still replaced" +echo '' > out.gate-off.xml +(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml) + title "pytest arguments must be passed with --pytest-arg" trace musterr $CLI pipelines test --pipeline-id 1234 -k orders trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders @@ -24,3 +28,10 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" trace musterr $CLI pipelines test --pipeline-id 1234 --resume "" + +title "--junit-xml needs the results" +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml + +title "An invalid flag replaces an older report" +echo '' > out.flag-error.xml +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml diff --git a/acceptance/pipelines/test/invalid-project/databricks.yml b/acceptance/pipelines/test/invalid-project/databricks.yml new file mode 100644 index 00000000000..b23fa23f1a1 --- /dev/null +++ b/acceptance/pipelines/test/invalid-project/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-pipeline-test-invalid-project + +resources: + pipelines: + my_pipeline: + name: ${var.missing} diff --git a/acceptance/pipelines/test/invalid-project/out.invalid.xml b/acceptance/pipelines/test/invalid-project/out.invalid.xml new file mode 100644 index 00000000000..ff48bd9dac9 --- /dev/null +++ b/acceptance/pipelines/test/invalid-project/out.invalid.xml @@ -0,0 +1,8 @@ + + + + + reference does not exist: ${var.missing} + + + diff --git a/acceptance/pipelines/test/invalid-project/out.test.toml b/acceptance/pipelines/test/invalid-project/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/invalid-project/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/invalid-project/output.txt b/acceptance/pipelines/test/invalid-project/output.txt new file mode 100644 index 00000000000..5b8058bab75 --- /dev/null +++ b/acceptance/pipelines/test/invalid-project/output.txt @@ -0,0 +1,5 @@ + +=== A project with errors names its first error in the report +>>> musterr [CLI] pipelines test my_pipeline --junit-xml out.invalid.xml +Error: reference does not exist: ${var.missing} + diff --git a/acceptance/pipelines/test/invalid-project/script b/acceptance/pipelines/test/invalid-project/script new file mode 100644 index 00000000000..95bce3496ca --- /dev/null +++ b/acceptance/pipelines/test/invalid-project/script @@ -0,0 +1,2 @@ +title "A project with errors names its first error in the report" +trace musterr $CLI pipelines test my_pipeline --junit-xml out.invalid.xml diff --git a/acceptance/pipelines/test/results/failed/out.report.xml b/acceptance/pipelines/test/results/failed/out.report.xml new file mode 100644 index 00000000000..ec831adc1a1 --- /dev/null +++ b/acceptance/pipelines/test/results/failed/out.report.xml @@ -0,0 +1,15 @@ + + + + + + AssertionError: assert 100 == 150 + + + + + + RuntimeError: fixture orders setup failed + + + diff --git a/acceptance/pipelines/test/results/failed/output.txt b/acceptance/pipelines/test/results/failed/output.txt index 72c51649bc1..2a9924f3a89 100644 --- a/acceptance/pipelines/test/results/failed/output.txt +++ b/acceptance/pipelines/test/results/failed/output.txt @@ -1,6 +1,6 @@ === Failed, errored and skipped tests ->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml Pipeline: [UUID][0] Update: [UUID][1] Running tests... @@ -17,4 +17,5 @@ Failure in tests/test_orders.py::test_backfill Error in tests/test_orders.py::test_fixture RuntimeError: fixture orders setup failed +JUnit report: out.report.xml Done. PASSED=1 FAILED=1 SKIPPED=1 ERROR=1 TOTAL=4 diff --git a/acceptance/pipelines/test/results/failed/script b/acceptance/pipelines/test/results/failed/script index 4942d54e76f..4dd938350c0 100644 --- a/acceptance/pipelines/test/results/failed/script +++ b/acceptance/pipelines/test/results/failed/script @@ -1,2 +1,2 @@ title "Failed, errored and skipped tests" -trace musterr $CLI pipelines test --pipeline-id $pipeline_id +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.report.xml diff --git a/acceptance/pipelines/test/results/incomplete/out.report.xml b/acceptance/pipelines/test/results/incomplete/out.report.xml new file mode 100644 index 00000000000..8bd8097922b --- /dev/null +++ b/acceptance/pipelines/test/results/incomplete/out.report.xml @@ -0,0 +1,12 @@ + + + + + + + + + RUN INCOMPLETE: the update failed before all test results were received. update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. + + + diff --git a/acceptance/pipelines/test/results/incomplete/output.txt b/acceptance/pipelines/test/results/incomplete/output.txt index 9bc31359298..ce9fddb75f9 100644 --- a/acceptance/pipelines/test/results/incomplete/output.txt +++ b/acceptance/pipelines/test/results/incomplete/output.txt @@ -1,11 +1,12 @@ === Compute ends before all results are received ->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml Pipeline: [UUID][0] Update: [UUID][1] Running tests... 1 of 2 PASSED tests/test_compute.py::test_small_order .......................... [PASSED] RUN INCOMPLETE: the update failed before all test results were received. +JUnit report: out.report.xml Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 Error: update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/incomplete/script b/acceptance/pipelines/test/results/incomplete/script index 5a3bdb6836d..b28d2bdee5e 100644 --- a/acceptance/pipelines/test/results/incomplete/script +++ b/acceptance/pipelines/test/results/incomplete/script @@ -1,2 +1,2 @@ title "Compute ends before all results are received" -trace musterr $CLI pipelines test --pipeline-id $pipeline_id +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.report.xml diff --git a/acceptance/pipelines/test/results/passed/out.report.xml b/acceptance/pipelines/test/results/passed/out.report.xml new file mode 100644 index 00000000000..2ece15eb5ea --- /dev/null +++ b/acceptance/pipelines/test/results/passed/out.report.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/acceptance/pipelines/test/results/passed/output.txt b/acceptance/pipelines/test/results/passed/output.txt index 2db22a1cd0a..9a5bafe8474 100644 --- a/acceptance/pipelines/test/results/passed/output.txt +++ b/acceptance/pipelines/test/results/passed/output.txt @@ -1,12 +1,13 @@ === All tests pass ->>> [CLI] pipelines test --pipeline-id [UUID][0] +>>> [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml Pipeline: [UUID][0] Update: [UUID][1] Running tests... 1 of 2 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED in 1.84s] 2 of 2 PASSED tests/test_orders.py::test_backfill .............................. [PASSED in 1.00s] +JUnit report: out.report.xml Done. PASSED=2 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2 >>> print_requests.py --get //events diff --git a/acceptance/pipelines/test/results/passed/script b/acceptance/pipelines/test/results/passed/script index 2d87eb30ab9..d7132c4dbc3 100644 --- a/acceptance/pipelines/test/results/passed/script +++ b/acceptance/pipelines/test/results/passed/script @@ -1,3 +1,3 @@ title "All tests pass" -trace $CLI pipelines test --pipeline-id $pipeline_id +trace $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.report.xml trace print_requests.py --get //events diff --git a/acceptance/pipelines/test/run-control/canceled/out.report.xml b/acceptance/pipelines/test/run-control/canceled/out.report.xml new file mode 100644 index 00000000000..3743b6239a3 --- /dev/null +++ b/acceptance/pipelines/test/run-control/canceled/out.report.xml @@ -0,0 +1,11 @@ + + + + + + + + RUN INCOMPLETE: the update was canceled before all test results were received. update [UUID][1] is CANCELED + + + diff --git a/acceptance/pipelines/test/run-control/canceled/output.txt b/acceptance/pipelines/test/run-control/canceled/output.txt index 72bd75f112a..a21b056a361 100644 --- a/acceptance/pipelines/test/run-control/canceled/output.txt +++ b/acceptance/pipelines/test/run-control/canceled/output.txt @@ -1,9 +1,10 @@ === A canceled update is an incomplete run ->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] --junit-xml out.report.xml Pipeline: [UUID][0] Update: [UUID][1] RUN INCOMPLETE: the update was canceled before all test results were received. +JUnit report: out.report.xml Done. PASSED=0 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=1 Error: update [UUID][1] is CANCELED diff --git a/acceptance/pipelines/test/run-control/canceled/script b/acceptance/pipelines/test/run-control/canceled/script index cfb7d487ab5..beb99f5b049 100644 --- a/acceptance/pipelines/test/run-control/canceled/script +++ b/acceptance/pipelines/test/run-control/canceled/script @@ -1,2 +1,2 @@ title "A canceled update is an incomplete run" -trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 --junit-xml out.report.xml diff --git a/acceptance/pipelines/test/run-control/timeout/out.report.xml b/acceptance/pipelines/test/run-control/timeout/out.report.xml new file mode 100644 index 00000000000..ac0d802ec6f --- /dev/null +++ b/acceptance/pipelines/test/run-control/timeout/out.report.xml @@ -0,0 +1,11 @@ + + + + + + + + stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml --resume [UUID][1] + + + diff --git a/acceptance/pipelines/test/run-control/timeout/output.txt b/acceptance/pipelines/test/run-control/timeout/output.txt index 5b9ea594437..2be2f36818e 100644 --- a/acceptance/pipelines/test/run-control/timeout/output.txt +++ b/acceptance/pipelines/test/run-control/timeout/output.txt @@ -1,7 +1,8 @@ === --timeout stops waiting but not the update, and prints the command that reads the results ->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] --timeout 1s +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] --timeout 1s --junit-xml out.report.xml Pipeline: [UUID][0] Update: [UUID][1] Running tests... -Error: stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --resume [UUID][1] +JUnit report: out.report.xml +Error: stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml --resume [UUID][1] diff --git a/acceptance/pipelines/test/run-control/timeout/script b/acceptance/pipelines/test/run-control/timeout/script index e0703287af5..b95c8689fdd 100644 --- a/acceptance/pipelines/test/run-control/timeout/script +++ b/acceptance/pipelines/test/run-control/timeout/script @@ -1,2 +1,2 @@ title "--timeout stops waiting but not the update, and prints the command that reads the results" -trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 --timeout 1s +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 --timeout 1s --junit-xml out.report.xml diff --git a/acceptance/pipelines/test/submit-error/out.start-failed.xml b/acceptance/pipelines/test/submit-error/out.start-failed.xml new file mode 100644 index 00000000000..be5603014cf --- /dev/null +++ b/acceptance/pipelines/test/submit-error/out.start-failed.xml @@ -0,0 +1,8 @@ + + + + + An active update '[UUID][1]' already exists for pipeline '[UUID][2]'. + + + diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 52690c65e82..909f46a871b 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -1,6 +1,6 @@ -=== A submit error from the service is shown as is, and no update ID is printed ->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] +=== A submit error from the service is shown as is, no update ID is printed, and an older report is replaced +>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.start-failed.xml Error: An active update '[UUID][1]' already exists for pipeline '[UUID][2]'. >>> print_requests.py //updates diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index 405aa9d1473..1035f977d33 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -1,6 +1,7 @@ pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) rm "$OUT_REQUESTS" -title "A submit error from the service is shown as is, and no update ID is printed" -trace musterr $CLI pipelines test --pipeline-id $pipeline_id +title "A submit error from the service is shown as is, no update ID is printed, and an older report is replaced" +echo '' > out.start-failed.xml +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.start-failed.xml trace print_requests.py //updates diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index ebecee1f3fb..a3547f494fb 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -384,6 +384,10 @@ func (t testTarget) resumeCommand(cmd *cobra.Command, updateID string) string { if profile := cmd.Flag("profile").Value.String(); profile != "" { parts = append(parts, "--profile", shellquote.BashArg(profile)) } + // Reading the results again replaces the partial report. + if flag := cmd.Flag("junit-xml"); flag != nil && flag.Value.String() != "" { + parts = append(parts, "--junit-xml", shellquote.BashArg(flag.Value.String())) + } // --resume loads the bundle again, so it needs the same variable values. if flag := cmd.Flag("var"); flag != nil { if values, ok := flag.Value.(pflag.SliceValue); ok { @@ -509,6 +513,89 @@ func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun return fmt.Errorf("update %s is %s", updateID, state) } +// testRunProblem describes why a run that ended did not pass for a reason other than +// failed tests, or returns "" otherwise. +func testRunProblem(run *testRun, incompleteReason string, runErr error) string { + var lines []string + switch { + case incompleteReason != "": + lines = append(lines, fmt.Sprintf("RUN INCOMPLETE: %s.", incompleteReason)) + case run.summaries[0].Total == 0: + lines = append(lines, "No tests were collected.") + } + if runErr != nil && !errors.Is(runErr, root.ErrAlreadyPrinted) { + lines = append(lines, runErr.Error()) + } + return strings.Join(lines, "\n") +} + +// testFlags holds the flags of pipelines test. +type testFlags struct { + pipelineID string + pytestArgs []string + noWait bool + timeout time.Duration + resumeUpdateID string + junitPath string +} + +// checkTestFlags rejects the flag combinations that pipelines test does not support, before any API call. +func checkTestFlags(cmd *cobra.Command, args []string, f testFlags) error { + if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { + return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) + } + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) + } + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) + } + if err := validatePytestArgs(f.pytestArgs); err != nil { + return err + } + if f.timeout < 0 { + return errors.New("--timeout must not be negative") + } + if f.noWait && f.timeout > 0 { + return errors.New("--timeout cannot be used with --no-wait") + } + if f.noWait && f.junitPath != "" { + return errors.New("--junit-xml cannot be used with --no-wait because no results are available when the command returns") + } + // An empty --resume, such as from an unset shell variable, must not start a new run. + if cmd.Flags().Changed("resume") { + selectionCount := len(args) + if f.pipelineID == "" { + selectionCount = max(len(args)-1, 0) + } + switch { + case f.noWait: + return errors.New("--resume cannot be used with --no-wait") + case len(f.pytestArgs) > 0: + return errors.New("--resume cannot be used with --pytest-arg") + case selectionCount > 0: + return errors.New("--resume cannot be used with test selections") + } + if _, err := uuid.Parse(f.resumeUpdateID); err != nil { + return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", f.resumeUpdateID) + } + } + return nil +} + +// resolveTestTarget identifies the pipeline by --pipeline-id or, in a bundle, by its KEY. +func resolveTestTarget(cmd *cobra.Command, args []string, f testFlags, resume bool) (testTarget, error) { + if f.pipelineID != "" { + id, err := uuid.Parse(f.pipelineID) + if err != nil { + return testTarget{}, fmt.Errorf("--pipeline-id %q is not a pipeline ID", f.pipelineID) + } + return resolveTestTargetByID(cmd, args, id.String(), !f.noWait) + } + cmd.SetContext(logdiag.InitContext(cmd.Context())) + return resolveTestTargetInBundle(cmd, args, resume) +} + // notPipelineKeyError explains that the first argument is not a pipeline KEY, names the bundle's // pipelines, and shows the command that passes the arguments as test selections of the first one. func notPipelineKeyError(cmd *cobra.Command, b *bundle.Bundle, key string, args []string) error { @@ -556,21 +643,24 @@ skipped succeeds; otherwise it exits with code 1. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume -to read the results later. To stop the update, run 'pipelines stop'.`, +to read the results later. To stop the update, run 'pipelines stop'. + +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause. The report replaces the file on every exit except a flag that +cannot be parsed. With workspace.sync_mode immutable, test names keep the path +of the deployed snapshot, which changes with each deployment.`, } - var pipelineID string - var pytestArgs []string - var noWait bool - var timeout time.Duration - var resumeUpdateID string + var f testFlags testGroup := cmdgroup.NewFlagGroup("Pipeline Test") - testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a KEY.") - testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") - testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Don't wait for the tests to complete.") - testGroup.FlagSet().DurationVar(&timeout, "timeout", 0, "Stop waiting for results after this duration, for example 30m. The update keeps running.") - testGroup.FlagSet().StringVar(&resumeUpdateID, "resume", "", "Read the results of an existing test update instead of starting one.") + testGroup.FlagSet().StringVar(&f.pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a KEY.") + testGroup.FlagSet().StringArrayVar(&f.pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + testGroup.FlagSet().BoolVar(&f.noWait, "no-wait", false, "Don't wait for the tests to complete.") + testGroup.FlagSet().DurationVar(&f.timeout, "timeout", 0, "Stop waiting for results after this duration, for example 30m. The update keeps running.") + testGroup.FlagSet().StringVar(&f.resumeUpdateID, "resume", "", "Read the results of an existing test update instead of starting one.") + testGroup.FlagSet().StringVar(&f.junitPath, "junit-xml", "", "Write the results as a JUnit XML report to this file, replacing it.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -578,61 +668,24 @@ to read the results later. To stop the update, run 'pipelines stop'.`, cmd.SetFlagErrorFunc(testFlagErrorFunc) cmd.RunE = func(cmd *cobra.Command, args []string) error { - if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { - return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) - } - if cmd.ArgsLenAtDash() >= 0 { - return errors.New("the -- separator is not supported: " + pytestArgHint) + // Every exit writes the report, so that an older report never survives. + report := &testReport{path: f.junitPath, out: cmd.OutOrStdout(), suiteName: f.pipelineID} + if report.suiteName == "" && len(args) > 0 { + report.suiteName = args[0] } - if root.OutputType(cmd) != flags.OutputText { - return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) + if err := checkTestFlags(cmd, args, f); err != nil { + return report.failed(cmd.Context(), err) } - if err := validatePytestArgs(pytestArgs); err != nil { - return err - } - if timeout < 0 { - return errors.New("--timeout must not be negative") - } - if noWait && timeout > 0 { - return errors.New("--timeout cannot be used with --no-wait") - } - // An empty --resume, such as from an unset shell variable, must not start a new run. resume := cmd.Flags().Changed("resume") - if resume { - selectionCount := len(args) - if pipelineID == "" { - selectionCount = max(len(args)-1, 0) - } - switch { - case noWait: - return errors.New("--resume cannot be used with --no-wait") - case len(pytestArgs) > 0: - return errors.New("--resume cannot be used with --pytest-arg") - case selectionCount > 0: - return errors.New("--resume cannot be used with test selections") - } - if _, err := uuid.Parse(resumeUpdateID); err != nil { - return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) - } + target, err := resolveTestTarget(cmd, args, f, resume) + if err != nil { + return report.failed(cmd.Context(), err) } - var target testTarget - if pipelineID != "" { - id, err := uuid.Parse(pipelineID) - if err != nil { - return fmt.Errorf("--pipeline-id %q is not a pipeline ID", pipelineID) - } - target, err = resolveTestTargetByID(cmd, args, id.String(), !noWait) - if err != nil { - return err - } - } else { - cmd.SetContext(logdiag.InitContext(cmd.Context())) - var err error - target, err = resolveTestTargetInBundle(cmd, args, resume) - if err != nil { - return err - } + report.suiteName = target.key + if report.suiteName == "" { + report.suiteName = target.pipelineID } + report.projectFolder = target.projectFolder ctx := cmd.Context() // The handler is installed before the update starts, but only the wait listens to it: a @@ -641,20 +694,19 @@ to read the results later. To stop the update, run 'pipelines stop'.`, waitCtx, stop := notifyTestInterrupt(ctx) defer stop() - updateID := resumeUpdateID + updateID := f.resumeUpdateID if !resume { - req := newStartTestUpdateRequest(target.selections, pytestArgs) - var err error + req := newStartTestUpdateRequest(target.selections, f.pytestArgs) updateID, err = startTestUpdate(ctx, target.w, target.pipelineID, req) if err != nil { - return err + return report.failed(ctx, err) } } else if err := checkTestUpdate(ctx, target.w, target.pipelineID, updateID); err != nil { - return err + return report.failed(ctx, err) } out := cmd.OutOrStdout() target.printHeader(out, updateID) - if noWait { + if f.noWait { return nil } @@ -663,25 +715,29 @@ to read the results later. To stop the update, run 'pipelines stop'.`, // The timeout is not a context deadline: the SDK rate limiter fails a request that would wait // past the deadline before the context expires, with an error that does not name the timeout. var timedOut <-chan time.Time - if timeout > 0 { - timedOut = time.After(timeout) + if f.timeout > 0 { + timedOut = time.After(f.timeout) } state, err := waitForTestUpdate(waitCtx, target.w, target.pipelineID, updateID, run, printer, timedOut) if err != nil { isTimeout := errors.Is(err, errTestWaitTimeout) if !isTimeout && waitCtx.Err() == nil { - return err + return report.finish(updateID, run, err.Error(), err) } resumeCmd := target.resumeCommand(cmd, updateID) + stopErr := fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resumeCmd) if isTimeout { - return fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resumeCmd) + stopErr = fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, f.timeout, resumeCmd) } - return fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resumeCmd) + return report.finish(updateID, run, stopErr.Error(), stopErr) } incompleteReason := run.incompleteReason(state) renderTestResult(out, run, incompleteReason, target.shortTestID) + runErr := testRunError(updateID, state, run, incompleteReason) + // The report line comes before the summary, so that the summary stays the last line. + err = report.finish(updateID, run, testRunProblem(run, incompleteReason, runErr), runErr) renderTestSummaryLine(out, run, incompleteReason) - return testRunError(updateID, state, run, incompleteReason) + return err } return cmd diff --git a/cmd/pipelines/test_junit.go b/cmd/pipelines/test_junit.go index 83500edb232..d452281f355 100644 --- a/cmd/pipelines/test_junit.go +++ b/cmd/pipelines/test_junit.go @@ -1,6 +1,7 @@ package pipelines import ( + "context" "encoding/xml" "errors" "fmt" @@ -10,6 +11,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/atomicfile" + "github.com/databricks/cli/libs/logdiag" ) type junitTestSuites struct { @@ -144,6 +146,37 @@ func (r *testReport) finish(updateID string, run *testRun, problem string, runEr return runErr } +// failed writes a report with one failing entry for an error that ended the command before the +// update could be read, so that an older report does not survive. A write error is joined with +// err, as in finish. +func (r *testReport) failed(ctx context.Context, err error) error { + if r.path == "" { + return err + } + message := err.Error() + printed := errors.Is(err, root.ErrAlreadyPrinted) + if printed { + // The error was printed as a diagnostic. Name it instead of the sentinel's text. + message = "the project has errors, see the command output" + if logdiag.IsSetup(ctx) && logdiag.GetFirstErrorSummary(ctx) != "" { + message = logdiag.GetFirstErrorSummary(ctx) + } + } + report := junitTestSuites{Suites: []junitTestSuite{{ + Name: r.suiteName, + Tests: 1, + Errors: 1, + Cases: []junitTestCase{{Name: "pipelines test", Error: newJUnitResult(message)}}, + }}} + if writeErr := writeJUnitReport(r.path, report); writeErr != nil { + if printed { + return writeErr + } + return errors.Join(err, writeErr) + } + return err +} + // writeJUnitReport replaces the file at path with the report. func writeJUnitReport(path string, report junitTestSuites) error { data, err := xml.MarshalIndent(report, "", " ") diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index d01259b56f7..23f2f229d36 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -377,3 +377,20 @@ func TestCheckTestUpdateWithoutTestOnly(t *testing.T) { assert.NoError(t, checkTestUpdate(t.Context(), w, "p1", updateID)) } + +func TestTestRunProblem(t *testing.T) { + ev := newTestEvents(t) + passed := newTestRun() + passed.add(ev.testCase("t::a", "PASSED", "")) + passed.add(ev.summary(1, 1, 0, 0, 0)) + assert.Empty(t, testRunProblem(passed, "", nil)) + assert.Empty(t, testRunProblem(passed, "", root.ErrAlreadyPrinted)) + assert.Equal(t, "update u1 is COMPLETED: result delivery failed", testRunProblem(passed, "", errors.New("update u1 is COMPLETED: result delivery failed"))) + + empty := newTestRun() + empty.add(ev.summary(0, 0, 0, 0, 0)) + assert.Equal(t, "No tests were collected.", testRunProblem(empty, "", root.ErrAlreadyPrinted)) + + assert.Equal(t, "RUN INCOMPLETE: the update was canceled.\nupdate u1 is CANCELED", + testRunProblem(newTestRun(), "the update was canceled", errors.New("update u1 is CANCELED"))) +}