Write a JUnit XML report from databricks pipelines test

acceptance/pipelines/test/arguments/out.flag-error.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="unsupported pytest argument &#34;-x&#34;: only -k and -m are supported">unsupported pytest argument &#34;-x&#34;: only -k and -m are supported</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.gate-off.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it">pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.no-wait.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="--junit-xml cannot be used with --no-wait because no results are available when the command returns">--junit-xml cannot be used with --no-wait because no results are available when the command returns</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/output.txt

@@ -31,10 +31,17 @@ update starts.
 --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume
 to read the results later. To stop the update, run 'pipelines stop'.
 
+--junit-xml writes one entry per test with its outcome and duration. An
+incomplete run, a run without tests, or a timeout adds a failing entry that
+names the cause. The report replaces the file on every exit except a flag that
+cannot be parsed. With workspace.sync_mode immutable, test names keep the path
+of the deployed snapshot, which changes with each deployment.
+
 Usage:
   databricks pipelines test [flags] [KEY] [TEST_SELECTION...]
 
 Pipeline Test Flags:
+      --junit-xml string         Write the results as a JUnit XML report to this file, replacing it.
       --no-wait                  Don't wait for the tests to complete.
       --pipeline-id string       ID of the pipeline to test, instead of a KEY.
       --pytest-arg stringArray   Argument to pass to pytest (repeatable). Only -k and -m are supported.
@@ -55,6 +62,10 @@ Global Flags:
 >>> musterr [CLI] pipelines test --pipeline-id 1234
 Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
 
+=== With the command off, an older report is still replaced
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml
+Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
+
 === pytest arguments must be passed with --pytest-arg
 >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders
 Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'
@@ -95,3 +106,11 @@ Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that p
 
 >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 
 Error: --resume "" is not an update ID; pass the update ID that pipelines test printed
+
+=== --junit-xml needs the results
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns
+
+=== An invalid flag replaces an older report
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml
+Error: unsupported pytest argument "-x": only -k and -m are supported

acceptance/pipelines/test/arguments/script

@@ -3,6 +3,10 @@ trace $CLI pipelines test --help
 title "The command is experimental and off by default"
 (unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234)
 
+title "With the command off, an older report is still replaced"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.gate-off.xml
+(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml)
+
 title "pytest arguments must be passed with --pytest-arg"
 trace musterr $CLI pipelines test --pipeline-id 1234 -k orders
 trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders
@@ -24,3 +28,10 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait
 trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m
 trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1"
 trace musterr $CLI pipelines test --pipeline-id 1234 --resume ""
+
+title "--junit-xml needs the results"
+trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+
+title "An invalid flag replaces an older report"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.flag-error.xml
+trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml

acceptance/pipelines/test/invalid-project/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-pipeline-test-invalid-project
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: ${var.missing}

acceptance/pipelines/test/invalid-project/out.invalid.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="reference does not exist: ${var.missing}">reference does not exist: ${var.missing}</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/invalid-project/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/pipelines/test/invalid-project/output.txt

@@ -0,0 +1,5 @@
+
+=== A project with errors names its first error in the report
+>>> musterr [CLI] pipelines test my_pipeline --junit-xml out.invalid.xml
+Error: reference does not exist: ${var.missing}
+

acceptance/pipelines/test/invalid-project/script

@@ -0,0 +1,2 @@
+title "A project with errors names its first error in the report"
+trace musterr $CLI pipelines test my_pipeline --junit-xml out.invalid.xml

acceptance/pipelines/test/results/failed/out.report.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="[UUID]" tests="4" failures="1" errors="1" skipped="1">
+    <testcase classname="tests.test_orders" name="test_valid_order"></testcase>
+    <testcase classname="tests.test_orders" name="test_backfill" time="2.140">
+      <failure message="AssertionError: assert 100 == 150">AssertionError: assert 100 == 150</failure>
+    </testcase>
+    <testcase classname="tests.test_orders" name="test_legacy_format">
+      <skipped></skipped>
+    </testcase>
+    <testcase classname="tests.test_orders" name="test_fixture">
+      <error message="RuntimeError: fixture orders setup failed">RuntimeError: fixture orders setup failed</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/failed/output.txt

@@ -1,6 +1,6 @@
 
 === Failed, errored and skipped tests
->>> musterr [CLI] pipelines test --pipeline-id [UUID][0]
+>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml
 Pipeline: [UUID][0]
 Update: [UUID][1]
 Running tests...
@@ -17,4 +17,5 @@ Failure in tests/test_orders.py::test_backfill
 Error in tests/test_orders.py::test_fixture
   RuntimeError: fixture orders setup failed
 
+JUnit report: out.report.xml
 Done. PASSED=1 FAILED=1 SKIPPED=1 ERROR=1 TOTAL=4

acceptance/pipelines/test/results/failed/script

@@ -1,2 +1,2 @@
 title "Failed, errored and skipped tests"
-trace musterr $CLI pipelines test --pipeline-id $pipeline_id
+trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.report.xml

acceptance/pipelines/test/results/incomplete/out.report.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="[UUID][0]" tests="3" failures="0" errors="2" skipped="0">
+    <testcase classname="tests.test_compute" name="test_small_order"></testcase>
+    <testcase classname="tests.test_compute" name="test_large_order">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase name="update [UUID][1]">
+      <error message="RUN INCOMPLETE: the update failed before all test results were received.">RUN INCOMPLETE: the update failed before all test results were received.&#xA;update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly.</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/incomplete/output.txt

@@ -1,11 +1,12 @@
 
 === Compute ends before all results are received
->>> musterr [CLI] pipelines test --pipeline-id [UUID][0]
+>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml
 Pipeline: [UUID][0]
 Update: [UUID][1]
 Running tests...
 1 of 2 PASSED tests/test_compute.py::test_small_order .......................... [PASSED]
 
 RUN INCOMPLETE: the update failed before all test results were received.
+JUnit report: out.report.xml
 Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2
 Error: update [UUID][1] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly.

acceptance/pipelines/test/results/incomplete/script

@@ -1,2 +1,2 @@
 title "Compute ends before all results are received"
-trace musterr $CLI pipelines test --pipeline-id $pipeline_id
+trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.report.xml

acceptance/pipelines/test/results/passed/out.report.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="[UUID]" tests="2" failures="0" errors="0" skipped="0">
+    <testcase classname="tests.test_orders" name="test_valid_order" time="1.840"></testcase>
+    <testcase classname="tests.test_orders" name="test_backfill" time="1.000"></testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/passed/output.txt

@@ -1,12 +1,13 @@
 
 === All tests pass
->>> [CLI] pipelines test --pipeline-id [UUID][0]
+>>> [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml
 Pipeline: [UUID][0]
 Update: [UUID][1]
 Running tests...
 1 of 2 PASSED tests/test_orders.py::test_valid_order ........................... [PASSED in 1.84s]
 2 of 2 PASSED tests/test_orders.py::test_backfill .............................. [PASSED in 1.00s]
 
+JUnit report: out.report.xml
 Done. PASSED=2 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=2
 
 >>> print_requests.py --get //events

acceptance/pipelines/test/results/passed/script

@@ -1,3 +1,3 @@
 title "All tests pass"
-trace $CLI pipelines test --pipeline-id $pipeline_id
+trace $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.report.xml
 trace print_requests.py --get //events

acceptance/pipelines/test/run-control/canceled/out.report.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="[UUID][0]" tests="2" failures="0" errors="2" skipped="0">
+    <testcase classname="tests.test_orders" name="test_slow">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase name="update [UUID][1]">
+      <error message="RUN INCOMPLETE: the update was canceled before all test results were received.">RUN INCOMPLETE: the update was canceled before all test results were received.&#xA;update [UUID][1] is CANCELED</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/run-control/canceled/output.txt

@@ -1,9 +1,10 @@
 
 === A canceled update is an incomplete run
->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1]
+>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] --junit-xml out.report.xml
 Pipeline: [UUID][0]
 Update: [UUID][1]
 
 RUN INCOMPLETE: the update was canceled before all test results were received.
+JUnit report: out.report.xml
 Done. PASSED=0 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=1
 Error: update [UUID][1] is CANCELED

acceptance/pipelines/test/run-control/canceled/script

@@ -1,2 +1,2 @@
 title "A canceled update is an incomplete run"
-trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1
+trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 --junit-xml out.report.xml

acceptance/pipelines/test/run-control/timeout/out.report.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="[UUID][0]" tests="2" failures="0" errors="2" skipped="0">
+    <testcase classname="tests.test_orders" name="test_slow">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase name="update [UUID][1]">
+      <error message="stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml --resume [UUID][1]">stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml --resume [UUID][1]</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/run-control/timeout/output.txt

@@ -1,7 +1,8 @@
 
 === --timeout stops waiting but not the update, and prints the command that reads the results
->>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] --timeout 1s
+>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --resume [UUID][1] --timeout 1s --junit-xml out.report.xml
 Pipeline: [UUID][0]
 Update: [UUID][1]
 Running tests...
-Error: stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --resume [UUID][1]
+JUnit report: out.report.xml
+Error: stopped waiting for update [UUID][1] after 1s; the update keeps running, to get its results run: databricks pipelines test --pipeline-id [UUID][0] --junit-xml out.report.xml --resume [UUID][1]

acceptance/pipelines/test/run-control/timeout/script

@@ -1,2 +1,2 @@
 title "--timeout stops waiting but not the update, and prints the command that reads the results"
-trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 --timeout 1s
+trace musterr $CLI pipelines test --pipeline-id $pipeline_id --resume 00000000-0000-0000-0000-0000000000a1 --timeout 1s --junit-xml out.report.xml

acceptance/pipelines/test/submit-error/out.start-failed.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="[UUID][0]" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="An active update &#39;[UUID][1]&#39; already exists for pipeline &#39;[UUID][2]&#39;.">An active update &#39;[UUID][1]&#39; already exists for pipeline &#39;[UUID][2]&#39;.</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/submit-error/output.txt

@@ -1,6 +1,6 @@
 
-=== A submit error from the service is shown as is, and no update ID is printed
->>> musterr [CLI] pipelines test --pipeline-id [UUID][0]
+=== A submit error from the service is shown as is, no update ID is printed, and an older report is replaced
+>>> musterr [CLI] pipelines test --pipeline-id [UUID][0] --junit-xml out.start-failed.xml
 Error: An active update '[UUID][1]' already exists for pipeline '[UUID][2]'.
 
 >>> print_requests.py //updates

acceptance/pipelines/test/submit-error/script

@@ -1,6 +1,7 @@
 pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id)
 rm "$OUT_REQUESTS"
 
-title "A submit error from the service is shown as is, and no update ID is printed"
-trace musterr $CLI pipelines test --pipeline-id $pipeline_id
+title "A submit error from the service is shown as is, no update ID is printed, and an older report is replaced"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.start-failed.xml
+trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.start-failed.xml
 trace print_requests.py //updates