Run pipelines test against the pipeline's testing.sources

acceptance/pipelines/databricks-cli-help/output.txt

@@ -30,6 +30,7 @@ Main Commands
   run                   Run a pipeline
   show                  Preview a table's columns and sample rows
   stop                  Stop a pipeline
+  test                  *Preview* Run the tests of a pipeline
 
 Management Commands
   apply-environment     *Public Preview* Apply the latest environment to the pipeline.

acceptance/pipelines/test/arguments/out.flag-error.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="unsupported pytest argument &#34;-x&#34;: only -k and -m are supported">unsupported pytest argument &#34;-x&#34;: only -k and -m are supported</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.gate-off.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it">pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.no-wait.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="--junit-xml cannot be used with --no-wait because no results are available when the command returns">--junit-xml cannot be used with --no-wait because no results are available when the command returns</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/pipelines/test/arguments/output.txt

@@ -0,0 +1,116 @@
+
+>>> [CLI] pipelines test --help
+Run the tests of a pipeline in a test-only update.
+
+This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it.
+
+KEY is the unique name of the pipeline, as defined in its YAML file.
+It is required when test selections are given. If there is only one pipeline
+in the project and no arguments are given, it is selected automatically.
+Outside a project, identify the pipeline with --pipeline-id; all positional
+arguments are then test selections.
+
+TEST_SELECTION is a test directory, file, or pytest test ID, for example
+tests/test_orders.py::test_backfill. In a project, a relative path is a local
+path, relative to the working directory, and is converted to its deployed
+workspace path. With --pipeline-id, a relative path is relative to the
+pipeline's root_path. Paths that start with / are workspace paths. Without a
+selection, all of the pipeline's tests run.
+
+Each --pytest-arg passes one argument to pytest, in the order given. Only -k
+and -m are supported, each followed by its value in the next --pytest-arg, for
+example:
+
+  databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'
+
+The command waits for the results. It exits with code 0 only if at least one
+test was collected and none failed or errored, so a run whose tests were all
+skipped succeeds; otherwise it exits with code 1. --no-wait returns after the
+update starts.
+
+--timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume
+to read the results later. To stop the update, run 'pipelines stop'.
+
+--junit-xml writes one entry per test with its outcome and duration. An
+incomplete run, a run without tests, or a timeout adds a failing entry that
+names the cause. The report replaces the file on every exit except a flag that
+cannot be parsed. With workspace.sync_mode immutable, test names keep the path
+of the deployed snapshot, which changes with each deployment.
+
+Usage:
+  databricks pipelines test [flags] [KEY] [TEST_SELECTION...]
+
+Pipeline Test Flags:
+      --junit-xml string         Write the results as a JUnit XML report to this file, replacing it.
+      --no-wait                  Don't wait for the tests to complete.
+      --pipeline-id string       ID of the pipeline to test, instead of a KEY.
+      --pytest-arg stringArray   Argument to pass to pytest (repeatable). Only -k and -m are supported.
+      --resume string            Read the results of an existing test update instead of starting one.
+      --timeout duration         Stop waiting for results after this duration, for example 30m. The update keeps running.
+
+Flags:
+  -h, --help   help for test
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in project config. Example: --var="foo=bar"
+
+=== The command is experimental and off by default
+>>> musterr [CLI] pipelines test --pipeline-id 1234
+Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
+
+=== With the command off, an older report is still replaced
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml
+Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
+
+=== pytest arguments must be passed with --pytest-arg
+>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders
+Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 -- -k orders
+Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'
+
+=== Invalid --pytest-arg values fail before any request
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x
+Error: unsupported pytest argument "-x": only -k and -m are supported
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=
+Error: --pytest-arg requires a non-empty value
+
+=== The pipeline ID must be a pipeline ID
+>>> musterr [CLI] pipelines test --pipeline-id 1234
+Error: --pipeline-id "1234" is not a pipeline ID
+
+=== JSON output is not supported
+>>> musterr [CLI] pipelines test --pipeline-id 1234 -o json
+Error: --output json is not supported by pipelines test
+
+=== Incompatible run-control flags
+>>> musterr [CLI] pipelines test --pipeline-id 1234 tests --resume 5678
+Error: --resume cannot be used with test selections
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke
+Error: --resume cannot be used with --pytest-arg
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678 --no-wait
+Error: --resume cannot be used with --no-wait
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --timeout 10m
+Error: --timeout cannot be used with --no-wait
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 5678' OR '1'='1
+Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that pipelines test printed
+
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 
+Error: --resume "" is not an update ID; pass the update ID that pipelines test printed
+
+=== --junit-xml needs the results
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns
+
+=== An invalid flag replaces an older report
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml
+Error: unsupported pytest argument "-x": only -k and -m are supported

acceptance/pipelines/test/arguments/script

@@ -0,0 +1,37 @@
+trace $CLI pipelines test --help
+
+title "The command is experimental and off by default"
+(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234)
+
+title "With the command off, an older report is still replaced"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.gate-off.xml
+(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml)
+
+title "pytest arguments must be passed with --pytest-arg"
+trace musterr $CLI pipelines test --pipeline-id 1234 -k orders
+trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders
+
+title "Invalid --pytest-arg values fail before any request"
+trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x
+trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=""
+
+title "The pipeline ID must be a pipeline ID"
+trace musterr $CLI pipelines test --pipeline-id 1234
+
+title "JSON output is not supported"
+trace musterr $CLI pipelines test --pipeline-id 1234 -o json
+
+title "Incompatible run-control flags"
+trace musterr $CLI pipelines test --pipeline-id 1234 tests --resume 5678
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --pytest-arg=-m --pytest-arg=smoke
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait
+trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1"
+trace musterr $CLI pipelines test --pipeline-id 1234 --resume ""
+
+title "--junit-xml needs the results"
+trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+
+title "An invalid flag replaces an older report"
+echo '<testsuites><testsuite name="old" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.flag-error.xml
+trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml

acceptance/pipelines/test/immutable/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: test-pipeline-test-immutable
+
+workspace:
+  sync_mode: immutable
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-test-immutable
+      libraries:
+        - file:
+            path: pipeline_file.py
+      testing:
+        sources:
+          - glob:
+              include: ./tests/**

acceptance/pipelines/test/immutable/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/pipelines/test/immutable/output.txt

@@ -0,0 +1,53 @@
+
+>>> [CLI] pipelines deploy
+Created internal_immutable_snapshots.immutable
+Created pipelines.my_pipeline
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID]
+
+=== Local test paths are not supported with workspace.sync_mode immutable
+>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py::test_backfill
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path
+
+=== Absolute workspace paths and running all tests still work
+>>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Pipeline: my_pipeline ([UUID][0])
+Update: [UUID][1]
+
+>>> [CLI] pipelines test my_pipeline --no-wait
+Recommendation: This command runs the last deployed version of the code
+
+If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
+
+Pipeline: my_pipeline ([UUID][0])
+Update: [UUID][2]
+
+>>> print_requests.py //updates
+{
+  "method": "POST",
+  "path": "/api/2.0/pipelines/[UUID][0]/updates",
+  "body": {
+    "test_only": true,
+    "test_options": {
+      "test_selection": [
+        "/Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill"
+      ]
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/pipelines/[UUID][0]/updates",
+  "body": {
+    "test_only": true
+  }
+}

acceptance/pipelines/test/immutable/pipeline_file.py

@@ -0,0 +1,6 @@
+from pyspark import pipelines as dp
+
+
+@dp.table
+def orders():
+    return spark.range(10)

acceptance/pipelines/test/immutable/script

@@ -0,0 +1,10 @@
+trace $CLI pipelines deploy
+rm "$OUT_REQUESTS"
+
+title "Local test paths are not supported with workspace.sync_mode immutable"
+trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill
+
+title "Absolute workspace paths and running all tests still work"
+trace MSYS_NO_PATHCONV=1 $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill --no-wait
+trace $CLI pipelines test my_pipeline --no-wait
+trace print_requests.py //updates

acceptance/pipelines/test/immutable/tests/test_orders.py

@@ -0,0 +1,2 @@
+def test_backfill():
+    assert True

acceptance/pipelines/test/invalid-project/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-pipeline-test-invalid-project
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: ${var.missing}

acceptance/pipelines/test/invalid-project/out.invalid.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="reference does not exist: ${var.missing}">reference does not exist: ${var.missing}</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/invalid-project/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/pipelines/test/invalid-project/output.txt

@@ -0,0 +1,5 @@
+
+=== A project with errors names its first error in the report
+>>> musterr [CLI] pipelines test my_pipeline --junit-xml out.invalid.xml
+Error: reference does not exist: ${var.missing}
+

acceptance/pipelines/test/invalid-project/script

@@ -0,0 +1,2 @@
+title "A project with errors names its first error in the report"
+trace musterr $CLI pipelines test my_pipeline --junit-xml out.invalid.xml

acceptance/pipelines/test/live/databricks.yml.tmpl

@@ -0,0 +1,22 @@
+bundle:
+  name: test-pipelines-test-live-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    orders:
+      # The Pipelines API rejects duplicate names, so the name is unique per run.
+      name: test-pipelines-test-live-$UNIQUE_NAME
+      catalog: main
+      schema: default
+      serverless: true
+      root_path: .
+      libraries:
+        - file:
+            path: ./transformations/orders.py
+      testing:
+        sources:
+          - glob:
+              include: ./tests/**
+        environment:
+          dependencies:
+            - -e ./helpers

acceptance/pipelines/test/live/helpers/order_helpers.py

@@ -0,0 +1,2 @@
+def order_total(amounts):
+    return sum(amounts)

acceptance/pipelines/test/live/helpers/pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "order-helpers"
+version = "0.1.0"
+
+[tool.setuptools]
+py-modules = ["order_helpers"]

acceptance/pipelines/test/live/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+CloudSlow = true

acceptance/pipelines/test/live/output.txt

@@ -0,0 +1,22 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default/files...
+Created pipelines.orders
+Files: 6 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Run the tests and wait for the results
+Done. PASSED=1 FAILED=0 SKIPPED=0 ERROR=0 TOTAL=1
+ name="test_order_total"
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.orders
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.orders
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipelines-test-live-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/pipelines/test/live/script

@@ -0,0 +1,20 @@
+if [ -n "$CLOUD_ENV" ] && [ -z "${TEST_PIPELINES_TESTING_ENABLED:-}" ]; then
+    echo "SKIP_TEST set TEST_PIPELINES_TESTING_ENABLED for a workspace with pipeline testing enabled"
+    exit 0
+fi
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+title "Run the tests and wait for the results\n"
+# Durations and test IDs differ between runs, so only the summary line and the case names are
+# compared. The script stops if the command fails.
+$CLI pipelines test --junit-xml junit.xml &> LOG.test
+tail -1 LOG.test
+grep -o ' name="test_[^"]*"' junit.xml