From 6092358dedd98294bbe64438af5c6248941ef8f3 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 10 Nov 2026 00:13:46 +0000 Subject: [PATCH 1/3] Map a project folder after -e in environment dependencies to its deployed folder With DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true, a local project folder after -e, with optional extras such as -e ./project[test], maps to its deployed folder instead of failing. This applies to job and pipeline environment dependencies. With the variable off, the old behavior stays. --- .../bundles/editable-project-folder.md | 1 + .../editable-folder/databricks.yml | 26 ++++++++++ .../editable-folder/out.test.toml | 2 + .../environments/editable-folder/output.txt | 10 ++++ .../editable-folder/project/pyproject.toml | 6 +++ .../environments/editable-folder/script | 1 + .../environments/editable-folder/test.toml | 2 + bundle/config/mutator/translate_paths.go | 24 +++++++++ bundle/config/mutator/translate_paths_test.go | 52 +++++++++++++++++++ bundle/env/experimental_pipeline_testing.go | 17 ++++++ 10 files changed, 141 insertions(+) create mode 100644 .nextchanges/bundles/editable-project-folder.md create mode 100644 acceptance/bundle/environments/editable-folder/databricks.yml create mode 100644 acceptance/bundle/environments/editable-folder/out.test.toml create mode 100644 acceptance/bundle/environments/editable-folder/output.txt create mode 100644 acceptance/bundle/environments/editable-folder/project/pyproject.toml create mode 100644 acceptance/bundle/environments/editable-folder/script create mode 100644 acceptance/bundle/environments/editable-folder/test.toml create mode 100644 bundle/env/experimental_pipeline_testing.go diff --git a/.nextchanges/bundles/editable-project-folder.md b/.nextchanges/bundles/editable-project-folder.md new file mode 100644 index 00000000000..616f3989b7c --- /dev/null +++ b/.nextchanges/bundles/editable-project-folder.md @@ -0,0 +1 @@ +* Fix `-e ./folder` in environment dependencies, which failed before deploy. A local project folder after `-e`, with optional extras such as `-e ./project[test]`, now maps to its deployed folder. ([#7427](https://github.com/databricks/cli/pull/7427)) diff --git a/acceptance/bundle/environments/editable-folder/databricks.yml b/acceptance/bundle/environments/editable-folder/databricks.yml new file mode 100644 index 00000000000..b6c43e86868 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/databricks.yml @@ -0,0 +1,26 @@ +bundle: + name: editable-folder + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: main + python_wheel_task: + package_name: orders + entry_point: main + environment_key: default + environments: + - environment_key: default + spec: + client: "1" + dependencies: + - -e ./project + + pipelines: + my_pipeline: + name: my pipeline + environment: + dependencies: + - -e ./project[test] diff --git a/acceptance/bundle/environments/editable-folder/out.test.toml b/acceptance/bundle/environments/editable-folder/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/editable-folder/output.txt b/acceptance/bundle/environments/editable-folder/output.txt new file mode 100644 index 00000000000..4c76692374c --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/output.txt @@ -0,0 +1,10 @@ + +>>> [CLI] bundle validate -o json +{ + "job": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "pipeline": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project[test]" + ] +} diff --git a/acceptance/bundle/environments/editable-folder/project/pyproject.toml b/acceptance/bundle/environments/editable-folder/project/pyproject.toml new file mode 100644 index 00000000000..468fddd1ca7 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/project/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1.0" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/environments/editable-folder/script b/acceptance/bundle/environments/editable-folder/script new file mode 100644 index 00000000000..86c80f6f792 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies}' diff --git a/acceptance/bundle/environments/editable-folder/test.toml b/acceptance/bundle/environments/editable-folder/test.toml new file mode 100644 index 00000000000..fdf8b9f0a35 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/test.toml @@ -0,0 +1,2 @@ +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..1bfc64d7537 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" @@ -162,6 +163,15 @@ func (t *translateContext) rewritePath( localPath = filepath.ToSlash(localPath) localRelPath = filepath.ToSlash(localRelPath) + // With pipeline testing enabled, the path after -e can also be a project folder, with optional + // extras such as ./project[test]. + if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" && env.ExperimentalPipelineTesting(ctx) { + if interp, ok := t.translateEditableFolder(localRelPath); ok { + t.seen[key] = interp + return interp, nil + } + } + // Convert local path into workspace path via specified function. var interp string switch opts.Mode { @@ -251,6 +261,20 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local return path.Join(t.remoteRoot, localRelPath), nil } +// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to +// its deployed folder. It reports false if the path is not a folder in the sync root. +func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) { + folder, extras := localRelPath, "" + if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { + folder, extras = localRelPath[:i], localRelPath[i:] + } + info, err := t.b.SyncRoot.Stat(folder) + if err != nil || !info.IsDir() { + return "", false + } + return "-e " + path.Join(t.remoteRoot, folder) + extras, true +} + func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { if t.skipLocalFileValidation { return path.Join(t.remoteRoot, localRelPath), nil diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8aaf9ce9c71..39c67692808 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -8,6 +8,9 @@ import ( "runtime" "testing" + bundleenv "github.com/databricks/cli/bundle/env" + libsenv "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" @@ -816,6 +819,55 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6]) } +func TestTranslatePathEditableProjectFolder(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) + + translate := func(ctx context.Context, dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies}, + }, + }, + }, + }, + }, + } + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) + return b, bundle.ApplySeq(ctx, b, mutator.NormalizePaths(), mutator.TranslatePaths()) + } + + // Without pipeline testing, a folder after -e stays an error, as it was before the feature. + _, diags := translate(t.Context(), "-e ./project") + assert.ErrorContains(t, diags.Error(), "unable to determine if "+filepath.Join(dir, "project")+" is not a notebook") + + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + b, diags := translate(ctx, "-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") + require.NoError(t, diags.Error()) + assert.Equal(t, []string{ + "-e /bundle/project", + "-e /bundle/project[test]", + "-e /bundle", + "-e /bundle[test]", + "-r /bundle/requirements.txt", + }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) + + // -r takes a file, so a folder after it is still an error. + _, diags = translate(ctx, "-r ./project") + assert.ErrorContains(t, diags.Error(), "project") +} + func TestTranslatePathWithComplexVariables(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go new file mode 100644 index 00000000000..b00a94176e9 --- /dev/null +++ b/bundle/env/experimental_pipeline_testing.go @@ -0,0 +1,17 @@ +package env + +import ( + "context" + + envlib "github.com/databricks/cli/libs/env" +) + +// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. +const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" + +// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is +// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. +func ExperimentalPipelineTesting(ctx context.Context) bool { + enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) + return enabled +} From 9791a452be10550a3c25678429d2b0725acbab7c Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 10 Nov 2026 00:16:43 +0000 Subject: [PATCH 2/3] Add the testing property of pipelines to bundles Add resources.pipelines..testing, with the test sources and test-only dependencies that databricks pipelines test runs with. The property is experimental. It needs DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true. While the variable is off, bundle validate rejects testing and names the variable, and bundles deploy exactly as before. Relative paths in testing are translated like libraries. Local wheels in test dependencies are uploaded like other libraries, and the same-name check covers them. bundle plan shows changes to testing made outside the bundle, and bundle deploy undoes them. The CLI uses its own types and raw calls for testing, because the Go SDK does not have the field yet. The test server stores spec.testing on Create and Edit and returns it on Get. --- .../bundles/pipelines-testing-plan.md | 1 + .nextchanges/bundles/pipelines-testing.md | 1 + .../editable-folder/databricks.yml | 9 +- .../environments/editable-folder/output.txt | 3 + .../environments/editable-folder/script | 2 +- .../bundle/refschema/output/pipelines.txt | 8 + .../pipelines/testing-disabled/databricks.yml | 11 + .../pipelines/testing-disabled/out.test.toml | 2 + .../pipelines/testing-disabled/output.txt | 10 + .../pipelines/testing-disabled/script | 1 + .../pipelines/testing-disabled/test.toml | 2 + .../testing-disabled/tests/test_orders.py | 2 + .../databricks.yml | 21 ++ .../testing-same-name-libraries/out.test.toml | 2 + .../testing-same-name-libraries/output.txt | 12 + .../test_helpers-0.1-py3-none-any.whl | 0 .../testing-same-name-libraries/script | 1 + .../testing-same-name-libraries/test.toml | 7 + .../tests/test_helpers-0.1-py3-none-any.whl | 0 .../tests/test_orders.py | 2 + .../transformations/orders.py | 1 + .../pipelines/testing/databricks.yml | 19 ++ .../dist/test_helpers-0.1-py3-none-any.whl | 0 .../resources/pipelines/testing/out.test.toml | 2 + .../resources/pipelines/testing/output.txt | 215 ++++++++++++++++++ .../bundle/resources/pipelines/testing/script | 34 +++ .../resources/pipelines/testing/test.toml | 5 + .../pipelines/testing/tests/requirements.txt | 1 + .../pipelines/testing/tests/test_orders.py | 2 + .../testing/transformations/orders.py | 1 + .../mutator/paths/pipeline_paths_visitor.go | 19 ++ .../paths/pipeline_paths_visitor_test.go | 7 + bundle/config/resources/pipeline.go | 28 +++ .../validate/pipeline_testing_enabled.go | 49 ++++ bundle/direct/dresources/pipeline.go | 86 +++++++ bundle/direct/dresources/pipeline_test.go | 71 ++++++ bundle/internal/schema/annotations.yml | 20 ++ bundle/libraries/expand_glob_references.go | 13 ++ bundle/libraries/remote_path.go | 1 + bundle/libraries/same_name_libraries.go | 1 + bundle/phases/initialize.go | 1 + bundle/schema/jsonschema.json | 78 +++++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/pipelines.go | 9 + libs/testserver/pipelines_spec_testing.go | 45 ++++ .../databricks/bundles/pipelines/__init__.py | 24 ++ .../bundles/pipelines/_models/pipeline.py | 18 ++ .../_models/pipeline_test_environment.py | 42 ++++ .../pipelines/_models/pipeline_test_source.py | 46 ++++ .../pipelines/_models/pipeline_testing.py | 60 +++++ 50 files changed, 995 insertions(+), 2 deletions(-) create mode 100644 .nextchanges/bundles/pipelines-testing-plan.md create mode 100644 .nextchanges/bundles/pipelines-testing.md create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/script create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/script create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/script create mode 100644 acceptance/bundle/resources/pipelines/testing/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/transformations/orders.py create mode 100644 bundle/config/validate/pipeline_testing_enabled.go create mode 100644 libs/testserver/pipelines_spec_testing.go create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_test_environment.py create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_test_source.py create mode 100644 python/databricks/bundles/pipelines/_models/pipeline_testing.py diff --git a/.nextchanges/bundles/pipelines-testing-plan.md b/.nextchanges/bundles/pipelines-testing-plan.md new file mode 100644 index 00000000000..501c4a3752d --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing-plan.md @@ -0,0 +1 @@ +* `bundle plan` now shows a change to a pipeline's `testing` that was made outside the bundle, and `bundle deploy` undoes it. ([#7300](https://github.com/databricks/cli/pull/7300)) diff --git a/.nextchanges/bundles/pipelines-testing.md b/.nextchanges/bundles/pipelines-testing.md new file mode 100644 index 00000000000..54b3280c82e --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing.md @@ -0,0 +1 @@ +* Add the `testing` property of pipelines, which sets the test sources and test-only dependencies that `databricks pipelines test` runs with. ([#7300](https://github.com/databricks/cli/pull/7300)) diff --git a/acceptance/bundle/environments/editable-folder/databricks.yml b/acceptance/bundle/environments/editable-folder/databricks.yml index b6c43e86868..f16db8d0d95 100644 --- a/acceptance/bundle/environments/editable-folder/databricks.yml +++ b/acceptance/bundle/environments/editable-folder/databricks.yml @@ -23,4 +23,11 @@ resources: name: my pipeline environment: dependencies: - - -e ./project[test] + - -e ./project + testing: + sources: + - glob: + include: /Workspace/Shared/orders/tests/** + environment: + dependencies: + - -e ./project[test] diff --git a/acceptance/bundle/environments/editable-folder/output.txt b/acceptance/bundle/environments/editable-folder/output.txt index 4c76692374c..5c032cead61 100644 --- a/acceptance/bundle/environments/editable-folder/output.txt +++ b/acceptance/bundle/environments/editable-folder/output.txt @@ -5,6 +5,9 @@ "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" ], "pipeline": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "testing": [ "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project[test]" ] } diff --git a/acceptance/bundle/environments/editable-folder/script b/acceptance/bundle/environments/editable-folder/script index 86c80f6f792..24314e31c00 100644 --- a/acceptance/bundle/environments/editable-folder/script +++ b/acceptance/bundle/environments/editable-folder/script @@ -1 +1 @@ -trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies}' +trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies, testing: .resources.pipelines.my_pipeline.testing.environment.dependencies}' diff --git a/acceptance/bundle/refschema/output/pipelines.txt b/acceptance/bundle/refschema/output/pipelines.txt index c8bc57f512c..9c2b4e56b5c 100644 --- a/acceptance/bundle/refschema/output/pipelines.txt +++ b/acceptance/bundle/refschema/output/pipelines.txt @@ -867,6 +867,14 @@ resources.pipelines.*.storage string ALL resources.pipelines.*.tags map[string]string ALL resources.pipelines.*.tags.* string ALL resources.pipelines.*.target string ALL +resources.pipelines.*.testing *resources.PipelineTesting ALL +resources.pipelines.*.testing.environment *resources.PipelineTestEnvironment ALL +resources.pipelines.*.testing.environment.dependencies []string ALL +resources.pipelines.*.testing.environment.dependencies[*] string ALL +resources.pipelines.*.testing.sources []resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*] resources.PipelineTestSource ALL +resources.pipelines.*.testing.sources[*].glob *pipelines.PathPattern ALL +resources.pipelines.*.testing.sources[*].glob.include string ALL resources.pipelines.*.trigger *pipelines.PipelineTrigger ALL resources.pipelines.*.trigger.cron *pipelines.CronTrigger ALL resources.pipelines.*.trigger.cron.quartz_cron_schedule string ALL diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml b/acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml new file mode 100644 index 00000000000..57c6246a47b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: pipeline-testing-disabled + +resources: + pipelines: + orders: + name: orders + testing: + sources: + - glob: + include: ./tests/** diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml b/acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/output.txt b/acceptance/bundle/resources/pipelines/testing-disabled/output.txt new file mode 100644 index 00000000000..97796bc8bc8 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle validate +Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it + at resources.pipelines.orders.testing + in databricks.yml:9:9 + +Name: pipeline-testing-disabled +Target: default + +Found 1 error diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/script b/acceptance/bundle/resources/pipelines/testing-disabled/script new file mode 100644 index 00000000000..44dbe24759c --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/script @@ -0,0 +1 @@ +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/test.toml b/acceptance/bundle/resources/pipelines/testing-disabled/test.toml new file mode 100644 index 00000000000..4e81fd86218 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/test.toml @@ -0,0 +1,2 @@ +# Runs only against the local test server. The experimental gate is off. +Cloud = false diff --git a/acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-disabled/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..b9f00092a5b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,12 @@ + +>>> errcode [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..e4b9ddaaa91 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d018f5851ee --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,7 @@ +# Runs only against the local test server. +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..a751a032169 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - --editable ${workspace.file_path}[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..e9296bd2e55 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,215 @@ + +=== Relative test paths resolve to the deployed workspace paths +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ] +} + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Created pipelines.orders +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } + } +} + +=== No changes after the deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A remote edit that drops testing shows in the plan +>>> [CLI] bundle plan -o json +{ + "action": "update", + "old": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + }, + "new": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } +} + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Changing testing updates the pipeline +>>> update_file.py databricks.yml ./tests/** ./tests/unit/** + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders", + "testing": { + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" + } + } + ], + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + } + } + } +} + +=== Removing testing sends an update without it +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/transformations/**" + } + } + ], + "name": "orders" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..aa150679e2d --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,34 @@ +title "Relative test paths resolve to the deployed workspace paths" +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +title "No changes after the deploy" +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "A remote edit that drops testing shows in the plan" +PIPELINE_ID=$(read_id.py orders) +MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json +MSYS_NO_PATHCONV=1 $CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit +rm edit.json +trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing' +trace $CLI bundle deploy +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "Changing testing updates the pipeline" +trace update_file.py databricks.yml './tests/**' './tests/unit/**' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +title "Removing testing sends an update without it" +update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..965c596a302 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,5 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..83be40afd7d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns @@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !libraries.IsLibraryLocal(s) }, }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }, } pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ @@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !ok }, }, + { + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, + }, } return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..85c29065c5b 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) { }, }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &resources.PipelineTesting{ + Environment: &resources.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index fd1fc61ce86..1b8b8e0ef43 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -22,6 +22,34 @@ type Pipeline struct { // Nil (unset) uses the server default, which cascades. Set to false to retain datasets // when the pipeline is deleted. Delete-time only: never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. Remove it, and the + // types below, once CreatePipeline has it. + Testing *PipelineTesting `json:"testing,omitempty"` +} + +// PipelineTesting is the test configuration of a pipeline. Only test-only updates use it. +type PipelineTesting struct { + // Sources are the workspace files and folders that hold the pipeline's tests. They must + // not overlap the pipeline's libraries. + Sources []PipelineTestSource `json:"sources,omitempty"` + + // Environment lists the packages that only the tests need. + Environment *PipelineTestEnvironment `json:"environment,omitempty"` +} + +// PipelineTestSource is a workspace location that holds pipeline tests. +type PipelineTestSource struct { + // Glob is a test file, or a folder followed by /**. + Glob *pipelines.PathPattern `json:"glob,omitempty"` +} + +// PipelineTestEnvironment lists the packages that a pipeline's tests need in addition to the +// pipeline's environment. +type PipelineTestEnvironment struct { + // Dependencies are files that declare the packages: a project folder, a requirements + // file, or a wheel. Package names and URLs are rejected by the API. + Dependencies []string `json:"dependencies,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go new file mode 100644 index 00000000000..85fdb85ed4b --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled.go @@ -0,0 +1,49 @@ +package validate + +import ( + "context" + "fmt" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type pipelineTestingEnabled struct{ bundle.RO } + +// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. +func PipelineTestingEnabled() bundle.ReadOnlyMutator { + return &pipelineTestingEnabled{} +} + +func (v *pipelineTestingEnabled) Name() string { + return "validate:pipeline_testing_enabled" +} + +func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if env.ExperimentalPipelineTesting(ctx) { + return nil + } + var diags diag.Diagnostics + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + if pipelines[key] == nil || pipelines[key].Testing == nil { + continue + } + path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) + value, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return diag.FromErr(err) + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), + Locations: value.Locations(), + Paths: dyn.ToStructPaths(path), + }) + } + return diags +} diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 6f367d4e498..3eacc3b5fa6 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -2,13 +2,19 @@ package dresources import ( "context" + "encoding/json" + "fmt" + "net/http" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -22,6 +28,9 @@ type PipelineState struct { // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at // delete time; never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + + // Testing is PipelineSpec.testing, which the Go SDK does not have yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } func (s *PipelineState) UnmarshalJSON(b []byte) error { @@ -52,6 +61,9 @@ type PipelineRemote struct { PipelineId string `json:"pipeline_id,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` State pipelines.PipelineState `json:"state,omitempty"` + + // Testing is spec.testing of the Get response, read by DoRead because the Go SDK does not have it yet. + Testing *resources.PipelineTesting `json:"testing,omitempty"` } // Custom marshaler needed because embedded CreatePipeline has its own MarshalJSON @@ -78,6 +90,7 @@ func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState return &PipelineState{ CreatePipeline: input.CreatePipeline, CascadeOnDestroy: input.CascadeOnDestroy, + Testing: input.Testing, } } @@ -86,10 +99,21 @@ func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { CreatePipeline: remote.CreatePipeline, // cascade_on_destroy is input-only and absent from PipelineRemote, so it stays nil here. CascadeOnDestroy: nil, + Testing: remote.Testing, } } +// pipelineTestingResponse holds spec.testing of a Get Pipeline response. +type pipelineTestingResponse struct { + Spec struct { + Testing *resources.PipelineTesting `json:"testing"` + } `json:"spec"` +} + func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { + if env.ExperimentalPipelineTesting(ctx) { + return r.doReadWithTesting(ctx, id) + } resp, err := r.client.Pipelines.GetByPipelineId(ctx, id) if err != nil { return nil, err @@ -97,6 +121,26 @@ func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemo return makePipelineRemote(resp), nil } +// doReadWithTesting sends the same request as GetByPipelineId, but raw, so that it can also read +// spec.testing, which GetPipelineResponse does not have yet. Remove it once GetPipelineResponse has it. +func (r *ResourcePipeline) doReadWithTesting(ctx context.Context, id string) (*PipelineRemote, error) { + var raw json.RawMessage + if err := r.doRaw(ctx, http.MethodGet, "/api/2.0/pipelines/"+id, nil, &raw); err != nil { + return nil, err + } + var resp pipelines.GetPipelineResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, err + } + var testing pipelineTestingResponse + if err := json.Unmarshal(raw, &testing); err != nil { + return nil, err + } + remote := makePipelineRemote(&resp) + remote.Testing = testing.Spec.Testing + return remote, nil +} + func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { var createPipeline pipelines.CreatePipeline if p.Spec != nil { @@ -155,10 +199,20 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { PipelineId: p.PipelineId, RunAsUserName: p.RunAsUserName, State: p.State, + // GetPipelineResponse has no spec.testing; DoRead sets it from the raw response. + Testing: nil, } } func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + if config.Testing != nil { + var response pipelines.CreatePipelineResponse + err := r.doWithTesting(ctx, http.MethodPost, "/api/2.0/pipelines", config.CreatePipeline, config.Testing, &response) + if err != nil { + return "", nil, err + } + return response.PipelineId, nil, nil + } response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err @@ -212,6 +266,9 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *Pipe ForceSendFields: utils.FilterFields[pipelines.EditPipeline](config.ForceSendFields), } + if config.Testing != nil { + return nil, r.doWithTesting(ctx, http.MethodPut, "/api/2.0/pipelines/"+id, request, config.Testing, nil) + } return nil, r.client.Pipelines.Update(ctx, request) } @@ -242,6 +299,35 @@ func (*ResourcePipeline) OverrideChangeDesc(_ context.Context, path *structpath. return nil } +// doWithTesting sends a Create or Edit request with the testing field, which the Go SDK does not +// have yet. The body is the SDK's JSON for the request plus testing. Remove this once +// CreatePipeline and EditPipeline have the field; pipelines without testing already use the SDK. +func (r *ResourcePipeline) doWithTesting(ctx context.Context, method, path string, request any, testing *resources.PipelineTesting, response any) error { + data, err := json.Marshal(request) + if err != nil { + return err + } + // RawMessage values keep the SDK's encoding, including large integers, unchanged. + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return err + } + body["testing"], err = json.Marshal(testing) + if err != nil { + return err + } + return r.doRaw(ctx, method, path, body, response) +} + +// doRaw sends a request that the Go SDK cannot express yet because it lacks the testing field. +func (r *ResourcePipeline) doRaw(ctx context.Context, method, path string, request, response any) error { + apiClient, err := client.New(r.client.Config) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + return apiClient.Do(ctx, method, path, auth.WorkspaceIDHeaders(r.client.Config), nil, request, response) +} + // Note, terraform provider either // a) reads back state at least once and fails create if state is "failed" // b) repeatededly reads state until state is "running" (if spec.Contionous is set). diff --git a/bundle/direct/dresources/pipeline_test.go b/bundle/direct/dresources/pipeline_test.go index 3629255a600..34f41ad005d 100644 --- a/bundle/direct/dresources/pipeline_test.go +++ b/bundle/direct/dresources/pipeline_test.go @@ -4,8 +4,15 @@ import ( "reflect" "testing" + bundleenv "github.com/databricks/cli/bundle/env" + libsenv "github.com/databricks/cli/libs/env" + + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,3 +60,67 @@ func TestPipelineOverrideChangeDescCascadeOnDestroy(t *testing.T) { assert.Equal(t, deployplan.Skip, change.Action) }) } + +func TestPipelineRawRequests(t *testing.T) { + server := testserver.New(t) + var bodies, workspaceIDs []string + server.Handle("POST", "/api/2.0/pipelines", func(req testserver.Request) any { + bodies = append(bodies, string(req.Body)) + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return pipelines.CreatePipelineResponse{PipelineId: "p1"} + }) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + workspaceIDs = append(workspaceIDs, req.Headers.Get(auth.WorkspaceIDHeader)) + return map[string]any{"pipeline_id": "p1", "spec": map[string]any{"name": "orders"}} + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token", WorkspaceID: "123"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + testing := &resources.PipelineTesting{Sources: []resources.PipelineTestSource{{Glob: &pipelines.PathPattern{Include: "/Workspace/orders/tests/**"}}}} + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders", Testing: testing}) + require.NoError(t, err) + _, err = r.DoRead(t.Context(), "p1") + require.NoError(t, err) + // Without testing, the request is the SDK's. + _, _, err = r.DoCreate(t.Context(), &PipelineState{Name: "orders"}) + require.NoError(t, err) + + // The raw requests carry the workspace ID, as the SDK's do. + assert.Equal(t, []string{"123", "123", "123"}, workspaceIDs) + assert.JSONEq(t, `{"name": "orders"}`, bodies[1]) +} + +func TestPipelineReadReturnsTesting(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req testserver.Request) any { + return map[string]any{ + "pipeline_id": req.Vars["pipeline_id"], + "state": "IDLE", + "spec": map[string]any{ + "name": "orders", + "testing": map[string]any{"sources": []any{map[string]any{"glob": map[string]any{"include": "/Workspace/orders/tests/**"}}}}, + }, + } + }) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + r := (&ResourcePipeline{}).New(w) + + // Without pipeline testing, the SDK reads the pipeline, as before the feature. + remote, err := r.DoRead(t.Context(), "p1") + require.NoError(t, err) + assert.Equal(t, "orders", remote.Name) + assert.Nil(t, remote.Testing) + + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + remote, err = r.DoRead(ctx, "p1") + require.NoError(t, err) + assert.Equal(t, "p1", remote.PipelineId) + assert.Equal(t, "orders", remote.Name) + assert.Equal(t, pipelines.PipelineStateIdle, remote.State) + require.NotNil(t, remote.Testing) + require.Len(t, remote.Testing.Sources, 1) + assert.Equal(t, "/Workspace/orders/tests/**", remote.Testing.Sources[0].Glob.Include) + assert.Same(t, remote.Testing, r.RemapState(remote).Testing) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 08b65053e3b..4747a404a6f 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1636,6 +1636,26 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW + "$fields": + "environment": + "description": |- + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + "sources": + "description": |- + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + "$fields": + "glob": + "description": |- + A test file, or a folder followed by `/**`. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 70e577a1a2a..359744328b4 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -143,6 +143,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8c5a68d3130..a1a4961fb11 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -36,6 +36,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.NoInterpolationInAuthConfig(), validate.NoInterpolationInBundleName(), validate.ValidateEngine(), + validate.PipelineTestingEnabled(), validate.Scripts(), mutator.ValidateSecretValueIsVariable(), diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index a060ec2e8a1..8b92df9a0cc 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2389,6 +2389,12 @@ "deprecationMessage": "This field is deprecated", "deprecated": true }, + "testing": { + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTesting", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", @@ -2445,6 +2451,64 @@ } ] }, + "resources.PipelineTestEnvironment": { + "oneOf": [ + { + "type": "object", + "properties": { + "dependencies": { + "description": "A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "object", + "properties": { + "glob": { + "description": "A test file, or a folder followed by `/**`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelineTesting": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment": { + "description": "The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestEnvironment" + }, + "sources": { + "description": "The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresBranch": { "oneOf": [ { @@ -22434,6 +22498,20 @@ } ] }, + "resources.PipelineTestSource": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelineTestSource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScopePermission": { "oneOf": [ { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c48ebb04df2..980f1367f43 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -223,6 +223,7 @@ type FakeWorkspace struct { JobRunIdempotency map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse PipelineUpdates map[string]bool + pipelineTesting map[string]json.RawMessage Monitors map[string]catalog.MonitorInfo DataQualityMonitors map[string]dataquality.Monitor Apps map[string]apps.App @@ -532,6 +533,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, PipelineUpdates: map[string]bool{}, + pipelineTesting: map[string]json.RawMessage{}, Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..97cf7073123 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -18,6 +18,13 @@ func (s *FakeWorkspace) PipelineGet(pipelineId string) Response { Body: map[string]string{"message": fmt.Sprintf("The specified pipeline %s was not found.", pipelineId)}, } } + if testing, ok := s.pipelineTesting[pipelineId]; ok { + body, err := withPipelineTesting(value, testing) + if err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + return Response{Body: body} + } return Response{ Body: value, } @@ -80,6 +87,7 @@ func (s *FakeWorkspace) PipelineCreate(req Request) Response { setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = r + s.setPipelineTesting(pipelineId, req.Body) return Response{ Body: pipelines.CreatePipelineResponse{ @@ -139,6 +147,7 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response } setSpecDefaults(&spec, pipelineId) s.Pipelines[pipelineId] = item + s.setPipelineTesting(pipelineId, req.Body) return Response{} } diff --git a/libs/testserver/pipelines_spec_testing.go b/libs/testserver/pipelines_spec_testing.go new file mode 100644 index 00000000000..ce397e5d8e4 --- /dev/null +++ b/libs/testserver/pipelines_spec_testing.go @@ -0,0 +1,45 @@ +package testserver + +import ( + "encoding/json" + + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +// pipelineTestingRequest holds the testing field of a Create or Edit request, which is not in the +// Go SDK yet. +type pipelineTestingRequest struct { + Testing json.RawMessage `json:"testing"` +} + +// setPipelineTesting records the testing field of a Create or Edit body, so that Get returns it. +// Like the rest of the spec, an Edit without testing removes it. +func (s *FakeWorkspace) setPipelineTesting(pipelineID string, body []byte) { + var request pipelineTestingRequest + _ = json.Unmarshal(body, &request) + if len(request.Testing) == 0 || string(request.Testing) == "null" { + delete(s.pipelineTesting, pipelineID) + return + } + s.pipelineTesting[pipelineID] = request.Testing +} + +// withPipelineTesting adds spec.testing to a Get Pipeline response, which the Go SDK type cannot +// hold. The JSON is merged as raw values, so numbers and the other fields are kept as they are. +func withPipelineTesting(value pipelines.GetPipelineResponse, testing json.RawMessage) (map[string]json.RawMessage, error) { + data, err := json.Marshal(value) + if err != nil { + return nil, err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return nil, err + } + var spec map[string]json.RawMessage + if err := json.Unmarshal(body["spec"], &spec); err != nil { + return nil, err + } + spec["testing"] = testing + body["spec"], err = json.Marshal(spec) + return body, err +} diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 412cbfa59b3..d9bf369bdb3 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -207,6 +207,15 @@ "PipelinePermissionLevel", "PipelinePermissionLevelParam", "PipelinePermissionParam", + "PipelineTestEnvironment", + "PipelineTestEnvironmentDict", + "PipelineTestEnvironmentParam", + "PipelineTestSource", + "PipelineTestSourceDict", + "PipelineTestSourceParam", + "PipelineTesting", + "PipelineTestingDict", + "PipelineTestingParam", "PipelinesEnvironment", "PipelinesEnvironmentDict", "PipelinesEnvironmentParam", @@ -645,6 +654,21 @@ PipelinePermissionLevel, PipelinePermissionLevelParam, ) +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentDict, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceDict, + PipelineTestSourceParam, +) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingDict, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentDict, diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index d9d898db40f..f73c11c4bee 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -41,6 +41,10 @@ PipelinePermission, PipelinePermissionParam, ) +from databricks.bundles.pipelines._models.pipeline_testing import ( + PipelineTesting, + PipelineTestingParam, +) from databricks.bundles.pipelines._models.pipelines_environment import ( PipelinesEnvironment, PipelinesEnvironmentParam, @@ -254,6 +258,13 @@ class Pipeline(Resource): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTesting] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + """ + usage_policy_id: VariableOrOptional[str] = None """ :meta private: [EXPERIMENTAL] @@ -467,6 +478,13 @@ class PipelineDict(TypedDict, total=False): [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field. """ + testing: VariableOrOptional[PipelineTestingParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + """ + usage_policy_id: VariableOrOptional[str] """ :meta private: [EXPERIMENTAL] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py new file mode 100644 index 00000000000..3386c1d69b1 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -0,0 +1,42 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestEnvironment: + """ + :meta private: [EXPERIMENTAL] + """ + + dependencies: VariableOrList[str] = field(default_factory=list) + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestEnvironmentDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestEnvironmentDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestEnvironmentDict(TypedDict, total=False): + """""" + + dependencies: VariableOrList[str] + """ + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and an extra, for example `--editable ${workspace.file_path}[test]`. Package names and URLs are rejected. + """ + + +PipelineTestEnvironmentParam = PipelineTestEnvironmentDict | PipelineTestEnvironment diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py new file mode 100644 index 00000000000..b893d161d69 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -0,0 +1,46 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.path_pattern import ( + PathPattern, + PathPatternParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTestSource: + """ + :meta private: [EXPERIMENTAL] + """ + + glob: VariableOrOptional[PathPattern] = None + """ + A test file, or a folder followed by `/**`. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestSourceDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestSourceDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestSourceDict(TypedDict, total=False): + """""" + + glob: VariableOrOptional[PathPatternParam] + """ + A test file, or a folder followed by `/**`. + """ + + +PipelineTestSourceParam = PipelineTestSourceDict | PipelineTestSource diff --git a/python/databricks/bundles/pipelines/_models/pipeline_testing.py b/python/databricks/bundles/pipelines/_models/pipeline_testing.py new file mode 100644 index 00000000000..c8f20835c95 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/pipeline_testing.py @@ -0,0 +1,60 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.pipelines._models.pipeline_test_environment import ( + PipelineTestEnvironment, + PipelineTestEnvironmentParam, +) +from databricks.bundles.pipelines._models.pipeline_test_source import ( + PipelineTestSource, + PipelineTestSourceParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PipelineTesting: + """ + :meta private: [EXPERIMENTAL] + """ + + environment: VariableOrOptional[PipelineTestEnvironment] = None + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSource] = field(default_factory=list) + """ + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + """ + + @classmethod + def from_dict(cls, value: "PipelineTestingDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PipelineTestingDict": + return _transform_to_json_value(self) # type:ignore + + +class PipelineTestingDict(TypedDict, total=False): + """""" + + environment: VariableOrOptional[PipelineTestEnvironmentParam] + """ + The packages that only the tests need. Test-only updates install them in addition to the pipeline's `environment`. Other updates do not. + """ + + sources: VariableOrList[PipelineTestSourceParam] + """ + The workspace files and folders that hold the pipeline's tests. They must not overlap the pipeline's `libraries`. + """ + + +PipelineTestingParam = PipelineTestingDict | PipelineTesting From 1b4896634ffe12ae7de628d418f6fe5086c68ac0 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 10 Nov 2026 00:17:43 +0000 Subject: [PATCH 3/3] Validate the test dependencies of pipelines in bundle validate bundle validate rejects the testing.environment.dependencies that the API rejects: package names, URLs, and a requirements file without -r. It runs after TranslatePaths, so it checks the paths that deploy sends. --- .../bundles/pipelines-testing-dependencies.md | 1 + .../testing-dependencies/databricks.yml | 23 ++++ .../orders/requirements.txt | 0 .../testing-dependencies/out.test.toml | 2 + .../pipelines/testing-dependencies/output.txt | 39 ++++++ .../pipelines/testing-dependencies/script | 1 + .../pipelines/testing-dependencies/test.toml | 3 + .../validate/pipeline_testing_dependencies.go | 123 ++++++++++++++++++ .../pipeline_testing_dependencies_test.go | 64 +++++++++ bundle/phases/initialize.go | 4 + 10 files changed, 260 insertions(+) create mode 100644 .nextchanges/bundles/pipelines-testing-dependencies.md create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/script create mode 100644 acceptance/bundle/resources/pipelines/testing-dependencies/test.toml create mode 100644 bundle/config/validate/pipeline_testing_dependencies.go create mode 100644 bundle/config/validate/pipeline_testing_dependencies_test.go diff --git a/.nextchanges/bundles/pipelines-testing-dependencies.md b/.nextchanges/bundles/pipelines-testing-dependencies.md new file mode 100644 index 00000000000..b105355be95 --- /dev/null +++ b/.nextchanges/bundles/pipelines-testing-dependencies.md @@ -0,0 +1 @@ +* `bundle validate` reports common mistakes in a pipeline's `testing.environment.dependencies`, such as package names, URLs and a requirements file without `-r`. ([#7428](https://github.com/databricks/cli/pull/7428)) diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml new file mode 100644 index 00000000000..6bc337c57d3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: pipeline-testing-dependencies + +resources: + pipelines: + orders: + name: orders + testing: + sources: + - glob: + include: /Workspace/Shared/orders/tests/** + environment: + dependencies: + # Accepted, as by the API. + - --editable /Workspace/Shared/orders[test] + - --requirement=/Workspace/Shared/orders/requirements.txt + - -r ./orders/requirements.txt + # Rejected, as by the API. + - faker==33.0 + - git+https://github.com/example/helpers + - /Workspace/Shared/orders/requirements.txt + - --requirement ./orders/requirements.txt + - /Workspace/Shared/orders -i https://example.com/simple diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt b/acceptance/bundle/resources/pipelines/testing-dependencies/orders/requirements.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt b/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt new file mode 100644 index 00000000000..434913a1bca --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/output.txt @@ -0,0 +1,39 @@ + +>>> musterr [CLI] bundle validate +Error: testing.environment.dependencies must point at files, not "faker==33.0" + at resources.pipelines.orders.testing.environment.dependencies[3] + in databricks.yml:19:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies must point at files, not "git+https://github.com/example/helpers" + at resources.pipelines.orders.testing.environment.dependencies[4] + in databricks.yml:20:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies needs -r before the requirements file "/Workspace/Shared/orders/requirements.txt" + at resources.pipelines.orders.testing.environment.dependencies[5] + in databricks.yml:21:15 + +Without -r, pip treats the file as a package to install. For example: -r ./tests/requirements.txt. + +Error: testing.environment.dependencies must point at files, not "--requirement ./orders/requirements.txt" + at resources.pipelines.orders.testing.environment.dependencies[6] + in databricks.yml:22:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Error: testing.environment.dependencies must point at files, not "/Workspace/Shared/orders -i https://example.com/simple" + at resources.pipelines.orders.testing.environment.dependencies[7] + in databricks.yml:23:15 + +A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported. + +Name: pipeline-testing-dependencies +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dependencies/default + +Found 5 errors diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/script b/acceptance/bundle/resources/pipelines/testing-dependencies/script new file mode 100644 index 00000000000..44dbe24759c --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/script @@ -0,0 +1 @@ +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml new file mode 100644 index 00000000000..b6b581497c5 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dependencies/test.toml @@ -0,0 +1,3 @@ +# Runs only against the local test server. +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/bundle/config/validate/pipeline_testing_dependencies.go b/bundle/config/validate/pipeline_testing_dependencies.go new file mode 100644 index 00000000000..047c55e4f8d --- /dev/null +++ b/bundle/config/validate/pipeline_testing_dependencies.go @@ -0,0 +1,123 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +// pipelineTestingDependencyOptions are the pip options that may come before a test dependency's path. +var pipelineTestingDependencyOptions = []string{"--requirement", "--editable", "-r", "-e"} + +const ( + pipelineTestingDependencyNotFile = "not a file" + pipelineTestingDependencyNeedsRequirementOption = "requirements file without -r" +) + +type pipelineTestingDependencies struct{ bundle.RO } + +// PipelineTestingDependencies rejects the entries of testing.environment.dependencies that the API +// rejects on deploy, such as package names and URLs. It runs after paths are translated. +func PipelineTestingDependencies() bundle.ReadOnlyMutator { + return &pipelineTestingDependencies{} +} + +func (v *pipelineTestingDependencies) Name() string { + return "validate:pipeline_testing_dependencies" +} + +func (v *pipelineTestingDependencies) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), + dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex(), + ) + _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + dependency, ok := v.AsString() + if !ok { + return v, nil + } + summary := fmt.Sprintf("testing.environment.dependencies must point at files, not %q", dependency) + detail := "A test dependency is a project folder, a wheel, or a requirements file after -r, in the bundle, under /Workspace/, or under /Volumes/. Declare test packages in a project file, such as the test extra of a pyproject.toml, and reference its folder, for example -e ./project[test]. For a path in the bundle, use -e or -r followed by a space. URLs, other pip options, and .. are not supported." + switch pipelineTestingDependencyProblem(dependency) { + case "": + return v, nil + case pipelineTestingDependencyNeedsRequirementOption: + summary = fmt.Sprintf("testing.environment.dependencies needs -r before the requirements file %q", dependency) + detail = "Without -r, pip treats the file as a package to install. For example: -r ./tests/requirements.txt." + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Detail: detail, + Locations: v.Locations(), + Paths: dyn.ToStructPaths(p), + }) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + return diags +} + +// pipelineTestingDependencyProblem returns why dependency is a common mistake that the API rejects, +// or "" if it is not: a package name, a URL, another pip option, "..", a path outside /Workspace and +// /Volumes, or a requirements file without -r. Anything else is left to the API. dependency is a +// value after path translation, so a relative path is a local library that deploy uploads. +func pipelineTestingDependencyProblem(dependency string) string { + // A reference resolved at deploy time, such as a volume path, is checked by the API. + if dynvar.ContainsVariableReference(dependency) { + return "" + } + value := strings.TrimSpace(dependency) + option, filePath := cutPipelineTestingDependencyOption(value) + switch { + case strings.Contains(value, "://"): + return pipelineTestingDependencyNotFile + case strings.HasPrefix(filePath, "-") || strings.Contains(filePath, " -"): + return pipelineTestingDependencyNotFile + case strings.HasPrefix(filePath, "/"): + if !strings.HasPrefix(filePath, "/Workspace/") && !strings.HasPrefix(filePath, "/Volumes/") { + return pipelineTestingDependencyNotFile + } + if slices.Contains(strings.Split(filePath, "/"), "..") { + return pipelineTestingDependencyNotFile + } + // Path translation rewrites a path in the bundle after "-r " or "-e ", so a relative path that + // remains after an option was not rewritten. + case option != "" || !libraries.IsLibraryLocal(filePath): + return pipelineTestingDependencyNotFile + } + if strings.HasSuffix(strings.ToLower(filePath), ".txt") && option != "-r" && option != "--requirement" { + return pipelineTestingDependencyNeedsRequirementOption + } + return "" +} + +// cutPipelineTestingDependencyOption splits a leading pip option, such as "-r", from the path after it. +func cutPipelineTestingDependencyOption(value string) (string, string) { + for _, option := range pipelineTestingDependencyOptions { + rest, ok := strings.CutPrefix(value, option) + if !ok { + continue + } + switch { + case strings.HasPrefix(rest, "=") && strings.HasPrefix(option, "--"): + return option, strings.TrimSpace(rest[1:]) + case strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "\t"): + return option, strings.TrimSpace(rest) + case strings.HasPrefix(rest, "/") && !strings.HasPrefix(option, "--"): + return option, rest + } + } + return "", value +} diff --git a/bundle/config/validate/pipeline_testing_dependencies_test.go b/bundle/config/validate/pipeline_testing_dependencies_test.go new file mode 100644 index 00000000000..10a8954815d --- /dev/null +++ b/bundle/config/validate/pipeline_testing_dependencies_test.go @@ -0,0 +1,64 @@ +package validate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPipelineTestingDependencyProblem(t *testing.T) { + accepted := []string{ + "--editable /Workspace/Users/someone@example.com/orders[test]", + "-e /Workspace/orders", + "-e/Workspace/orders", + "--editable=/Workspace/orders", + "-r /Workspace/orders/tests/requirements.txt", + "--requirement=/Workspace/orders/requirements.txt", + "--requirement /Volumes/main/default/reqs/requirements.TXT", + "/Volumes/main/default/wheels/helpers-0.1-py3-none-any.whl", + "/Workspace/Users/someone@example.com/my orders/project", + "./dist/*.whl", + "dist/helpers-0.1-py3-none-any.whl", + "${resources.volumes.wheels.volume_path}/helpers-0.1-py3-none-any.whl", + // Not a common mistake, so the API decides. + "/Workspace/orders\n/Workspace/shared", + } + for _, dependency := range accepted { + assert.Empty(t, pipelineTestingDependencyProblem(dependency), dependency) + } + + notFiles := []string{ + // Package names. + "faker", + "faker==33.0", + "requests>=2", + // URLs. + "git+https://github.com/example/helpers", + "https://example.com/helpers-0.1-py3-none-any.whl", + "--editable git+https://github.com/example/helpers", + // Other pip options. + "-i https://example.com/simple", + "/Workspace/orders -i example", + "--no-deps /Workspace/orders", + // Paths outside /Workspace and /Volumes. + "/dbfs/helpers-0.1-py3-none-any.whl", + "/Users/someone@example.com/helpers-0.1-py3-none-any.whl", + // "..". + "/Workspace/orders/../shared", + // A path in the bundle that path translation did not rewrite. + "--editable ./project", + "--requirement ./tests/requirements.txt", + } + for _, dependency := range notFiles { + assert.Equal(t, pipelineTestingDependencyNotFile, pipelineTestingDependencyProblem(dependency), dependency) + } + + needsRequirementOption := []string{ + "/Workspace/orders/requirements.txt", + "--editable /Workspace/orders/requirements.txt", + "./tests/requirements.txt", + } + for _, dependency := range needsRequirementOption { + assert.Equal(t, pipelineTestingDependencyNeedsRequirementOption, pipelineTestingDependencyProblem(dependency), dependency) + } +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a1a4961fb11..03998ceb7f9 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -234,6 +234,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { mutator.TranslatePaths(), + // Reject the pipelines' testing.environment.dependencies that the API rejects. Runs after + // TranslatePaths, so that it checks the paths that deploy sends. + validate.PipelineTestingDependencies(), + // Reads (typed): b.Config.Experimental.PythonWheelWrapper, b.Config.Workspace.SyncMode (checks Python wheel wrapper and deployment mode settings) // Reads (dynamic): resources.jobs.*.tasks (checks for tasks with local libraries and incompatible DBR versions) // Provides warnings when Python wheel tasks are used with DBR < 13.3 or when wheel wrapper is incompatible with source-linked deployment