From 9a2cc3ca6b82f2c8567af26543ba9480bafce06c Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 10 Nov 2026 00:13:46 +0000 Subject: [PATCH] Map a project folder after -e in environment dependencies to its deployed folder With DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true, a local project folder after -e, with optional extras such as -e ./project[test], maps to its deployed folder instead of failing. This applies to job and pipeline environment dependencies. With the variable off, the old behavior stays. Co-authored-by: Isaac --- .../editable-folder/databricks.yml | 26 +++++++++ .../editable-folder/out.test.toml | 2 + .../environments/editable-folder/output.txt | 15 +++++ .../editable-folder/project/pyproject.toml | 6 ++ .../environments/editable-folder/script | 7 +++ .../environments/editable-folder/test.toml | 7 +++ bundle/config/mutator/translate_paths.go | 24 ++++++++ bundle/config/mutator/translate_paths_test.go | 55 +++++++++++++++++++ bundle/env/experimental_pipeline_testing.go | 17 ++++++ 9 files changed, 159 insertions(+) create mode 100644 acceptance/bundle/environments/editable-folder/databricks.yml create mode 100644 acceptance/bundle/environments/editable-folder/out.test.toml create mode 100644 acceptance/bundle/environments/editable-folder/output.txt create mode 100644 acceptance/bundle/environments/editable-folder/project/pyproject.toml create mode 100644 acceptance/bundle/environments/editable-folder/script create mode 100644 acceptance/bundle/environments/editable-folder/test.toml create mode 100644 bundle/env/experimental_pipeline_testing.go diff --git a/acceptance/bundle/environments/editable-folder/databricks.yml b/acceptance/bundle/environments/editable-folder/databricks.yml new file mode 100644 index 00000000000..b6c43e86868 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/databricks.yml @@ -0,0 +1,26 @@ +bundle: + name: editable-folder + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: main + python_wheel_task: + package_name: orders + entry_point: main + environment_key: default + environments: + - environment_key: default + spec: + client: "1" + dependencies: + - -e ./project + + pipelines: + my_pipeline: + name: my pipeline + environment: + dependencies: + - -e ./project[test] diff --git a/acceptance/bundle/environments/editable-folder/out.test.toml b/acceptance/bundle/environments/editable-folder/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/environments/editable-folder/output.txt b/acceptance/bundle/environments/editable-folder/output.txt new file mode 100644 index 00000000000..acd2f1ee237 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/output.txt @@ -0,0 +1,15 @@ + +=== With pipeline testing, a folder after -e maps to its deployed folder + +>>> [CLI] bundle validate -o json +{ + "job": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project" + ], + "pipeline": [ + "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project[test]" + ] +} + +=== Without pipeline testing, a folder after -e is an error, as before +Error: unable to determine if [TEST_TMP_DIR]/project is not a notebook diff --git a/acceptance/bundle/environments/editable-folder/project/pyproject.toml b/acceptance/bundle/environments/editable-folder/project/pyproject.toml new file mode 100644 index 00000000000..468fddd1ca7 --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/project/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1.0" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/environments/editable-folder/script b/acceptance/bundle/environments/editable-folder/script new file mode 100644 index 00000000000..a9a2c4a9e4b --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/script @@ -0,0 +1,7 @@ +title "With pipeline testing, a folder after -e maps to its deployed folder\n" +trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies}' + +title "Without pipeline testing, a folder after -e is an error, as before\n" +# The OS error after the message differs on Windows. +musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate &> LOG.gate-off +grep -o 'Error: unable to determine if .* is not a notebook' LOG.gate-off diff --git a/acceptance/bundle/environments/editable-folder/test.toml b/acceptance/bundle/environments/editable-folder/test.toml new file mode 100644 index 00000000000..de87a97132d --- /dev/null +++ b/acceptance/bundle/environments/editable-folder/test.toml @@ -0,0 +1,7 @@ +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" + +# Windows paths in the gate-off error. +[[Repls]] +Old = '\\' +New = '/' diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 956d010ec95..774fa7a1919 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/notebook" @@ -162,6 +163,15 @@ func (t *translateContext) rewritePath( localPath = filepath.ToSlash(localPath) localRelPath = filepath.ToSlash(localRelPath) + // With pipeline testing enabled, the path after -e can also be a project folder, with optional + // extras such as ./project[test]. The result is not cached: the cache key has no flag, so a + // later "-r ./project" would get the "-e" result instead of its error. + if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" && env.ExperimentalPipelineTesting(ctx) { + if interp, ok := t.translateEditableFolder(localRelPath); ok { + return interp, nil + } + } + // Convert local path into workspace path via specified function. var interp string switch opts.Mode { @@ -251,6 +261,20 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local return path.Join(t.remoteRoot, localRelPath), nil } +// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to +// its deployed folder. It reports false if the path is not a folder in the sync root. +func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) { + folder, extras := localRelPath, "" + if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") { + folder, extras = localRelPath[:i], localRelPath[i:] + } + info, err := t.b.SyncRoot.Stat(folder) + if err != nil || !info.IsDir() { + return "", false + } + return "-e " + path.Join(t.remoteRoot, folder) + extras, true +} + func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { if t.skipLocalFileValidation { return path.Join(t.remoteRoot, localRelPath), nil diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8aaf9ce9c71..fdaf2ac552b 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -8,6 +8,9 @@ import ( "runtime" "testing" + bundleenv "github.com/databricks/cli/bundle/env" + libsenv "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" @@ -816,6 +819,58 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6]) } +func TestTranslatePathEditableProjectFolder(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) + + translate := func(ctx context.Context, dependencies ...string) (*bundle.Bundle, diag.Diagnostics) { + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies}, + }, + }, + }, + }, + }, + } + bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}}) + return b, bundle.ApplySeq(ctx, b, mutator.NormalizePaths(), mutator.TranslatePaths()) + } + + // Without pipeline testing, a folder after -e stays an error, as it was before the feature. + _, diags := translate(t.Context(), "-e ./project") + assert.ErrorContains(t, diags.Error(), "unable to determine if "+filepath.Join(dir, "project")+" is not a notebook") + + ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true") + b, diags := translate(ctx, "-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt") + require.NoError(t, diags.Error()) + assert.Equal(t, []string{ + "-e /bundle/project", + "-e /bundle/project[test]", + "-e /bundle", + "-e /bundle[test]", + "-r /bundle/requirements.txt", + }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) + + // -r takes a file, so a folder after it is still an error, also after the same folder after -e. + notebookErr := "unable to determine if " + filepath.Join(dir, "project") + " is not a notebook" + _, diags = translate(ctx, "-r ./project") + assert.ErrorContains(t, diags.Error(), notebookErr) + _, diags = translate(ctx, "-e ./project", "-r ./project") + assert.ErrorContains(t, diags.Error(), notebookErr) +} + func TestTranslatePathWithComplexVariables(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go new file mode 100644 index 00000000000..b00a94176e9 --- /dev/null +++ b/bundle/env/experimental_pipeline_testing.go @@ -0,0 +1,17 @@ +package env + +import ( + "context" + + envlib "github.com/databricks/cli/libs/env" +) + +// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. +const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" + +// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is +// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. +func ExperimentalPipelineTesting(ctx context.Context) bool { + enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) + return enabled +}