Map a project folder after -e in environment dependencies to its deployed folder

acceptance/bundle/environments/editable-folder/databricks.yml

@@ -0,0 +1,26 @@
+bundle:
+  name: editable-folder
+
+resources:
+  jobs:
+    my_job:
+      name: my job
+      tasks:
+        - task_key: main
+          python_wheel_task:
+            package_name: orders
+            entry_point: main
+          environment_key: default
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+            dependencies:
+              - -e ./project
+
+  pipelines:
+    my_pipeline:
+      name: my pipeline
+      environment:
+        dependencies:
+          - -e ./project[test]

acceptance/bundle/environments/editable-folder/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/environments/editable-folder/output.txt

@@ -0,0 +1,15 @@
+
+=== With pipeline testing, a folder after -e maps to its deployed folder
+
+>>> [CLI] bundle validate -o json
+{
+  "job": [
+    "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project"
+  ],
+  "pipeline": [
+    "-e /Workspace/Users/[USERNAME]/.bundle/editable-folder/default/files/project[test]"
+  ]
+}
+
+=== Without pipeline testing, a folder after -e is an error, as before
+Error: unable to determine if [TEST_TMP_DIR]/project is not a notebook

acceptance/bundle/environments/editable-folder/project/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "orders"
+version = "0.1.0"
+
+[project.optional-dependencies]
+test = ["faker"]

acceptance/bundle/environments/editable-folder/script

@@ -0,0 +1,7 @@
+title "With pipeline testing, a folder after -e maps to its deployed folder\n"
+trace $CLI bundle validate -o json | jq '{job: .resources.jobs.my_job.environments[0].spec.dependencies, pipeline: .resources.pipelines.my_pipeline.environment.dependencies}'
+
+title "Without pipeline testing, a folder after -e is an error, as before\n"
+# The OS error after the message differs on Windows.
+musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate &> LOG.gate-off
+grep -o 'Error: unable to determine if .* is not a notebook' LOG.gate-off

acceptance/bundle/environments/editable-folder/test.toml

@@ -0,0 +1,7 @@
+Cloud = false
+Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true"
+
+# Windows paths in the gate-off error.
+[[Repls]]
+Old = '\\'
+New = '/'

bundle/config/mutator/translate_paths.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/env"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/notebook"
@@ -162,6 +163,15 @@ func (t *translateContext) rewritePath(
 	localPath = filepath.ToSlash(localPath)
 	localRelPath = filepath.ToSlash(localRelPath)
 
+	// With pipeline testing enabled, the path after -e can also be a project folder, with optional
+	// extras such as ./project[test]. The result is not cached: the cache key has no flag, so a
+	// later "-r ./project" would get the "-e" result instead of its error.
+	if opts.Mode == paths.TranslateModeEnvironmentPipFlag && flag == "-e" && env.ExperimentalPipelineTesting(ctx) {
+		if interp, ok := t.translateEditableFolder(localRelPath); ok {
+			return interp, nil
+		}
+	}
+
 	// Convert local path into workspace path via specified function.
 	var interp string
 	switch opts.Mode {
@@ -251,6 +261,20 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local
 	return path.Join(t.remoteRoot, localRelPath), nil
 }
 
+// translateEditableFolder maps a project folder after -e, with optional extras such as [test], to
+// its deployed folder. It reports false if the path is not a folder in the sync root.
+func (t *translateContext) translateEditableFolder(localRelPath string) (string, bool) {
+	folder, extras := localRelPath, ""
+	if i := strings.LastIndex(localRelPath, "["); i > 0 && strings.HasSuffix(localRelPath, "]") {
+		folder, extras = localRelPath[:i], localRelPath[i:]
+	}
+	info, err := t.b.SyncRoot.Stat(folder)
+	if err != nil || !info.IsDir() {
+		return "", false
+	}
+	return "-e " + path.Join(t.remoteRoot, folder) + extras, true
+}
+
 func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) {
 	if t.skipLocalFileValidation {
 		return path.Join(t.remoteRoot, localRelPath), nil

bundle/config/mutator/translate_paths_test.go

@@ -8,6 +8,9 @@ import (
 	"runtime"
 	"testing"
 
+	bundleenv "github.com/databricks/cli/bundle/env"
+	libsenv "github.com/databricks/cli/libs/env"
+
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
@@ -816,6 +819,58 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
 	assert.Equal(t, "https://foo@bar.test/packages/pypi/simple", b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies[6])
 }
 
+func TestTranslatePathEditableProjectFolder(t *testing.T) {
+	dir := t.TempDir()
+	touchEmptyFile(t, filepath.Join(dir, "project", "pyproject.toml"))
+	touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
+
+	translate := func(ctx context.Context, dependencies ...string) (*bundle.Bundle, diag.Diagnostics) {
+		b := &bundle.Bundle{
+			SyncRootPath:   dir,
+			BundleRootPath: dir,
+			SyncRoot:       vfs.MustNew(dir),
+			Config: config.Root{
+				Workspace: config.Workspace{
+					FilePath: "/bundle",
+				},
+				Resources: config.Resources{
+					Pipelines: map[string]*resources.Pipeline{
+						"pipeline": {
+							CreatePipeline: pipelines.CreatePipeline{
+								Environment: &pipelines.PipelinesEnvironment{Dependencies: dependencies},
+							},
+						},
+					},
+				},
+			},
+		}
+		bundletest.SetLocation(b, ".", []diag.Location{{File: filepath.Join(dir, "fake.yml")}})
+		return b, bundle.ApplySeq(ctx, b, mutator.NormalizePaths(), mutator.TranslatePaths())
+	}
+
+	// Without pipeline testing, a folder after -e stays an error, as it was before the feature.
+	_, diags := translate(t.Context(), "-e ./project")
+	assert.ErrorContains(t, diags.Error(), "unable to determine if "+filepath.Join(dir, "project")+" is not a notebook")
+
+	ctx := libsenv.Set(t.Context(), bundleenv.ExperimentalPipelineTestingVariable, "true")
+	b, diags := translate(ctx, "-e ./project", "-e ./project[test]", "-e .", "-e .[test]", "-r ./requirements.txt")
+	require.NoError(t, diags.Error())
+	assert.Equal(t, []string{
+		"-e /bundle/project",
+		"-e /bundle/project[test]",
+		"-e /bundle",
+		"-e /bundle[test]",
+		"-r /bundle/requirements.txt",
+	}, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies)
+
+	// -r takes a file, so a folder after it is still an error, also after the same folder after -e.
+	notebookErr := "unable to determine if " + filepath.Join(dir, "project") + " is not a notebook"
+	_, diags = translate(ctx, "-r ./project")
+	assert.ErrorContains(t, diags.Error(), notebookErr)
+	_, diags = translate(ctx, "-e ./project", "-r ./project")
+	assert.ErrorContains(t, diags.Error(), notebookErr)
+}
+
 func TestTranslatePathWithComplexVariables(t *testing.T) {
 	dir := t.TempDir()
 	b := &bundle.Bundle{

bundle/env/experimental_pipeline_testing.go

@@ -0,0 +1,17 @@
+package env
+
+import (
+	"context"
+
+	envlib "github.com/databricks/cli/libs/env"
+)
+
+// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines.
+const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING"
+
+// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is
+// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed.
+func ExperimentalPipelineTesting(ctx context.Context) bool {
+	enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable)
+	return enabled
+}