Translate the testing paths of pipelines and upload local test wheels

acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: pipeline-testing-same-name-libraries
+
+resources:
+  pipelines:
+    orders:
+      name: orders
+      libraries:
+        - file:
+            path: ./transformations/orders.py
+      environment:
+        dependencies:
+          - ./pipeline/test_helpers-0.1-py3-none-any.whl
+      testing:
+        sources:
+          - glob:
+              include: ./tests/**
+        environment:
+          dependencies:
+            # A different wheel with the same file name would be uploaded to the same path.
+            - ./tests/test_helpers-0.1-py3-none-any.whl

acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt

@@ -0,0 +1,10 @@
+
+>>> musterr [CLI] bundle deploy
+Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl
+  at resources.pipelines.orders.environment.dependencies[0]
+     resources.pipelines.orders.testing.environment.dependencies[0]
+  in databricks.yml:13:13
+     databricks.yml:21:15
+
+Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl
+

acceptance/bundle/resources/pipelines/testing-same-name-libraries/script

@@ -0,0 +1 @@
+trace musterr $CLI bundle deploy

acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml

@@ -0,0 +1,7 @@
+# Runs only against the local test server.
+Cloud = false
+Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true"
+
+[[Repls]]
+Old = '\\'
+New = '/'

acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py

@@ -0,0 +1,2 @@
+def test_orders():
+    pass

acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py

@@ -0,0 +1 @@
+x = 1

acceptance/bundle/resources/pipelines/testing/output.txt

@@ -1,17 +1,19 @@
 
+=== Relative test paths resolve to the deployed workspace paths
+
 >>> [CLI] bundle validate -o json
 {
   "environment": {
     "dependencies": [
-      "-e .[test]",
-      "-r ./tests/requirements.txt",
+      "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]",
+      "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt",
       "./dist/*.whl"
     ]
   },
   "sources": [
     {
       "glob": {
-        "include": "./tests/**"
+        "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**"
       }
     }
   ]

acceptance/bundle/resources/pipelines/testing/script

@@ -1,3 +1,4 @@
+title "Relative test paths resolve to the deployed workspace paths\n"
 trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing
 
 title "Without pipeline testing, the property is an error\n"

bundle/config/mutator/paths/pipeline_paths_visitor.go

@@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern {
 			mode:        TranslateModeDirectory,
 			skipRewrite: noSkipRewrite,
 		},
+		{
+			pattern:     base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")),
+			mode:        TranslateModeGlob,
+			skipRewrite: noSkipRewrite,
+		},
 	}
 
 	return allPatterns
@@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern {
 				return !libraries.IsLibraryLocal(s)
 			},
 		},
+		// Test dependencies take the same forms as the pipeline's environment dependencies.
+		{
+			pattern:     base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
+			mode:        TranslateModeLocalRelativeWithPrefix,
+			skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) },
+		},
 	}
 
 	pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{
@@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern {
 				return !ok
 			},
 		},
+		{
+			base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()),
+			TranslateModeEnvironmentPipFlag,
+			func(s string) bool {
+				_, _, ok := libraries.IsLocalPathInPipFlag(s)
+				return !ok
+			},
+		},
 	}
 
 	return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...)

bundle/config/mutator/paths/pipeline_paths_visitor_test.go

@@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) {
 							},
 						},
 					},
+					// The acceptance test bundle/resources/pipelines/testing covers the testing paths
+					// that are rewritten. A package name is not a path.
+					Testing: &resources.PipelineTesting{
+						Environment: &resources.PipelineTestEnvironment{
+							Dependencies: []string{"faker"},
+						},
+					},
 				},
 			},
 		},

bundle/libraries/expand_glob_references.go

@@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern(
 	dyn.Key("dependencies"),
 )
 
+var pipelineTestingEnvDepsPattern = dyn.NewPattern(
+	dyn.Key("resources"),
+	dyn.Key("pipelines"),
+	dyn.AnyKey(),
+	dyn.Key("testing"),
+	dyn.Key("environment"),
+	dyn.Key("dependencies"),
+)
+
 var clusterLibrariesPattern = dyn.NewPattern(
 	dyn.Key("resources"),
 	dyn.Key("clusters"),
@@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 			pattern: pipelineEnvDepsPattern,
 			fn:      expandEnvironmentDeps,
 		},
+		{
+			pattern: pipelineTestingEnvDepsPattern,
+			fn:      expandEnvironmentDeps,
+		},
 		{
 			pattern: clusterLibrariesPattern,
 			fn:      expandLibraries,

bundle/libraries/remote_path.go

@@ -143,6 +143,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err
 		clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")),
 		envDepsPattern.Append(dyn.AnyIndex()),
 		pipelineEnvDepsPattern.Append(dyn.AnyIndex()),
+		pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()),
 		// The AI Runtime task's code_source_path is a local archive (typically an
 		// artifact-built .tar.gz) that must be uploaded and referenced by its remote
 		// path, exactly like a wheel or jar library.

bundle/libraries/same_name_libraries.go

@@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{
 	clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")),
 	envDepsPattern.Append(dyn.AnyIndex()),
 	pipelineEnvDepsPattern.Append(dyn.AnyIndex()),
+	pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()),
 }
 
 type libData struct {