diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..68399999172 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..32f10d10630 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d018f5851ee --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,7 @@ +# Runs only against the local test server. +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index 6133e93eb75..02708b42ed8 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -1,17 +1,19 @@ +=== Relative test paths resolve to the deployed workspace paths + >>> [CLI] bundle validate -o json { "environment": { "dependencies": [ - "-e .[test]", - "-r ./tests/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", "./dist/*.whl" ] }, "sources": [ { "glob": { - "include": "./tests/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" } } ] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index bd798996774..245a0693110 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1,3 +1,4 @@ +title "Relative test paths resolve to the deployed workspace paths\n" trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing title "Without pipeline testing, the property is an error\n" diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..83be40afd7d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns @@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !libraries.IsLibraryLocal(s) }, }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }, } pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ @@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !ok }, }, + { + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, + }, } return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..85c29065c5b 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -40,6 +40,13 @@ func TestVisitPipelinePaths(t *testing.T) { }, }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &resources.PipelineTesting{ + Environment: &resources.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 70e577a1a2a..359744328b4 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -143,6 +143,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct {