diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml new file mode 100644 index 00000000000..f67f45222fb --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: whl-not-uploaded + +resources: + jobs: + my_job: + name: whl-not-uploaded-job + tasks: + - task_key: main + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: Standard_DS3_v2 + num_workers: 1 + python_wheel_task: + package_name: my_test_code + entry_point: run + libraries: + # Local wheel, prebuilt (no build: step) to keep the test hermetic. See script + # for the bug this exercises. Behaves the same as a wheel with a build: step. + - whl: ./dist/*.whl diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 00000000000..4bb80477caf Binary files /dev/null and b/acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl differ diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json new file mode 100644 index 00000000000..da6e6cae939 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json @@ -0,0 +1,45 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "plan": { + "resources.jobs.my_job": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "whl-not-uploaded-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "main" + } + ] + } + } + } + } +} diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt new file mode 100644 index 00000000000..73e36e6a842 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt @@ -0,0 +1,44 @@ + +=== bundle deploy uploads the wheel +>>> [CLI] bundle deploy +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //artifacts/.internal --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default + +Destroy: 1 deleted + +=== bundle deploy --plan does NOT upload the wheel (bug) +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan out.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //artifacts/.internal --del-field raw_body + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script new file mode 100644 index 00000000000..bd507766e34 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -0,0 +1,25 @@ +# A local wheel referenced by a job library is uploaded by `bundle deploy` but NOT by +# `bundle deploy --plan`: applying a saved plan skips the build phase, which is what +# computes the artifact upload list (phases.Build -> libraries.ReplaceWithRemotePath). +# The job still points at the wheel's workspace path, so its run fails at runtime with a +# missing-file error. This test pins the difference between the two deploy paths. + +title "bundle deploy uploads the wheel" +trace $CLI bundle deploy +# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. +# raw_body is the wheel's binary payload; drop it to keep the golden readable. +trace print_requests.py //artifacts/.internal --del-field raw_body + +trace $CLI bundle destroy --auto-approve +# Discard destroy's requests so the saved-plan deploy below starts from a clean recording. +rm out.requests.txt + +title "bundle deploy --plan does NOT upload the wheel (bug)" +trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle deploy --plan out.plan.json +# BUG: no import-file request for the wheel here (output below is empty), yet the job was +# created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. +trace print_requests.py //artifacts/.internal --del-field raw_body + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml new file mode 100644 index 00000000000..9325fa161c0 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml @@ -0,0 +1,6 @@ +Badness = "bundle deploy --plan skips the build phase, so a local wheel referenced by a job is never uploaded and the job fails at runtime." + +# --plan is direct-engine only. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true +Ignore = [".databricks"]