From 3d29310bcb5a2b6962df8e98ae5441150e542320 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 2 Sep 2026 15:09:16 +0000 Subject: [PATCH 1/2] Add acceptance test: bundle deploy --plan skips wheel upload Applying a saved plan skips the build phase, which computes the artifact upload list, so a local wheel referenced by a job is never uploaded. The test contrasts a normal deploy (wheel uploaded) with a --plan deploy (wheel missing) and documents the bug until it is fixed. Co-authored-by: Isaac --- .../readplan/whl-not-uploaded/databricks.yml | 20 +++ .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin 0 -> 1832 bytes .../readplan/whl-not-uploaded/out.plan.json | 45 +++++++ .../readplan/whl-not-uploaded/out.test.toml | 3 + .../readplan/whl-not-uploaded/output.txt | 116 ++++++++++++++++++ .../deploy/readplan/whl-not-uploaded/script | 25 ++++ .../readplan/whl-not-uploaded/test.toml | 4 + 7 files changed, 213 insertions(+) create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/script create mode 100644 acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml new file mode 100644 index 00000000000..f67f45222fb --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: whl-not-uploaded + +resources: + jobs: + my_job: + name: whl-not-uploaded-job + tasks: + - task_key: main + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: Standard_DS3_v2 + num_workers: 1 + python_wheel_task: + package_name: my_test_code + entry_point: run + libraries: + # Local wheel, prebuilt (no build: step) to keep the test hermetic. See script + # for the bug this exercises. Behaves the same as a wheel with a build: step. + - whl: ./dist/*.whl diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/deploy/readplan/whl-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..4bb80477caf51393354453b18c0c88711098825e GIT binary patch literal 1832 zcmWIWW@Zs#U|`^2xLh&ryk0?NcfY>3uGYCT z=ezsEH-rWlT`<029P+5E-(N@fdGY9=0ryhVJ$tvvUNQc+{d3{D zn|B2?ro>EK#LF7E*yLqccPAsi^Am}D%e30AoeArVlYJ((&Chx9t8w97&hq5+iY(pM z?WKI0`Ng-xP8DwL5?S^qs;1iP{ao7%o!XY-3yZ&e5Ut|$zxe;x^oV^&AD++Fd7iem zR(X?DP}|ALQzYCuG|lZak1|#q*u;_lsY}J+e|O){(NT&M^DqET*~S;Ni)SUsN4C z9Dn1{?j=1*vsGR>2Ap1Urdi@ME2@w3r*1Nd0r~`ll?eGL+{4w?hx@#T@7asIkTkJ5 z$l!|cgChU4-nw2oC-pZ4d3c@F(d*P_Um1A8;DWK?CF2WExK5qdzxY+>8n1?~*V&Uk zn}RenwS1p)dHQS*(pX_~?d<9E-dBCktbX!{2`Tk}Lc4^`-&YapE?q)yOU)}Os*Eql z&&(?+)+?zf>Gsnum6bPz^n=Yu)|leb1cf?%&Q77=vb)*4}*!*?@Mi z0I?P!yGrs4;&W2VQgc8SY3q4CLw9*{w)FQ{Mg|65CI$v&LZ$_|I{OE?w4OU%)MUWm zaB=T_A&mrw*ph>*mAaB@F8Q(@M!#{WU5W^A%QRiSMW z{ozJ=^Fq0UM|XJ53fr~pZf|;Euh=Z{Z-Tx4a;F>EwmRNScHD8mvs7)8lJ+L8`I_QF zshMAec0^w`-2TiwZ^_?(T*a4Um+uz%9hJHAo6n-@<%iYQ%+*hCy{@^uvZID|#pH|s ze;h2ndvtru#jke9t4>^Ne8ji;gOB{FCkgtet-sDum^X=S@}n%?$yyAP)vvd7E<0Z8 zI`hG%7f1XLTdv`o%)7mde~0q)?pa+LTXvaz|5|P@sQzMo+2)nERD@WZf7CmNnCf|W zcinFk%sFDWcj{{;(KVwNY7ngq3`-iL;hNz^9I|F?B?v;ZHZZ*qRi2<5iJpZK zM*byaBsfE(n}?q55a!JUCTF}(M9+liMx*C4gwd?b1dK+>bLi%x=Qo(STN;B2nG4B% V0p6^j+|R%cgo;34egu^R3;+-1eXjrj literal 0 HcmV?d00001 diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json new file mode 100644 index 00000000000..da6e6cae939 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json @@ -0,0 +1,45 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "plan": { + "resources.jobs.my_job": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "whl-not-uploaded-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "main" + } + ] + } + } + } + } +} diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt new file mode 100644 index 00000000000..9cca951a672 --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt @@ -0,0 +1,116 @@ + +=== bundle deploy uploads the wheel +>>> [CLI] bundle deploy +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "whl-not-uploaded-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "main" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default + +Destroy: 1 deleted + +=== bundle deploy --plan does NOT upload the wheel (bug) +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan out.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/files... +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "whl-not-uploaded-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + }, + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "main" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script new file mode 100644 index 00000000000..8253331b3ce --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -0,0 +1,25 @@ +# A local wheel referenced by a job library is uploaded by `bundle deploy` but NOT by +# `bundle deploy --plan`: applying a saved plan skips the build phase, which is what +# computes the artifact upload list (phases.Build -> libraries.ReplaceWithRemotePath). +# The job still points at the wheel's workspace path, so its run fails at runtime with a +# missing-file error. This test pins the difference between the two deploy paths. + +title "bundle deploy uploads the wheel" +trace $CLI bundle deploy +# The job references the wheel path, and an import-file request uploaded it there. +# raw_body is the wheel's binary payload; drop it to keep the golden readable. +trace print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body + +trace $CLI bundle destroy --auto-approve +# Discard destroy's requests so the saved-plan deploy below starts from a clean recording. +rm out.requests.txt + +title "bundle deploy --plan does NOT upload the wheel (bug)" +trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle deploy --plan out.plan.json +# BUG: the same .../artifacts/.internal/*.whl upload is missing here. Only jobs/create +# appears, still pointing the task at a wheel path that was never uploaded. +trace print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml new file mode 100644 index 00000000000..f987e32edfd --- /dev/null +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml @@ -0,0 +1,4 @@ +# --plan is direct-engine only. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true +Ignore = [".databricks"] From a0d5a84aa4149d058ccff7f1f677aaefb13dc570 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 3 Sep 2026 09:02:33 +0000 Subject: [PATCH 2/2] Address review: add Badness, filter to wheel import only Document the captured bug via test.toml Badness, and narrow the recorded requests to the wheel's artifact upload so its absence under --plan is obvious in the golden. Co-authored-by: Isaac --- .../readplan/whl-not-uploaded/output.txt | 76 +------------------ .../deploy/readplan/whl-not-uploaded/script | 10 +-- .../readplan/whl-not-uploaded/test.toml | 2 + 3 files changed, 9 insertions(+), 79 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt index 9cca951a672..73e36e6a842 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/output.txt @@ -7,7 +7,7 @@ Created jobs.my_job Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body +>>> print_requests.py //artifacts/.internal --del-field raw_body { "method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", @@ -15,42 +15,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "overwrite": "true" } } -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "whl-not-uploaded-job", - "queue": { - "enabled": true - }, - "tasks": [ - { - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" - } - ], - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 1, - "spark_version": "15.4.x-scala2.12" - }, - "python_wheel_task": { - "entry_point": "run", - "package_name": "my_test_code" - }, - "task_key": "main" - } - ] - } -} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -69,43 +33,7 @@ Created jobs.my_job Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "whl-not-uploaded-job", - "queue": { - "enabled": true - }, - "tasks": [ - { - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/whl-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" - } - ], - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 1, - "spark_version": "15.4.x-scala2.12" - }, - "python_wheel_task": { - "entry_point": "run", - "package_name": "my_test_code" - }, - "task_key": "main" - } - ] - } -} +>>> print_requests.py //artifacts/.internal --del-field raw_body >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script index 8253331b3ce..bd507766e34 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/script +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/script @@ -6,9 +6,9 @@ title "bundle deploy uploads the wheel" trace $CLI bundle deploy -# The job references the wheel path, and an import-file request uploaded it there. +# An import-file request uploaded the wheel to its .../artifacts/.internal/ workspace path. # raw_body is the wheel's binary payload; drop it to keep the golden readable. -trace print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body +trace print_requests.py //artifacts/.internal --del-field raw_body trace $CLI bundle destroy --auto-approve # Discard destroy's requests so the saved-plan deploy below starts from a clean recording. @@ -17,9 +17,9 @@ rm out.requests.txt title "bundle deploy --plan does NOT upload the wheel (bug)" trace $CLI bundle plan -o json > out.plan.json trace $CLI bundle deploy --plan out.plan.json -# BUG: the same .../artifacts/.internal/*.whl upload is missing here. Only jobs/create -# appears, still pointing the task at a wheel path that was never uploaded. -trace print_requests.py //jobs //artifacts/.internal --sort --del-field raw_body +# BUG: no import-file request for the wheel here (output below is empty), yet the job was +# created pointing the task at the same .../artifacts/.internal/ path that was never uploaded. +trace print_requests.py //artifacts/.internal --del-field raw_body trace $CLI bundle destroy --auto-approve rm out.requests.txt diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml index f987e32edfd..9325fa161c0 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/test.toml @@ -1,3 +1,5 @@ +Badness = "bundle deploy --plan skips the build phase, so a local wheel referenced by a job is never uploaded and the job fails at runtime." + # --plan is direct-engine only. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true