diff --git a/.nextchanges/bundles/builtin-templates-workspace-url.md b/.nextchanges/bundles/builtin-templates-workspace-url.md new file mode 100644 index 00000000000..76e97992a1e --- /dev/null +++ b/.nextchanges/bundles/builtin-templates-workspace-url.md @@ -0,0 +1 @@ +* Built-in templates add the workspace ID to `workspace.host` (`?w=`) when the profile or an environment variable sets it, so generated bundles select the right workspace on hosts shared by several workspaces. Welcome messages show the same URL, and `dbt-sql` also adds `?o=` to the HTTP path it suggests for `dbt init`. ([#7350](https://github.com/databricks/cli/pull/7350)) diff --git a/.nextchanges/bundles/workspace-template-urlid.md b/.nextchanges/bundles/workspace-template-urlid.md new file mode 100644 index 00000000000..f5a8fbca631 --- /dev/null +++ b/.nextchanges/bundles/workspace-template-urlid.md @@ -0,0 +1 @@ +* Add `.workspace.URLID` to bundle templates: the workspace ID that `.workspace.URL` adds, or empty, so templates can use it without failing when the ID is unknown. ([#7388](https://github.com/databricks/cli/pull/7388)) diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.protocol-relative.json b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.protocol-relative.json new file mode 100644 index 00000000000..f31d8b9756a --- /dev/null +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.protocol-relative.json @@ -0,0 +1 @@ +{"path": "//other.test/jobs"} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-w.json b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-w.json new file mode 100644 index 00000000000..89f9c8c81ab --- /dev/null +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-w.json @@ -0,0 +1 @@ +{"path": "/jobs?w=123"} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-workspace-id.json b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-workspace-id.json new file mode 100644 index 00000000000..9ab3ccc9fcc --- /dev/null +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-workspace-id.json @@ -0,0 +1 @@ +{"path": "/jobs?workspace_id=123"} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.supplied.json b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.supplied.json new file mode 100644 index 00000000000..cb4e5d152c5 --- /dev/null +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/input.supplied.json @@ -0,0 +1 @@ +{"value": "set by user"} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/output.txt b/acceptance/bundle/templates-machinery/workspace_variable/runtime/output.txt index c6980cdead1..eec7fe20829 100644 --- a/acceptance/bundle/templates-machinery/workspace_variable/runtime/output.txt +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/output.txt @@ -25,6 +25,7 @@ Hostname: [DATABRICKS_HOST] URL: [DATABRICKS_URL] URL "/jobs": [DATABRICKS_URL]/jobs URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list#top +URLID: "" jobs_url: [DATABRICKS_URL]/jobs Root keys: jobs_url path workspace Root data: map[jobs_url:[DATABRICKS_URL]/jobs path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] [WORKSPACE_ID] }] @@ -55,6 +56,7 @@ Hostname: [DATABRICKS_HOST] URL: [DATABRICKS_URL]?w=[WORKSPACE_ID] URL "/jobs": [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID] URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top +URLID: "[WORKSPACE_ID]" jobs_url: [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID] Root keys: jobs_url path workspace Root data: map[jobs_url:[DATABRICKS_URL]/jobs?w=[WORKSPACE_ID] path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] [WORKSPACE_ID] [WORKSPACE_ID]}] @@ -85,6 +87,7 @@ Hostname: [CUSTOM_HOSTNAME] URL: [CUSTOM_HOST]?w=[WORKSPACE_ID] URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] URL "/jobs?view=list#top": [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top +URLID: "[WORKSPACE_ID]" jobs_url: [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] Root keys: jobs_url path workspace Root data: map[jobs_url:[CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] path:/jobs?view=list#top workspace:{[CUSTOM_HOSTNAME] [CUSTOM_HOST] [WORKSPACE_ID] [WORKSPACE_ID]}] @@ -115,6 +118,7 @@ Hostname: [DATABRICKS_HOST] URL: [DATABRICKS_URL] URL "/jobs": [DATABRICKS_URL]/jobs URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list#top +URLID: "" jobs_url: [DATABRICKS_URL]/jobs Root keys: jobs_url path workspace Root data: map[jobs_url:[DATABRICKS_URL]/jobs path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] }] @@ -145,20 +149,54 @@ Hostname: [CUSTOM_HOSTNAME] URL: [CUSTOM_HOST]?w=[WORKSPACE_ID] URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] URL "/pipelines?tab=all": [CUSTOM_HOST]/pipelines?tab=all&w=[WORKSPACE_ID] +URLID: "[WORKSPACE_ID]" jobs_url: https://example.test/my-jobs Root keys: jobs_url path workspace Root data: map[jobs_url:https://example.test/my-jobs path:/pipelines?tab=all workspace:{[CUSTOM_HOSTNAME] [CUSTOM_HOST] [WORKSPACE_ID] [WORKSPACE_ID]}] +=== Failing default of an input with a value + +>>> [CLI] bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json +{ + "welcome_message": null, + "success_message": null, + "properties": { + "value": { + "description": "Value", + "default": "{{ .workspace.ID }}" + } + } +} + +>>> [CLI] bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json +✨ Successfully initialized template +value: set by user + +>>> musterr [CLI] bundle debug render-template-schema ./supplied --profile no_workspace_id +Error: property value default: failed to render template "{{ .workspace.ID }}": template: base:1:13: executing "base" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle + === Invalid URL paths >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.absolute-url.json Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID] Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "https://other.test/jobs" must be relative to the workspace host +>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.protocol-relative.json +Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID] +Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "//other.test/jobs" must be relative to the workspace host + >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.workspace-selector.json Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID] Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?o=123" must not set the "o" query parameter, the workspace is selected by the profile +>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.selector-w.json +Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID] +Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?w=123" must not set the "w" query parameter, the workspace is selected by the profile + +>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.selector-workspace-id.json +Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID] +Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?workspace_id=123" must not set the "workspace_id" query parameter, the workspace is selected by the profile + >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.semicolon.json Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID] Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: failed to parse query of workspace URL path "/jobs?w=999;x=1": invalid semicolon separator in query @@ -171,6 +209,7 @@ Open [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top URL: [CUSTOM_HOST]?w=[WORKSPACE_ID] URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] URL "/jobs?view=list#top": [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top +URLID: "[WORKSPACE_ID]" === Workspace ID from the bundle in the working directory @@ -180,6 +219,7 @@ Open [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top URL: [DATABRICKS_URL]?w=[WORKSPACE_ID] URL "/jobs": [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID] URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top +URLID: "[WORKSPACE_ID]" === ID @@ -192,6 +232,6 @@ ID: [WORKSPACE_ID] ID: [WORKSPACE_ID] >>> musterr [CLI] bundle init ./id --profile no_workspace_id -Error: failed to compute file content for id.txt.tmpl. template: :1:17: executing "" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile or the DATABRICKS_WORKSPACE_ID environment variable +Error: failed to compute file content for id.txt.tmpl. template: :1:17: executing "" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle >>> print_requests.py --get --oneline ^//telemetry-ext diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/script b/acceptance/bundle/templates-machinery/workspace_variable/runtime/script index 4a773ef943f..45e8bb48226 100644 --- a/acceptance/bundle/templates-machinery/workspace_variable/runtime/script +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/script @@ -15,10 +15,20 @@ trace $CLI bundle init ./urls --profile workspace_custom_host --config-file inpu cat urls.txt rm urls.txt +# A default that fails, here because the profile has no workspace ID, doesn't +# matter when the input has a value: init doesn't use it, and the schema keeps +# it as written. Without a value, the schema still fails. +title "Failing default of an input with a value\n" +trace $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json | schema_fields +trace $CLI bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json +cat value.txt +rm value.txt +trace musterr $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id + title "Invalid URL paths\n" -trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.absolute-url.json -trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.workspace-selector.json -trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.semicolon.json +for input in absolute-url protocol-relative workspace-selector selector-w selector-workspace-id semicolon; do + trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.$input.json +done # No profile, so that only the environment variable sets the ID. title "Workspace ID from environment variables\n" diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/databricks_template_schema.json b/acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/databricks_template_schema.json new file mode 100644 index 00000000000..c5cef0564be --- /dev/null +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/databricks_template_schema.json @@ -0,0 +1,9 @@ +{ + "properties": { + "value": { + "type": "string", + "description": "Value", + "default": "{{ .workspace.ID }}" + } + } +} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/template/value.txt.tmpl b/acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/template/value.txt.tmpl new file mode 100644 index 00000000000..4c952daa4cf --- /dev/null +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/template/value.txt.tmpl @@ -0,0 +1 @@ +value: {{ .value }} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/urls/template/urls.txt.tmpl b/acceptance/bundle/templates-machinery/workspace_variable/runtime/urls/template/urls.txt.tmpl index 539714bfc4a..f5ca6fa67cc 100644 --- a/acceptance/bundle/templates-machinery/workspace_variable/runtime/urls/template/urls.txt.tmpl +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/urls/template/urls.txt.tmpl @@ -4,6 +4,7 @@ Hostname: {{ .workspace.Hostname }} URL: {{ .workspace.URL }} URL "/jobs": {{ .workspace.URL "/jobs" }} URL "{{ .path }}": {{ $ws.URL .path }} +URLID: "{{ .workspace.URLID }}" jobs_url: {{ .jobs_url }} Root keys:{{ range $key, $value := . }} {{ $key }}{{ end }} Root data: {{ . }} diff --git a/acceptance/bundle/templates-machinery/workspace_variable/script.prepare b/acceptance/bundle/templates-machinery/workspace_variable/script.prepare index 531e2c9565e..ce84bfa6a19 100644 --- a/acceptance/bundle/templates-machinery/workspace_variable/script.prepare +++ b/acceptance/bundle/templates-machinery/workspace_variable/script.prepare @@ -1,9 +1,4 @@ -# The test server is reachable as both 127.0.0.1 and localhost. The second -# stands in for a host shared by several workspaces. -CUSTOM_HOST=${DATABRICKS_HOST/127.0.0.1/localhost} -add_repl "$CUSTOM_HOST" CUSTOM_HOST -add_repl "${CUSTOM_HOST#*://}" CUSTOM_HOSTNAME -WORKSPACE_ID=$(curl -s "$DATABRICKS_HOST/.well-known/databricks-config" | jq -r .workspace_id) +setup_custom_host # workspace_classic has no workspace_id, so the SDK resolves it from host # metadata. The "none" sentinel prevents that. diff --git a/acceptance/bundle/templates/dbt-sql-workspace-id/input.json b/acceptance/bundle/templates/dbt-sql-workspace-id/input.json new file mode 100644 index 00000000000..ca84320e6d7 --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-workspace-id/input.json @@ -0,0 +1,6 @@ +{ + "project_name": "my_dbt_sql", + "http_path": "/sql/2.0/warehouses/f00dcafe", + "default_catalog": "main", + "personal_schemas": "yes" +} diff --git a/acceptance/bundle/templates/dbt-sql-workspace-id/out.test.toml b/acceptance/bundle/templates/dbt-sql-workspace-id/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-workspace-id/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/templates/dbt-sql-workspace-id/output.txt b/acceptance/bundle/templates/dbt-sql-workspace-id/output.txt new file mode 100644 index 00000000000..836fba4f276 --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-workspace-id/output.txt @@ -0,0 +1,48 @@ + +=== Profile no_workspace_id + +>>> [CLI] bundle init dbt-sql --config-file ./input.json --profile no_workspace_id + +Welcome to the dbt template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [DATABRICKS_URL] + +📊 Your new project has been created in the 'my_dbt_sql' directory! +If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started. +Refer to the README.md file for full "getting started" guide and production setup instructions. + + +>>> grep host: my_dbt_sql/databricks.yml + host: [DATABRICKS_URL] + host: [DATABRICKS_URL] + +>>> grep -E ^ default: [^{] my_dbt_sql/profile_template.yml + default: [DATABRICKS_HOST] + default: /sql/2.0/warehouses/f00dcafe + default: main + default: 4 + +=== Profile custom_host + +>>> [CLI] bundle init dbt-sql --config-file ./input.json --profile custom_host + +Welcome to the dbt template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [CUSTOM_HOST]?w=[WORKSPACE_ID] + +📊 Your new project has been created in the 'my_dbt_sql' directory! +If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started. +Refer to the README.md file for full "getting started" guide and production setup instructions. + + +>>> grep host: my_dbt_sql/databricks.yml + host: [CUSTOM_HOST]?w=[WORKSPACE_ID] + host: [CUSTOM_HOST]?w=[WORKSPACE_ID] + +>>> grep -E ^ default: [^{] my_dbt_sql/profile_template.yml + default: [CUSTOM_HOSTNAME] + default: /sql/2.0/warehouses/f00dcafe?o=[WORKSPACE_ID] + default: main + default: 4 diff --git a/acceptance/bundle/templates/dbt-sql-workspace-id/script b/acceptance/bundle/templates/dbt-sql-workspace-id/script new file mode 100644 index 00000000000..f7cfa1bd350 --- /dev/null +++ b/acceptance/bundle/templates/dbt-sql-workspace-id/script @@ -0,0 +1,25 @@ +setup_custom_host + +cat > .databrickscfg <>> [CLI] bundle init default-minimal --config-file ./input.json --profile classic +Welcome to the minimal Declarative Automation Bundle template! + +This template creates a minimal project structure without sample code, ideal for advanced users. +(For getting started with Python or SQL code, use the default-python or default-sql templates instead.) + +Your workspace at [DATABRICKS_URL] is used for initialization. +(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.) + +✨ Your new project has been created in the 'my_default_minimal' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> grep host: databricks.yml + host: [DATABRICKS_URL] + host: [DATABRICKS_URL] + +>>> DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH=1 [CLI] bundle validate -t dev +Name: my_default_minimal +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev + +Validation OK! + +>>> [CLI] bundle deploy -t dev +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev/files... +Files: 10 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +=== Profile custom_host + +>>> [CLI] bundle init default-minimal --config-file ./input.json --profile custom_host +Welcome to the minimal Declarative Automation Bundle template! + +This template creates a minimal project structure without sample code, ideal for advanced users. +(For getting started with Python or SQL code, use the default-python or default-sql templates instead.) + +Your workspace at [CUSTOM_HOST]?w=[WORKSPACE_ID] is used for initialization. +(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.) + +✨ Your new project has been created in the 'my_default_minimal' directory! + +To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> grep host: databricks.yml + host: [CUSTOM_HOST]?w=[WORKSPACE_ID] + host: [CUSTOM_HOST]?w=[WORKSPACE_ID] + +>>> DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH=1 [CLI] bundle validate -t dev +Name: my_default_minimal +Target: dev +Workspace: + Host: [CUSTOM_HOST]?w=[WORKSPACE_ID] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev + +Validation OK! + +>>> [CLI] bundle deploy -t dev +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev/files... +Files: 10 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/templates/default-minimal/profiles/script b/acceptance/bundle/templates/default-minimal/profiles/script new file mode 100644 index 00000000000..f3d1d94a153 --- /dev/null +++ b/acceptance/bundle/templates/default-minimal/profiles/script @@ -0,0 +1,29 @@ +setup_custom_host + +# The classic profile has no workspace_id. The SDK resolves the ID from host +# metadata, but the generated host stays without ?w= so that it matches the +# profile in strict mode too. +cat > .databrickscfg < in the + HTTP path. Add it when the workspace URL has the ID. */}} + default: {{.http_path}}{{with .workspace.URLID}}?o={{urlquery .}}{{end}} {{- if eq .default_catalog ""}} _choose_unity_catalog: 'use the default workspace catalog (or do not use Unity Catalog)': diff --git a/libs/template/templates/default-minimal/databricks_template_schema.json b/libs/template/templates/default-minimal/databricks_template_schema.json index 0935684256b..b8df334a017 100644 --- a/libs/template/templates/default-minimal/databricks_template_schema.json +++ b/libs/template/templates/default-minimal/databricks_template_schema.json @@ -1,6 +1,6 @@ { "template_dir": "../default", - "welcome_message": "Welcome to the minimal Declarative Automation Bundle template!\n\nThis template creates a minimal project structure without sample code, ideal for advanced users.\n(For getting started with Python or SQL code, use the default-python or default-sql templates instead.)\n\nYour workspace at {{workspace_host}} is used for initialization.\n(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.)", + "welcome_message": "Welcome to the minimal Declarative Automation Bundle template!\n\nThis template creates a minimal project structure without sample code, ideal for advanced users.\n(For getting started with Python or SQL code, use the default-python or default-sql templates instead.)\n\nYour workspace at {{.workspace.URL}} is used for initialization.\n(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.)", "properties": { "template_name": { "//": "This property is always set to 'default-minimal'", diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index d8d5e485ced..e9e61f4a924 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -1,6 +1,6 @@ { "template_dir": "../default", - "welcome_message": "Welcome to the default Python template for Declarative Automation Bundles!\n\nAnswer the following questions to customize your project.\nYou can always change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization.\n(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.)", + "welcome_message": "Welcome to the default Python template for Declarative Automation Bundles!\n\nAnswer the following questions to customize your project.\nYou can always change your configuration in the databricks.yml file later.\n\nNote that {{.workspace.URL}} is used for initialization.\n(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.)", "properties": { "template_name": { "//": "This property is always set to 'default-python'", diff --git a/libs/template/templates/default-scala/databricks_template_schema.json b/libs/template/templates/default-scala/databricks_template_schema.json index 9501c56196d..652c0d374e4 100644 --- a/libs/template/templates/default-scala/databricks_template_schema.json +++ b/libs/template/templates/default-scala/databricks_template_schema.json @@ -1,5 +1,5 @@ { - "welcome_message": "\nWelcome to the default-scala template for Declarative Automation Bundles!\n\nA workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.\nworkspace_host: {{workspace_host}}", + "welcome_message": "\nWelcome to the default-scala template for Declarative Automation Bundles!\n\nA workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.\nworkspace_host: {{.workspace.URL}}", "properties": { "project_name": { "type": "string", diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl index 749f2a4e993..ec11354f3b6 100644 --- a/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-scala/template/{{.project_name}}/databricks.yml.tmpl @@ -22,7 +22,7 @@ variables: {{- end}} workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} artifact_path: {{.artifacts_dest_path}}/${bundle.name}/${bundle.target}/${workspace.current_user.short_name} artifacts: @@ -42,7 +42,7 @@ targets: mode: development default: true workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} variables: catalog: {{.default_catalog}} schema: {{$dev_schema}} @@ -50,7 +50,7 @@ targets: prod: mode: production workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy. root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: diff --git a/libs/template/templates/default-sql/databricks_template_schema.json b/libs/template/templates/default-sql/databricks_template_schema.json index c6f66d495a8..6967c3ec123 100644 --- a/libs/template/templates/default-sql/databricks_template_schema.json +++ b/libs/template/templates/default-sql/databricks_template_schema.json @@ -1,5 +1,5 @@ { - "welcome_message": "\nWelcome to the default SQL template for Declarative Automation Bundles!\n\nA workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.\nworkspace_host: {{workspace_host}}", + "welcome_message": "\nWelcome to the default SQL template for Declarative Automation Bundles!\n\nA workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.\nworkspace_host: {{.workspace.URL}}", "properties": { "project_name": { "type": "string", diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index 0645de3dadf..cfc2c2cc2fc 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -33,7 +33,7 @@ targets: mode: development default: true workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} variables: warehouse_id: {{index ((regexp "[^/]+$").FindStringSubmatch .http_path) 0}} catalog: {{.default_catalog}} @@ -42,7 +42,7 @@ targets: prod: mode: production workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy. root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} variables: diff --git a/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl index 73cddd42052..42b15164c8e 100644 --- a/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/databricks.yml.tmpl @@ -49,7 +49,7 @@ targets: mode: development default: true workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} variables: catalog: {{.default_catalog}} schema: {{$dev_schema}} @@ -60,7 +60,7 @@ targets: prod: mode: production workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy. root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} variables: diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl index 8087d648d4d..ba651def676 100644 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/databricks.yml.tmpl @@ -38,11 +38,11 @@ targets: mode: development default: true workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} prod: mode: production workspace: - host: {{workspace_host}} + host: {{.workspace.URL}} # We explicitly deploy to /Workspace/Users/{{user_name}} to make sure we only have a single copy. root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: diff --git a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json index 8639acc49e9..ddd17278829 100644 --- a/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json +++ b/libs/template/templates/lakeflow-pipelines/databricks_template_schema.json @@ -1,6 +1,6 @@ { "template_dir": "../default", - "welcome_message": "Welcome to the template for Lakeflow Spark Declarative Pipelines!\n\nPlease answer the below to tailor your project to your preferences.\nYou can always change your mind and change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization\n(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile).", + "welcome_message": "Welcome to the template for Lakeflow Spark Declarative Pipelines!\n\nPlease answer the below to tailor your project to your preferences.\nYou can always change your mind and change your configuration in the databricks.yml file later.\n\nNote that {{.workspace.URL}} is used for initialization\n(see https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile).", "properties": { "template_name": { "//": "This property is always set to 'lakeflow-pipelines'", diff --git a/libs/template/templates/pydabs/databricks_template_schema.json b/libs/template/templates/pydabs/databricks_template_schema.json index 4819b5bb773..d53f8ba85cc 100644 --- a/libs/template/templates/pydabs/databricks_template_schema.json +++ b/libs/template/templates/pydabs/databricks_template_schema.json @@ -1,6 +1,6 @@ { "template_dir": "../default", - "welcome_message": "Welcome to PyDABs template for Declarative Automation Bundles!\n\nAnswer the following questions to customize your project.\nYou can always change your configuration in the databricks.yml file later.\n\nNote that {{workspace_host}} is used for initialization.\n(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.)", + "welcome_message": "Welcome to PyDABs template for Declarative Automation Bundles!\n\nAnswer the following questions to customize your project.\nYou can always change your configuration in the databricks.yml file later.\n\nNote that {{.workspace.URL}} is used for initialization.\n(For information on how to change your profile, see https://docs.databricks.com/dev-tools/cli/profiles.html.)", "properties": { "template_name": { "//": "This property is always set to 'PyDABs'", diff --git a/libs/template/workspace_context.go b/libs/template/workspace_context.go index 20bb5e5a5f8..8b430cfb209 100644 --- a/libs/template/workspace_context.go +++ b/libs/template/workspace_context.go @@ -35,7 +35,8 @@ var userSetWorkspaceIDSources = []sdkconfig.SourceType{ // it carries the workspace ID, which is required to address a workspace on a host shared by // several workspaces. // -// The fields are strings so that templates which print the root data get stable output. +// The fields are strings, not pointers, so that templates which print the root data with +// {{ . }} get the values instead of addresses. type workspaceContext struct { // Hostname is the workspace host without scheme, path, or query. It keeps the port if the // host has one. @@ -44,16 +45,22 @@ type workspaceContext struct { host string workspaceID string - // urlWorkspaceID is the workspace ID that URL adds as ?w=. It is empty unless the user set the - // ID. Strict profile matching (DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH) compares the ID in the - // host with the profile's workspace_id, and a profile without workspace_id matches only a host - // without an ID. - urlWorkspaceID string + // URLID is the workspace ID that URL adds as ?w=. It is set when a workspace_id in the + // profile, DATABRICKS_WORKSPACE_ID, or the bundle's workspace_id sets the ID, and empty + // otherwise, so templates can check it without failing like ID does. It is empty for an ID + // that comes from ?o= in the profile's host or from host metadata. Strict profile matching + // (DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH) compares the bundle's workspace ID, which comes + // from ?w= in its host, with the profile's workspace_id, and a profile without workspace_id + // matches only a bundle without an ID. + // + // It comes last so that templates which print the root data keep their output. + URLID string } -// loadWorkspaceContext returns the workspace context of the selected profile, or nil if the -// schema declares a workspace input. A declared input owns the name even before it has a value. -// Input values the schema doesn't declare don't claim the name, because bundle init ignores them. +// loadWorkspaceContext returns the workspace context of the resolved configuration, or nil +// if the schema declares a workspace input. A declared input owns the name even before it has +// a value. Input values the schema doesn't declare don't claim the name, because bundle init +// ignores them. func loadWorkspaceContext(ctx context.Context, schema *jsonschema.Schema) (*workspaceContext, error) { if _, ok := schema.Properties[workspaceVariable]; ok { return nil, nil @@ -67,15 +74,15 @@ func loadWorkspaceContext(ctx context.Context, schema *jsonschema.Schema) (*work if workspaceID == profile.WorkspaceIDNone { workspaceID = "" } - var urlWorkspaceID string + var urlID string if attr, ok := cfg.GetAuthDetails().Configuration["workspace_id"]; ok && slices.Contains(userSetWorkspaceIDSources, attr.Source.Type) { - urlWorkspaceID = workspaceID + urlID = workspaceID } return &workspaceContext{ - Hostname: host.Host, - host: cfg.Host, - workspaceID: workspaceID, - urlWorkspaceID: urlWorkspaceID, + Hostname: host.Host, + host: cfg.Host, + workspaceID: workspaceID, + URLID: urlID, }, nil } @@ -97,14 +104,13 @@ func templateData(values map[string]any, workspace *workspaceContext) map[string // shared by several workspaces. It doesn't make a request to discover the ID. func (c workspaceContext) ID() (string, error) { if c.workspaceID == "" { - return "", errors.New("cannot determine workspace ID, please set workspace_id in the configuration profile or the DATABRICKS_WORKSPACE_ID environment variable") + return "", errors.New("cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle") } return c.workspaceID, nil } // URL returns the workspace URL with an optional path relative to the workspace host, -// for example URL "/jobs". It adds ?w= when the profile, an environment variable, or the -// bundle sets the workspace ID. +// for example URL "/jobs". It adds ?w= when URLID is set. func (c workspaceContext) URL(path ...string) (string, error) { if len(path) > 1 { return "", fmt.Errorf("URL accepts at most one path, got %d", len(path)) @@ -113,11 +119,11 @@ func (c workspaceContext) URL(path ...string) (string, error) { if err != nil { return "", fmt.Errorf("failed to parse workspace host %q: %w", c.host, err) } - // Unlike workspaceurls.WorkspaceBaseURL, this adds ?w= also when the hostname has the ID - // (adb-), because strict profile matching reads the ID only from the query. - if c.urlWorkspaceID != "" { + // The workspaceurls package leaves out ?w= when the hostname has the ID (adb-). This + // adds it, because strict profile matching doesn't read the ID from the hostname. + if c.URLID != "" { query := base.Query() - query.Set("w", c.urlWorkspaceID) + query.Set("w", c.URLID) base.RawQuery = query.Encode() } if len(path) == 0 { diff --git a/libs/template/workspace_context_test.go b/libs/template/workspace_context_test.go index d20a45f621a..97016f138a3 100644 --- a/libs/template/workspace_context_test.go +++ b/libs/template/workspace_context_test.go @@ -87,7 +87,7 @@ func TestPromptForValuesWithWorkspace(t *testing.T) { // Acceptance tests can't use an adb- hostname, because the test server is reachable only // on localhost. func TestWorkspaceURLKeepsIDOnHostnameWithID(t *testing.T) { - ws := workspaceContext{host: "https://adb-111.4.azuredatabricks.net", workspaceID: "111", urlWorkspaceID: "111"} + ws := workspaceContext{host: "https://adb-111.4.azuredatabricks.net", workspaceID: "111", URLID: "111"} // Strict profile matching reads the ID from the query, not from the hostname. host, err := ws.URL()