Use .workspace.URL for the host in built-in templates

.nextchanges/bundles/builtin-templates-workspace-url.md

@@ -0,0 +1 @@
+* Built-in templates add the workspace ID to `workspace.host` (`?w=<id>`) when the profile or an environment variable sets it, so generated bundles select the right workspace on hosts shared by several workspaces. Welcome messages show the same URL, and `dbt-sql` also adds `?o=<id>` to the HTTP path it suggests for `dbt init`. ([#7350](https://github.com/databricks/cli/pull/7350))

.nextchanges/bundles/workspace-template-urlid.md

@@ -0,0 +1 @@
+* Add `.workspace.URLID` to bundle templates: the workspace ID that `.workspace.URL` adds, or empty, so templates can use it without failing when the ID is unknown. ([#7388](https://github.com/databricks/cli/pull/7388))

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.protocol-relative.json

@@ -0,0 +1 @@
+{"path": "//other.test/jobs"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-w.json

@@ -0,0 +1 @@
+{"path": "/jobs?w=123"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.selector-workspace-id.json

@@ -0,0 +1 @@
+{"path": "/jobs?workspace_id=123"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/input.supplied.json

@@ -0,0 +1 @@
+{"value": "set by user"}

acceptance/bundle/templates-machinery/workspace_variable/runtime/output.txt

@@ -25,6 +25,7 @@ Hostname: [DATABRICKS_HOST]
 URL: [DATABRICKS_URL]
 URL "/jobs": [DATABRICKS_URL]/jobs
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list#top
+URLID: ""
 jobs_url: [DATABRICKS_URL]/jobs
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[DATABRICKS_URL]/jobs path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] [WORKSPACE_ID] }]
@@ -55,6 +56,7 @@ Hostname: [DATABRICKS_HOST]
 URL: [DATABRICKS_URL]?w=[WORKSPACE_ID]
 URL "/jobs": [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 jobs_url: [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID]
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[DATABRICKS_URL]/jobs?w=[WORKSPACE_ID] path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL] [WORKSPACE_ID] [WORKSPACE_ID]}]
@@ -85,6 +87,7 @@ Hostname: [CUSTOM_HOSTNAME]
 URL: [CUSTOM_HOST]?w=[WORKSPACE_ID]
 URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 jobs_url: [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[CUSTOM_HOST]/jobs?w=[WORKSPACE_ID] path:/jobs?view=list#top workspace:{[CUSTOM_HOSTNAME] [CUSTOM_HOST] [WORKSPACE_ID] [WORKSPACE_ID]}]
@@ -115,6 +118,7 @@ Hostname: [DATABRICKS_HOST]
 URL: [DATABRICKS_URL]
 URL "/jobs": [DATABRICKS_URL]/jobs
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list#top
+URLID: ""
 jobs_url: [DATABRICKS_URL]/jobs
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:[DATABRICKS_URL]/jobs path:/jobs?view=list#top workspace:{[DATABRICKS_HOST] [DATABRICKS_URL]  }]
@@ -145,20 +149,54 @@ Hostname: [CUSTOM_HOSTNAME]
 URL: [CUSTOM_HOST]?w=[WORKSPACE_ID]
 URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 URL "/pipelines?tab=all": [CUSTOM_HOST]/pipelines?tab=all&w=[WORKSPACE_ID]
+URLID: "[WORKSPACE_ID]"
 jobs_url: https://example.test/my-jobs
 Root keys: jobs_url path workspace
 Root data: map[jobs_url:https://example.test/my-jobs path:/pipelines?tab=all workspace:{[CUSTOM_HOSTNAME] [CUSTOM_HOST] [WORKSPACE_ID] [WORKSPACE_ID]}]
 
+=== Failing default of an input with a value
+
+>>> [CLI] bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json
+{
+  "welcome_message": null,
+  "success_message": null,
+  "properties": {
+    "value": {
+      "description": "Value",
+      "default": "{{ .workspace.ID }}"
+    }
+  }
+}
+
+>>> [CLI] bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json
+✨ Successfully initialized template
+value: set by user
+
+>>> musterr [CLI] bundle debug render-template-schema ./supplied --profile no_workspace_id
+Error: property value default: failed to render template "{{ .workspace.ID }}": template: base:1:13: executing "base" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle
+
 === Invalid URL paths
 
 >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.absolute-url.json
 Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
 Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "https://other.test/jobs" must be relative to the workspace host
 
+>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.protocol-relative.json
+Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
+Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "//other.test/jobs" must be relative to the workspace host
+
 >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.workspace-selector.json
 Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
 Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?o=123" must not set the "o" query parameter, the workspace is selected by the profile
 
+>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.selector-w.json
+Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
+Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?w=123" must not set the "w" query parameter, the workspace is selected by the profile
+
+>>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.selector-workspace-id.json
+Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
+Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: workspace URL path "/jobs?workspace_id=123" must not set the "workspace_id" query parameter, the workspace is selected by the profile
+
 >>> musterr [CLI] bundle init ./urls --profile workspace_custom_host --config-file input.semicolon.json
 Initializing for [CUSTOM_HOST]?w=[WORKSPACE_ID]
 Error: failed to compute file content for urls.txt.tmpl. template: :6:25: executing "" at <$ws.URL>: error calling URL: failed to parse query of workspace URL path "/jobs?w=999;x=1": invalid semicolon separator in query
@@ -171,6 +209,7 @@ Open [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top
 URL: [CUSTOM_HOST]?w=[WORKSPACE_ID]
 URL "/jobs": [CUSTOM_HOST]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [CUSTOM_HOST]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 
 === Workspace ID from the bundle in the working directory
 
@@ -180,6 +219,7 @@ Open [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top
 URL: [DATABRICKS_URL]?w=[WORKSPACE_ID]
 URL "/jobs": [DATABRICKS_URL]/jobs?w=[WORKSPACE_ID]
 URL "/jobs?view=list#top": [DATABRICKS_URL]/jobs?view=list&w=[WORKSPACE_ID]#top
+URLID: "[WORKSPACE_ID]"
 
 === ID
 
@@ -192,6 +232,6 @@ ID: [WORKSPACE_ID]
 ID: [WORKSPACE_ID]
 
 >>> musterr [CLI] bundle init ./id --profile no_workspace_id
-Error: failed to compute file content for id.txt.tmpl. template: :1:17: executing "" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile or the DATABRICKS_WORKSPACE_ID environment variable
+Error: failed to compute file content for id.txt.tmpl. template: :1:17: executing "" at <.workspace.ID>: error calling ID: cannot determine workspace ID, please set workspace_id in the configuration profile, the DATABRICKS_WORKSPACE_ID environment variable, or workspace.workspace_id in the bundle
 
 >>> print_requests.py --get --oneline ^//telemetry-ext

acceptance/bundle/templates-machinery/workspace_variable/runtime/script

@@ -15,10 +15,20 @@ trace $CLI bundle init ./urls --profile workspace_custom_host --config-file inpu
 cat urls.txt
 rm urls.txt
 
+# A default that fails, here because the profile has no workspace ID, doesn't
+# matter when the input has a value: init doesn't use it, and the schema keeps
+# it as written. Without a value, the schema still fails.
+title "Failing default of an input with a value\n"
+trace $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json | schema_fields
+trace $CLI bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json
+cat value.txt
+rm value.txt
+trace musterr $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id
+
 title "Invalid URL paths\n"
-trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.absolute-url.json
-trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.workspace-selector.json
-trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.semicolon.json
+for input in absolute-url protocol-relative workspace-selector selector-w selector-workspace-id semicolon; do
+  trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.$input.json
+done
 
 # No profile, so that only the environment variable sets the ID.
 title "Workspace ID from environment variables\n"

acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/databricks_template_schema.json

@@ -0,0 +1,9 @@
+{
+    "properties": {
+        "value": {
+            "type": "string",
+            "description": "Value",
+            "default": "{{ .workspace.ID }}"
+        }
+    }
+}

acceptance/bundle/templates-machinery/workspace_variable/runtime/supplied/template/value.txt.tmpl

@@ -0,0 +1 @@
+value: {{ .value }}

acceptance/bundle/templates-machinery/workspace_variable/runtime/urls/template/urls.txt.tmpl

@@ -4,6 +4,7 @@ Hostname: {{ .workspace.Hostname }}
 URL: {{ .workspace.URL }}
 URL "/jobs": {{ .workspace.URL "/jobs" }}
 URL "{{ .path }}": {{ $ws.URL .path }}
+URLID: "{{ .workspace.URLID }}"
 jobs_url: {{ .jobs_url }}
 Root keys:{{ range $key, $value := . }} {{ $key }}{{ end }}
 Root data: {{ . }}

acceptance/bundle/templates-machinery/workspace_variable/script.prepare

@@ -1,9 +1,4 @@
-# The test server is reachable as both 127.0.0.1 and localhost. The second
-# stands in for a host shared by several workspaces.
-CUSTOM_HOST=${DATABRICKS_HOST/127.0.0.1/localhost}
-add_repl "$CUSTOM_HOST" CUSTOM_HOST
-add_repl "${CUSTOM_HOST#*://}" CUSTOM_HOSTNAME
-WORKSPACE_ID=$(curl -s "$DATABRICKS_HOST/.well-known/databricks-config" | jq -r .workspace_id)
+setup_custom_host
 
 # workspace_classic has no workspace_id, so the SDK resolves it from host
 # metadata. The "none" sentinel prevents that.

acceptance/bundle/templates/dbt-sql-workspace-id/input.json

@@ -0,0 +1,6 @@
+{
+    "project_name": "my_dbt_sql",
+    "http_path": "/sql/2.0/warehouses/f00dcafe",
+    "default_catalog": "main",
+    "personal_schemas": "yes"
+}

acceptance/bundle/templates/dbt-sql-workspace-id/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/templates/dbt-sql-workspace-id/output.txt

@@ -0,0 +1,48 @@
+
+=== Profile no_workspace_id
+
+>>> [CLI] bundle init dbt-sql --config-file ./input.json --profile no_workspace_id
+
+Welcome to the dbt template for Declarative Automation Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: [DATABRICKS_URL]
+
+📊 Your new project has been created in the 'my_dbt_sql' directory!
+If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
+Refer to the README.md file for full "getting started" guide and production setup instructions.
+
+
+>>> grep host: my_dbt_sql/databricks.yml
+      host: [DATABRICKS_URL]
+      host: [DATABRICKS_URL]
+
+>>> grep -E ^    default: [^{] my_dbt_sql/profile_template.yml
+    default: [DATABRICKS_HOST]
+    default: /sql/2.0/warehouses/f00dcafe
+    default: main
+    default: 4
+
+=== Profile custom_host
+
+>>> [CLI] bundle init dbt-sql --config-file ./input.json --profile custom_host
+
+Welcome to the dbt template for Declarative Automation Bundles!
+
+A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
+workspace_host: [CUSTOM_HOST]?w=[WORKSPACE_ID]
+
+📊 Your new project has been created in the 'my_dbt_sql' directory!
+If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
+Refer to the README.md file for full "getting started" guide and production setup instructions.
+
+
+>>> grep host: my_dbt_sql/databricks.yml
+      host: [CUSTOM_HOST]?w=[WORKSPACE_ID]
+      host: [CUSTOM_HOST]?w=[WORKSPACE_ID]
+
+>>> grep -E ^    default: [^{] my_dbt_sql/profile_template.yml
+    default: [CUSTOM_HOSTNAME]
+    default: /sql/2.0/warehouses/f00dcafe?o=[WORKSPACE_ID]
+    default: main
+    default: 4

acceptance/bundle/templates/dbt-sql-workspace-id/script

@@ -0,0 +1,25 @@
+setup_custom_host
+
+cat > .databrickscfg <<EOT
+[no_workspace_id]
+host = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[custom_host]
+host = $CUSTOM_HOST
+workspace_id = $WORKSPACE_ID
+token = $DATABRICKS_TOKEN
+EOT
+export DATABRICKS_CONFIG_FILE=$PWD/.databrickscfg
+unset DATABRICKS_HOST
+unset DATABRICKS_TOKEN
+
+# dbt reads the workspace ID from ?o= in the HTTP path. The template adds it
+# when the profile sets the ID, like ?w= in the bundle host.
+for profile in no_workspace_id custom_host; do
+  title "Profile $profile\n"
+  trace $CLI bundle init dbt-sql --config-file ./input.json --profile $profile
+  trace grep 'host:' my_dbt_sql/databricks.yml
+  trace grep -E "^    default: [^{]" my_dbt_sql/profile_template.yml
+  rm -r my_dbt_sql
+done

acceptance/bundle/templates/dbt-sql-workspace-id/test.toml

@@ -0,0 +1 @@
+Ignore = [".databrickscfg"]

acceptance/bundle/templates/default-minimal/profiles/input.json

@@ -0,0 +1,4 @@
+{
+    "project_name": "my_default_minimal",
+    "language_choice": "python"
+}

acceptance/bundle/templates/default-minimal/profiles/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/templates/default-minimal/profiles/output.txt

@@ -0,0 +1,68 @@
+
+=== Profile classic
+
+>>> [CLI] bundle init default-minimal --config-file ./input.json --profile classic
+Welcome to the minimal Declarative Automation Bundle template!
+
+This template creates a minimal project structure without sample code, ideal for advanced users.
+(For getting started with Python or SQL code, use the default-python or default-sql templates instead.)
+
+Your workspace at [DATABRICKS_URL] is used for initialization.
+(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.)
+
+✨ Your new project has been created in the 'my_default_minimal' directory!
+
+To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> grep host: databricks.yml
+      host: [DATABRICKS_URL]
+      host: [DATABRICKS_URL]
+
+>>> DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH=1 [CLI] bundle validate -t dev
+Name: my_default_minimal
+Target: dev
+Workspace:
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev
+
+Validation OK!
+
+>>> [CLI] bundle deploy -t dev
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev/files...
+Files: 10 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Profile custom_host
+
+>>> [CLI] bundle init default-minimal --config-file ./input.json --profile custom_host
+Welcome to the minimal Declarative Automation Bundle template!
+
+This template creates a minimal project structure without sample code, ideal for advanced users.
+(For getting started with Python or SQL code, use the default-python or default-sql templates instead.)
+
+Your workspace at [CUSTOM_HOST]?w=[WORKSPACE_ID] is used for initialization.
+(See https://docs.databricks.com/dev-tools/cli/profiles.html for how to change your profile.)
+
+✨ Your new project has been created in the 'my_default_minimal' directory!
+
+To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> grep host: databricks.yml
+      host: [CUSTOM_HOST]?w=[WORKSPACE_ID]
+      host: [CUSTOM_HOST]?w=[WORKSPACE_ID]
+
+>>> DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH=1 [CLI] bundle validate -t dev
+Name: my_default_minimal
+Target: dev
+Workspace:
+  Host: [CUSTOM_HOST]?w=[WORKSPACE_ID]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev
+
+Validation OK!
+
+>>> [CLI] bundle deploy -t dev
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_minimal/dev/files...
+Files: 10 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 0 unchanged

acceptance/bundle/templates/default-minimal/profiles/script

@@ -0,0 +1,29 @@
+setup_custom_host
+
+# The classic profile has no workspace_id. The SDK resolves the ID from host
+# metadata, but the generated host stays without ?w= so that it matches the
+# profile in strict mode too.
+cat > .databrickscfg <<EOT
+[classic]
+host = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[custom_host]
+host = $CUSTOM_HOST
+workspace_id = $WORKSPACE_ID
+token = $DATABRICKS_TOKEN
+EOT
+export DATABRICKS_CONFIG_FILE=$PWD/.databrickscfg
+unset DATABRICKS_HOST
+unset DATABRICKS_TOKEN
+
+for profile in classic custom_host; do
+  title "Profile $profile\n"
+  trace $CLI bundle init default-minimal --config-file ./input.json --profile $profile
+  cd my_default_minimal
+  trace grep 'host:' databricks.yml
+  trace DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH=1 $CLI bundle validate -t dev
+  trace $CLI bundle deploy -t dev
+  cd ..
+  rm -r my_default_minimal
+done

acceptance/bundle/templates/default-minimal/profiles/test.toml

@@ -0,0 +1,2 @@
+# The script writes .databrickscfg.
+Ignore = [".databrickscfg"]

acceptance/script.prepare

@@ -115,6 +115,19 @@ add_repl() {
     env -u MSYS_NO_PATHCONV add_repl.py "$@"
 }
 
+# setup_custom_host sets CUSTOM_HOST to the test server reached as localhost
+# instead of 127.0.0.1, which stands in for a host shared by several workspaces,
+# and WORKSPACE_ID to the workspace ID that the test server reports. Call it
+# once per test, while DATABRICKS_HOST is still set: add_repl gives a repeated
+# replacement name a suffix, so a second call adds duplicate replacements.
+setup_custom_host() {
+    export CUSTOM_HOST=${DATABRICKS_HOST/127.0.0.1/localhost}
+    add_repl "$CUSTOM_HOST" CUSTOM_HOST
+    add_repl "${CUSTOM_HOST#*://}" CUSTOM_HOSTNAME
+    WORKSPACE_ID=$(curl -s "$DATABRICKS_HOST/.well-known/databricks-config" | jq -r .workspace_id)
+    export WORKSPACE_ID
+}
+
 nostamp() {
     # Same MSYS_NO_PATHCONV workaround as envsubst above: otherwise Python cannot
     # find nostamp.py on Windows.