Find a catalog renamed outside the bundle by its id

.nextchanges/bundles/catalog-renamed-out-of-band.md

@@ -0,0 +1 @@
+* A catalog renamed outside the bundle is now found by its id, instead of being created again (which failed with `CATALOG_ALREADY_EXISTS`). ([#7418](https://github.com/databricks/cli/pull/7418))

acceptance/bundle/refschema/output/catalogs.txt

@@ -11,7 +11,7 @@ resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type
 resources.catalogs.*.effective_predictive_optimization_flag.value	catalog.EnablePredictiveOptimization	REMOTE
 resources.catalogs.*.enable_predictive_optimization	catalog.EnablePredictiveOptimization	REMOTE
 resources.catalogs.*.full_name	string	REMOTE
-resources.catalogs.*.id	string	INPUT
+resources.catalogs.*.id	string	ALL
 resources.catalogs.*.isolation_mode	catalog.CatalogIsolationMode	REMOTE
 resources.catalogs.*.lifecycle	resources.Lifecycle	INPUT
 resources.catalogs.*.lifecycle.depends_on	[]resources.DependsOn	INPUT

acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt

@@ -13,6 +13,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
   "comment": {
     "action": "update",
     "remote": "set outside the bundle"
+  },
+  "id": {
+    "action": "skip",
+    "reason": "internal_id",
+    "old": "[UUID]",
+    "remote": "[UUID]"
   }
 }
 

acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt

@@ -13,6 +13,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 === The remote-only properties map is skipped as a backend default (confirms the matched rule)
 >>> [CLI] bundle plan --output json
 {
+  "id": {
+    "action": "skip",
+    "reason": "internal_id",
+    "old": "[UUID]",
+    "remote": "[UUID]"
+  },
   "properties": {
     "action": "skip",
     "reason": "backend_default",

acceptance/bundle/resources/catalogs/rename/output.txt

@@ -53,10 +53,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
       "created_at": [UNIX_TIME_MILLIS][0],
       "created_by": "[USERNAME]",
       "full_name": "test_catalog_[UNIQUE_NAME]",
+      "id": "[UUID]",
       "name": "test_catalog_[UNIQUE_NAME]",
       "owner": "[USERNAME]"
     },
     "changes": {
+      "id": {
+        "action": "skip",
+        "reason": "internal_id",
+        "old": "[UUID]",
+        "remote": "[UUID]"
+      },
       "name": {
         "action": "update_id",
         "reason": "id_changes",

acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt

@@ -31,22 +31,40 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> update_file.py databricks.yml name: myvolume name: myvolume2
 
-=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again
+=== The plan finds the catalog, the schema and the volume at their new names by their ids
 >>> [CLI] bundle plan --output json
 {
   "resources.catalogs.catalog1": {
-    "action": "create",
+    "id": "test_catalog_renamed_[UNIQUE_NAME]",
+    "action": "update_id",
+    "state_only": true,
     "new_state": {
       "value": {
         "name": "test_catalog_renamed_[UNIQUE_NAME]"
       }
     },
+    "remote_state": {
+      "catalog_type": "MANAGED_CATALOG",
+      "created_at": [UNIX_TIME_MILLIS][0],
+      "created_by": "[USERNAME]",
+      "full_name": "test_catalog_renamed_[UNIQUE_NAME]",
+      "id": "[CATALOG_ID]",
+      "name": "test_catalog_renamed_[UNIQUE_NAME]",
+      "owner": "[USERNAME]"
+    },
     "changes": {
+      "id": {
+        "action": "skip",
+        "reason": "internal_id",
+        "old": "[CATALOG_ID]",
+        "remote": "[CATALOG_ID]"
+      },
       "name": {
-        "action": "update_id",
-        "reason": "id_changes",
+        "action": "skip",
+        "reason": "remote_already_set",
         "old": "test_catalog_[UNIQUE_NAME]",
-        "new": "test_catalog_renamed_[UNIQUE_NAME]"
+        "new": "test_catalog_renamed_[UNIQUE_NAME]",
+        "remote": "test_catalog_renamed_[UNIQUE_NAME]"
       }
     }
   },
@@ -69,7 +87,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
     "remote_state": {
       "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]",
       "catalog_type": "MANAGED_CATALOG",
-      "created_at": [UNIX_TIME_MILLIS][0],
+      "created_at": [UNIX_TIME_MILLIS][1],
       "created_by": "[USERNAME]",
       "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2",
       "name": "myschema2",
@@ -123,7 +141,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
     },
     "remote_state": {
       "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]",
-      "created_at": [UNIX_TIME_MILLIS][1],
+      "created_at": [UNIX_TIME_MILLIS][2],
       "created_by": "[USERNAME]",
       "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2",
       "name": "myvolume2",
@@ -164,25 +182,46 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
   }
 }
 
-=== The catalog's create fails because the renamed catalog has its name
->>> musterr [CLI] bundle deploy
+=== Deploy only records the new names in the state
+>>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
-Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS)
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 3 unchanged
+
+>>> print_requests.py //unity-catalog
+
+>>> [CLI] bundle summary --output json
+{
+  "catalogs": {
+    "catalog1": "test_catalog_renamed_[UNIQUE_NAME]"
+  },
+  "schemas": {
+    "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2"
+  },
+  "volumes": {
+    "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2"
+  }
+}
 
-Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs
-HTTP Status: 400 Bad Request
-API error_code: CATALOG_ALREADY_EXISTS
-API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists
+=== Redeploy is a no-op
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
 
-Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.catalog1
+  delete resources.schemas.schema1
+  delete resources.volumes.volume1
 
-Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.schema1
 
-Files: 3 uploaded, 0 deleted
+This action will result in the deletion of the following volumes.
+For managed volumes, the files stored in the volume are also deleted from your
+cloud tenant within 30 days. For external volumes, the metadata about the volume
+is removed from the catalog, but the underlying files are not deleted:
+  delete resources.volumes.volume1
 
->>> errcode [CLI] bundle destroy --auto-approve
 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
 
-Destroy: 0 deleted
-
->>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force
+Destroy: 3 deleted

acceptance/bundle/resources/catalogs/rename_all_out_of_band/script

@@ -4,9 +4,7 @@ OLD_CATALOG="test_catalog_${UNIQUE_NAME}"
 NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}"
 
 cleanup() {
-    trace errcode $CLI bundle destroy --auto-approve
-    # The renamed catalog is not where the state looks for it, so destroy cannot remove it.
-    trace errcode $CLI catalogs delete "$NEW_CATALOG" --force
+    trace $CLI bundle destroy --auto-approve
 
     # Consume the destroy's requests so none are left recorded.
     print_requests.py //unity-catalog &> LOG.destroy_requests
@@ -21,6 +19,8 @@ plan() {
 title "Deploy a catalog with a schema and a volume in it"
 trace $CLI bundle deploy
 print_requests.py //unity-catalog &> LOG.deploy_requests
+# The SDK drops the catalog's id, so read it with a raw GET.
+add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID
 add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID
 add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID
 
@@ -34,11 +34,14 @@ title "Rename all three in the config to match"
 trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_
 trace update_file.py databricks.yml "name: myschema" "name: myschema2"
 trace update_file.py databricks.yml "name: myvolume" "name: myvolume2"
-NEW_SCHEMA="$NEW_CATALOG.myschema2"
-NEW_VOLUME="$NEW_SCHEMA.myvolume2"
 
-title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again"
+title "The plan finds the catalog, the schema and the volume at their new names by their ids"
 plan
 
-title "The catalog's create fails because the renamed catalog has its name"
-trace musterr $CLI bundle deploy
+title "Deploy only records the new names in the state"
+trace $CLI bundle deploy
+trace print_requests.py //unity-catalog
+trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))'
+
+title "Redeploy is a no-op"
+trace $CLI bundle plan

acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml

@@ -1,4 +1,3 @@
-Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main."
 Cloud = true
 
 Ignore = [

acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt

@@ -17,22 +17,40 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
 === Rename the catalog in the config to match
 >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_
 
-=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again
+=== The plan finds the catalog, the schema and the volume at their new names by their ids
 >>> [CLI] bundle plan --output json
 {
   "resources.catalogs.catalog1": {
-    "action": "create",
+    "id": "test_catalog_renamed_[UNIQUE_NAME]",
+    "action": "update_id",
+    "state_only": true,
     "new_state": {
       "value": {
         "name": "test_catalog_renamed_[UNIQUE_NAME]"
       }
     },
+    "remote_state": {
+      "catalog_type": "MANAGED_CATALOG",
+      "created_at": [UNIX_TIME_MILLIS][0],
+      "created_by": "[USERNAME]",
+      "full_name": "test_catalog_renamed_[UNIQUE_NAME]",
+      "id": "[CATALOG_ID]",
+      "name": "test_catalog_renamed_[UNIQUE_NAME]",
+      "owner": "[USERNAME]"
+    },
     "changes": {
+      "id": {
+        "action": "skip",
+        "reason": "internal_id",
+        "old": "[CATALOG_ID]",
+        "remote": "[CATALOG_ID]"
+      },
       "name": {
-        "action": "update_id",
-        "reason": "id_changes",
+        "action": "skip",
+        "reason": "remote_already_set",
         "old": "test_catalog_[UNIQUE_NAME]",
-        "new": "test_catalog_renamed_[UNIQUE_NAME]"
+        "new": "test_catalog_renamed_[UNIQUE_NAME]",
+        "remote": "test_catalog_renamed_[UNIQUE_NAME]"
       }
     }
   },
@@ -55,7 +73,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
     "remote_state": {
       "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]",
       "catalog_type": "MANAGED_CATALOG",
-      "created_at": [UNIX_TIME_MILLIS][0],
+      "created_at": [UNIX_TIME_MILLIS][1],
       "created_by": "[USERNAME]",
       "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema",
       "name": "myschema",
@@ -102,7 +120,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
     },
     "remote_state": {
       "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]",
-      "created_at": [UNIX_TIME_MILLIS][1],
+      "created_at": [UNIX_TIME_MILLIS][2],
       "created_by": "[USERNAME]",
       "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume",
       "name": "myvolume",
@@ -129,25 +147,46 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
   }
 }
 
-=== The catalog's create fails because the renamed catalog has its name
->>> musterr [CLI] bundle deploy
+=== Deploy only records the new names in the state
+>>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
-Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS)
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 3 unchanged
+
+>>> print_requests.py //unity-catalog
+
+>>> [CLI] bundle summary --output json
+{
+  "catalogs": {
+    "catalog1": "test_catalog_renamed_[UNIQUE_NAME]"
+  },
+  "schemas": {
+    "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema"
+  },
+  "volumes": {
+    "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume"
+  }
+}
 
-Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs
-HTTP Status: 400 Bad Request
-API error_code: CATALOG_ALREADY_EXISTS
-API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists
+=== Redeploy is a no-op
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
 
-Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.catalog1
+  delete resources.schemas.schema1
+  delete resources.volumes.volume1
 
-Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.schema1
 
-Files: 3 uploaded, 0 deleted
+This action will result in the deletion of the following volumes.
+For managed volumes, the files stored in the volume are also deleted from your
+cloud tenant within 30 days. For external volumes, the metadata about the volume
+is removed from the catalog, but the underlying files are not deleted:
+  delete resources.volumes.volume1
 
->>> errcode [CLI] bundle destroy --auto-approve
 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
 
-Destroy: 0 deleted
-
->>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force
+Destroy: 3 deleted

acceptance/bundle/resources/catalogs/rename_out_of_band/script

@@ -4,9 +4,7 @@ OLD_CATALOG="test_catalog_${UNIQUE_NAME}"
 NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}"
 
 cleanup() {
-    trace errcode $CLI bundle destroy --auto-approve
-    # The renamed catalog is not where the state looks for it, so destroy cannot remove it.
-    trace errcode $CLI catalogs delete "$NEW_CATALOG" --force
+    trace $CLI bundle destroy --auto-approve
 
     # Consume the destroy's requests so none are left recorded.
     print_requests.py //unity-catalog &> LOG.destroy_requests
@@ -21,6 +19,8 @@ plan() {
 title "Deploy a catalog with a schema and a volume in it"
 trace $CLI bundle deploy
 print_requests.py //unity-catalog &> LOG.deploy_requests
+# The SDK drops the catalog's id, so read it with a raw GET.
+add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID
 add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID
 add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID
 
@@ -30,11 +30,14 @@ print_requests.py //unity-catalog &> LOG.rename_requests
 
 title "Rename the catalog in the config to match"
 trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_
-NEW_SCHEMA="$NEW_CATALOG.myschema"
-NEW_VOLUME="$NEW_SCHEMA.myvolume"
 
-title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again"
+title "The plan finds the catalog, the schema and the volume at their new names by their ids"
 plan
 
-title "The catalog's create fails because the renamed catalog has its name"
-trace musterr $CLI bundle deploy
+title "Deploy only records the new names in the state"
+trace $CLI bundle deploy
+trace print_requests.py //unity-catalog
+trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))'
+
+title "Redeploy is a no-op"
+trace $CLI bundle plan

acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml

@@ -1,4 +1,3 @@
-Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main."
 Cloud = true
 
 Ignore = [