diff --git a/.nextchanges/bundles/schema-rename.md b/.nextchanges/bundles/schema-rename.md new file mode 100644 index 00000000000..2ac01106dbe --- /dev/null +++ b/.nextchanges/bundles/schema-rename.md @@ -0,0 +1 @@ +* Rename a schema in place when its `name` changes, instead of deleting and recreating it. Schemas and volumes inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7343](https://github.com/databricks/cli/pull/7343)) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 02b3d5450f9..dacc77db623 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -98,7 +98,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", "state": { "catalog_name": "main", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 94f0a68a122..02cbb2e5705 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index c12e022ced2..5e169e86ebd 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -52,7 +52,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -170,7 +170,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "__id__": "main.dms_empty_grants_[UNIQUE_NAME]", "state": { "catalog_name": "main", - "name": "dms_empty_grants_[UNIQUE_NAME]" + "name": "dms_empty_grants_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 2e599afbde7..0d930eff2c0 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -100,7 +100,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index c8079249d34..e1ec322a14c 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -47,7 +47,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } @@ -106,7 +107,8 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "state": { "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } @@ -153,7 +155,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 41c9d29e8e2..f6776520bc3 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "error_message": "", "resource_id": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 0, - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -100,7 +100,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "main", "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } @@ -159,7 +160,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "error_message": "", "resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", "sequence_id": 1, - "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\",\"schema_id\":\"[UUID]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -212,7 +213,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state": { "catalog_name": "dms_other_[UNIQUE_NAME]", "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "schema_id": "[UUID]" } } } diff --git a/acceptance/bundle/lifecycle/prevent-destroy/script b/acceptance/bundle/lifecycle/prevent-destroy/script index a2c90cea5b5..94c074cfc0b 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/script +++ b/acceptance/bundle/lifecycle/prevent-destroy/script @@ -12,8 +12,8 @@ update_file.py resources/pipeline.yml ' catalog: main' ' storage: "dbf trace errcode $CLI bundle plan >>out.txt 2>&1 trace musterr $CLI bundle deploy -qq >>out.txt 2>&1 -# Changing the schema name, deploy must fail because schema will be recreated -update_file.py resources/schema.yml 'name: test-schema' 'name: test-schema-new' +# Changing the schema catalog, deploy must fail because schema will be recreated +update_file.py resources/schema.yml 'catalog_name: "test-catalog"' 'catalog_name: "test-catalog-new"' trace errcode $CLI bundle plan >>out.txt 2>&1 trace musterr $CLI bundle deploy -qq >>out.txt 2>&1 diff --git a/acceptance/bundle/migrate/auto/schema-rename/output.txt b/acceptance/bundle/migrate/auto/schema-rename/output.txt index 65e060f08f9..6985c121152 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/output.txt +++ b/acceptance/bundle/migrate/auto/schema-rename/output.txt @@ -2,38 +2,15 @@ === Replay the prior terraform deployment (captured from a real terraform run) >>> replay_tfstate.py terraform-requests.json terraform.tfstate -=== Rename the schema (an immutable id field). Without --auto-approve the recreate is refused ->>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy -Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.my_schema -Warn: Migration not committed, keeping Terraform state -Error: the deployment requires destructive actions, but the current console does not support prompting. -Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. -To proceed, use --auto-approve after reviewing the plan above. - -Files: 2 uploaded, 0 deleted - -Exit code: 1 - ->>> print_migration_telemetry -direct_migrate_recreate_planned true - -=== With --auto-approve the recreate is applied on direct under the new name ->>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve -Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. +=== Rename the schema (an updatable id field) and deploy on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be renamed. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.my_schema Migrated 1 resource to direct deployment engine. -Recreated schemas.my_schema -Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 0 unchanged +Updated schemas.my_schema +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Redeploy on direct: the schema now matches config, so the plan is a clean no-op >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy diff --git a/acceptance/bundle/migrate/auto/schema-rename/script b/acceptance/bundle/migrate/auto/schema-rename/script index 2286ccca1b2..273c43e1917 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/script +++ b/acceptance/bundle/migrate/auto/schema-rename/script @@ -10,17 +10,12 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate rm -f "$OUT_REQUESTS" -title "Rename the schema (an immutable id field). Without --auto-approve the recreate is refused" -# name is a provided id field, so a rename recreates. reconcileIDFields seeds the deployed -# name into the converted state, so the migration's plan sees a real recreate. Schemas are in -# the destructive-approval group, so without --auto-approve the deploy refuses the recreate -# (exit 1) and the migration is not committed - the run stays on terraform. +title "Rename the schema (an updatable id field) and deploy on direct" +# name is an updatable id field: the schema is renamed in place (UpdateWithID), so the deploy +# needs no --auto-approve. reconcileIDFields seeds the deployed name into the converted state, +# so the migration's plan sees a rename rather than snapshotting the new name as already applied. update_file.py databricks.yml "myschema-$UNIQUE_NAME" "myschema-$UNIQUE_NAME-renamed" -trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy -trace print_migration_telemetry - -title "With --auto-approve the recreate is applied on direct under the new name" -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy title "Redeploy on direct: the schema now matches config, so the plan is a clean no-op" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/script b/acceptance/bundle/migrate/auto/schema-storage-change/script index c0ab15856de..ef97ca99a82 100644 --- a/acceptance/bundle/migrate/auto/schema-storage-change/script +++ b/acceptance/bundle/migrate/auto/schema-storage-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: an explicit storage_root needs a registered external location on a real # workspace, so this is not Cloud-enabled. The recreate/refusal class it covers is verified -# on cloud by schema-rename and pipeline-storage-change. +# on cloud by schema-catalog-change and pipeline-storage-change. title "Replay the prior terraform deployment (captured from a real terraform run)" trace replay_tfstate.py terraform-requests.json terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/volume-rename/script b/acceptance/bundle/migrate/auto/volume-rename/script index 5ef772365b7..b4d3abb6b3b 100644 --- a/acceptance/bundle/migrate/auto/volume-rename/script +++ b/acceptance/bundle/migrate/auto/volume-rename/script @@ -10,7 +10,7 @@ title "Replay the prior terraform deployment (captured from a real terraform run trace replay_tfstate.py terraform-requests.json terraform.tfstate title "Rename the volume (an updatable id field) and deploy on direct" -# Unlike a schema (whose name recreates), a volume name is an updatable id field: it is +# A volume name is an updatable id field: it is # renamed in place (UpdateWithID). The migration seeds the deployed name into the converted # state, so its plan sees a rename and the migration succeeds; the deploy then renames the # volume rather than the migration snapshotting the new name as already applied. Verifies the diff --git a/acceptance/bundle/migrate/auto/volume-type-change/script b/acceptance/bundle/migrate/auto/volume-type-change/script index 6b02a55f247..ea5ebdb1499 100644 --- a/acceptance/bundle/migrate/auto/volume-type-change/script +++ b/acceptance/bundle/migrate/auto/volume-type-change/script @@ -2,7 +2,7 @@ export DATABRICKS_BUNDLE_ENGINE= # Testserver-only: changing volume_type to EXTERNAL needs a storage_location backed by a # registered external location on a real workspace, so this is not Cloud-enabled. The -# recreate class it covers is verified on cloud by schema-rename / registered-model-rename. +# recreate class it covers is verified on cloud by schema-catalog-change / registered-model-rename. # volume_type is both a provided id field and a recreate_on_changes field. reconcileIDFields # records the deployed value, so the migration's plan sees a real change and recreates. Volumes diff --git a/acceptance/bundle/migrate/command/basic/out.plan_update.json b/acceptance/bundle/migrate/command/basic/out.plan_update.json index b873a52a785..e39e6411c8e 100644 --- a/acceptance/bundle/migrate/command/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/command/basic/out.plan_update.json @@ -240,6 +240,11 @@ "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "remote": "[UUID]" } } } diff --git a/acceptance/bundle/refschema/output/schemas.txt b/acceptance/bundle/refschema/output/schemas.txt index 694d59c6211..566999fd664 100644 --- a/acceptance/bundle/refschema/output/schemas.txt +++ b/acceptance/bundle/refschema/output/schemas.txt @@ -23,7 +23,7 @@ resources.schemas.*.name string ALL resources.schemas.*.owner string REMOTE resources.schemas.*.properties map[string]string ALL resources.schemas.*.properties.* string ALL -resources.schemas.*.schema_id string REMOTE +resources.schemas.*.schema_id string REMOTE STATE resources.schemas.*.storage_location string REMOTE resources.schemas.*.storage_root string ALL resources.schemas.*.updated_at int64 REMOTE diff --git a/acceptance/bundle/refschema/output/volumes.txt b/acceptance/bundle/refschema/output/volumes.txt index 1c573ed1931..52e07594399 100644 --- a/acceptance/bundle/refschema/output/volumes.txt +++ b/acceptance/bundle/refschema/output/volumes.txt @@ -24,7 +24,7 @@ resources.volumes.*.storage_location string ALL resources.volumes.*.updated_at int64 REMOTE resources.volumes.*.updated_by string REMOTE resources.volumes.*.url string INPUT -resources.volumes.*.volume_id string REMOTE +resources.volumes.*.volume_id string REMOTE STATE resources.volumes.*.volume_path string INPUT resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt index 8da224f9751..d546194dba3 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.txt @@ -1,5 +1,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in catalog.VolumeInfo Error: planning failed diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt b/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt index 6a2b5d1157c..232d9f0b3d4 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.plan.txt @@ -1,4 +1,4 @@ -Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo +Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in dresources.VolumeState; non_existent: field "non_existent" not found in catalog.VolumeInfo Error: planning failed diff --git a/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename/out.test.toml b/acceptance/bundle/resources/catalogs/rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt new file mode 100644 index 00000000000..e07288217d4 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -0,0 +1,237 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } +} + +=== Rename the catalog +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +=== UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][2], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} + +=== The old catalog is gone and the schema and the volume are under the new one, with their original ids +old catalog is gone + +>>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]" +} + +=== The state tracks the new paths +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename/script b/acceptance/bundle/resources/catalogs/rename/script new file mode 100644 index 00000000000..e84ef070fc8 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/script @@ -0,0 +1,43 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the catalog" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +title "UC moves the schema and the volume with the catalog, so the plan re-keys them instead of recreating them" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + +title "Deploy renames the catalog. UC moves the schema and the volume with it, so they get no request" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The old catalog is gone and the schema and the volume are under the new one, with their original ids\n" +# Not traced: the "not found" message differs between UC and the test server. +$CLI catalogs get "$OLD_CATALOG" &> /dev/null && echo "old catalog still exists" || echo "old catalog is gone" +trace $CLI schemas get "$NEW_CATALOG.myschema" | jq '{full_name, catalog_name, schema_id}' +trace $CLI volumes read "$NEW_CATALOG.myschema.myvolume" | jq '{full_name, catalog_name, volume_id}' + +title "The state tracks the new paths" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/catalogs/rename/test.toml b/acceptance/bundle/resources/catalogs/rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt new file mode 100644 index 00000000000..bd744dd4686 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -0,0 +1,188 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume, the schema and the catalog out of band +>>> [CLI] volumes update test_catalog_[UNIQUE_NAME].myschema.myvolume --new-name myvolume2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume2" +} + +>>> [CLI] schemas update test_catalog_[UNIQUE_NAME].myschema --new-name myschema2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2" +} + +>>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== Rename all three in the config to match +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "create", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myschema", + "new": "myschema2", + "remote": "myschema2" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume2", + "schema_name": "myschema2", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", + "name": "myvolume2", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myvolume", + "new": "myvolume2", + "remote": "myvolume2" + }, + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myschema", + "new": "myschema2", + "remote": "myschema2" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== The catalog's create fails because the renamed catalog has its name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs +HTTP Status: 400 Bad Request +API error_code: CATALOG_ALREADY_EXISTS +API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists + +Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 + +Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 + +Files: 3 uploaded, 0 deleted + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script new file mode 100644 index 00000000000..9645fc0dc02 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + # The renamed catalog is not where the state looks for it, so destroy cannot remove it. + trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +} + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the volume, the schema and the catalog out of band" +trace $CLI volumes update "$OLD_CATALOG.myschema.myvolume" --new-name myvolume2 | jq "{full_name}" +trace $CLI schemas update "$OLD_CATALOG.myschema" --new-name myschema2 | jq "{full_name}" +trace $CLI catalogs update "$OLD_CATALOG" --new-name "$NEW_CATALOG" | jq "{name}" +print_requests.py //unity-catalog &> LOG.rename_requests + +title "Rename all three in the config to match" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +NEW_SCHEMA="$NEW_CATALOG.myschema2" +NEW_VOLUME="$NEW_SCHEMA.myvolume2" + +title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +plan + +title "The catalog's create fails because the renamed catalog has its name" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml new file mode 100644 index 00000000000..698c1c6ca49 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -0,0 +1,8 @@ +Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt new file mode 100644 index 00000000000..350b55d2a5e --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -0,0 +1,153 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog out of band. UC moves the schema and the volume with it +>>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]" +} + +=== Rename the catalog in the config to match +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "create", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "id": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== The catalog's create fails because the renamed catalog has its name +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs +HTTP Status: 400 Bad Request +API error_code: CATALOG_ALREADY_EXISTS +API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists + +Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 + +Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 + +Files: 3 uploaded, 0 deleted + +>>> errcode [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> errcode [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script new file mode 100644 index 00000000000..8b161d76a37 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_CATALOG="test_catalog_${UNIQUE_NAME}" +NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + # The renamed catalog is not where the state looks for it, so destroy cannot remove it. + trace errcode $CLI catalogs delete "$NEW_CATALOG" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location +} + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the catalog out of band. UC moves the schema and the volume with it" +trace $CLI catalogs update "$OLD_CATALOG" --new-name "$NEW_CATALOG" | jq "{name}" +print_requests.py //unity-catalog &> LOG.rename_requests + +title "Rename the catalog in the config to match" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +NEW_SCHEMA="$NEW_CATALOG.myschema" +NEW_VOLUME="$NEW_SCHEMA.myvolume" + +title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +plan + +title "The catalog's create fails because the renamed catalog has its name" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml new file mode 100644 index 00000000000..698c1c6ca49 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -0,0 +1,8 @@ +Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl new file mode 100644 index 00000000000..90953dac6dc --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/databricks.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + comment: c1 diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml b/acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt b/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt new file mode 100644 index 00000000000..0359d87b31a --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/output.txt @@ -0,0 +1,47 @@ + +=== Deploy a catalog with a comment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog and change its comment in one deploy: both go in one PATCH +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "c2", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} + +=== Both landed, so the redeploy is a no-op +>>> [CLI] catalogs get test_catalog_renamed_[UNIQUE_NAME] +{ + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/script b/acceptance/bundle/resources/catalogs/rename_with_comment/script new file mode 100644 index 00000000000..f3c87c55be4 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a comment" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the catalog and change its comment in one deploy: both go in one PATCH" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "comment: c1" "comment: c2" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Both landed, so the redeploy is a no-op" +trace $CLI catalogs get "test_catalog_renamed_${UNIQUE_NAME}" | jq '{name, comment}' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/catalogs/rename_with_comment/test.toml b/acceptance/bundle/resources/catalogs/rename_with_comment/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/rename_with_comment/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/grants/schemas/change_name/output.txt b/acceptance/bundle/resources/grants/schemas/change_name/output.txt index 6c4e069ee0c..b7009427f4b 100644 --- a/acceptance/bundle/resources/grants/schemas/change_name/output.txt +++ b/acceptance/bundle/resources/grants/schemas/change_name/output.txt @@ -54,28 +54,19 @@ schemas schema1.grants id='schema/main.test-schema-[UNIQUE_NAME]' === Deploy moves the schema to the new name and applies the grants there >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - recreate resources.schemas.schema1 -Recreated schemas.schema1 +Updated schemas.schema1 Updated schemas.schema1.grants Files: 1 uploaded, 0 deleted -Resources: 1 created, 1 changed, 1 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged >>> print_requests.py //unity-catalog { - "method": "DELETE", + "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", - "q": { - "force": "true" - } -} -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", "body": { - "catalog_name": "main", - "name": "test-schema-renamed-[UNIQUE_NAME]" + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } { diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json index 4586c6ab570..f97896ce484 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.json @@ -18,6 +18,14 @@ "name": "schema_dup_grants_[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.schemas.apps_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json index 038a187a8ed..ed4a70c40d4 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.json @@ -18,6 +18,14 @@ "name": "schema_out_of_band_principal_[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.schemas.grants_schema.grants": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json index 3b9720936dc..160c56686a0 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.json @@ -18,6 +18,14 @@ "name": "schema_out_of_band_principal_[UNIQUE_NAME]", "owner": "[USERNAME]", "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.schemas.grants_schema.grants": { diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt b/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt index ff470fc8acd..43a7d78e7bb 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt @@ -13,6 +13,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "comment": { "action": "update", "remote": "set outside the bundle" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt index 66b4711e1d6..9643739f832 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -25,6 +25,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "unity.catalog.managed.delta.defaults.delta.parquet.format.version.afe.internal": "2.12.0", "unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true" } + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..a965908256c --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + storage_root: s3://mybucket/a + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt new file mode 100644 index 00000000000..8d92949d3e0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/output.txt @@ -0,0 +1,126 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml s3://mybucket/a s3://mybucket/b + +>>> errcode [CLI] bundle deploy -q --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.schema1 + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "q": { + "force": "true" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_root": "s3://mybucket/b" + } +} + +=== The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it +>>> [CLI] bundle plan --output json +{ + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" + } + } +} + +=== The next deploy creates a new volume +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created volumes.volume1 +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[UUID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/script b/acceptance/bundle/resources/schemas/recreate_interrupted/script new file mode 100644 index 00000000000..df501ef6124 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/script @@ -0,0 +1,36 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_VOLUME="main.test-schema-${UNIQUE_NAME}.myvolume" +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema and change its storage_root, which recreates it. The CLI is killed on the volume's recreate, after the schema's" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml s3://mybucket/a s3://mybucket/b +kill_after.py "DELETE /api/2.1/unity-catalog/volumes/$OLD_VOLUME" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q --auto-approve +trace print_requests.py //unity-catalog + +title "The volume went with the old schema. Nothing with its volume_id is at its new path, so the plan creates it" +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' + +title "The next deploy creates a new volume" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml new file mode 100644 index 00000000000..e149a3f7289 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_interrupted/test.toml @@ -0,0 +1,36 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace +# cannot, and an explicit storage_root needs an external location on a real workspace. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# approval text, so its last line is followed directly by "Exit code: 1". +[[Repls]] +Old = '( recreate resources\.volumes\.volume1)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt new file mode 100644 index 00000000000..36e38a1c807 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/output.txt @@ -0,0 +1,101 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, drop the schema and create another one under a new name, with a volume of the same name +>>> [CLI] schemas delete main.test-schema-[UNIQUE_NAME] --force + +=== With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[NEW_SCHEMA_ID]" +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[NEW_VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "schema_name": { + "action": "recreate", + "reason": "id_field", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 0 deleted + +>>> [CLI] schemas delete main.test-schema-renamed-[UNIQUE_NAME] --force diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/script b/acceptance/bundle/resources/schemas/recreate_out_of_band/script new file mode 100644 index 00000000000..cadca8544af --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/script @@ -0,0 +1,39 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + # The schema created out of band is not in the state, so destroy does not remove it. + trace $CLI schemas delete "$NEW_SCHEMA" --force + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Out of band, drop the schema and create another one under a new name, with a volume of the same name" +trace $CLI schemas delete "$OLD_SCHEMA" --force +add_repl "$($CLI schemas create "test-schema-renamed-${UNIQUE_NAME}" main | jq -r .schema_id)" NEW_SCHEMA_ID +add_repl "$($CLI volumes create main "test-schema-renamed-${UNIQUE_NAME}" myvolume MANAGED | jq -r .volume_id)" NEW_VOLUME_ID +print_requests.py //unity-catalog &> LOG.recreate_requests + +title "With the config renamed to match, the plan finds a schema and a volume at the new names, but their ids differ, so it plans creates" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace print_state.py | jq '.state["resources.schemas.schema1"] | {__id__, schema_id: .state.schema_id}' +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +plan diff --git a/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml b/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/recreate_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl new file mode 100644 index 00000000000..080c703a351 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: COMMENT1 diff --git a/acceptance/bundle/resources/schemas/rename/out.test.toml b/acceptance/bundle/resources/schemas/rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename/output.txt b/acceptance/bundle/resources/schemas/rename/output.txt new file mode 100644 index 00000000000..57cab932c15 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/output.txt @@ -0,0 +1,124 @@ + +=== Deploy the schema +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "comment": "COMMENT1", + "name": "test-schema-[UNIQUE_NAME]" + } +} + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== The plan renames the schema in place instead of recreating it +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "COMMENT1", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "COMMENT1", + "created_at": [UNIX_TIME_MILLIS], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + } +} + +=== Deploy renames the schema without a deletion prompt +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "COMMENT1", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The schema is only reachable under the new name, with its original schema_id +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "comment": "COMMENT1", + "schema_id": "[SCHEMA_ID]" +} + +=== The state and summary track the new name +>>> [CLI] bundle summary --output json +{ + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename/script b/acceptance/bundle/resources/schemas/rename/script new file mode 100644 index 00000000000..c8617676747 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/script @@ -0,0 +1,40 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy the schema" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "The plan renames the schema in place instead of recreating it" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by effective_predictive_optimization_flag + +title "Deploy renames the schema without a deletion prompt" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The schema is only reachable under the new name, with its original schema_id" +trace musterr $CLI schemas get "$OLD_SCHEMA" +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, name, comment, schema_id}' + +title "The state and summary track the new name" +trace $CLI bundle summary --output json | jq '.resources.schemas.schema1 | {id, name}' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename/test.toml b/acceptance/bundle/resources/schemas/rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl new file mode 100644 index 00000000000..1a1d53a02c5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/databricks.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test_schema_$UNIQUE_NAME diff --git a/acceptance/bundle/resources/schemas/rename_case_only/out.test.toml b/acceptance/bundle/resources/schemas/rename_case_only/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_case_only/output.txt b/acceptance/bundle/resources/schemas/rename_case_only/output.txt new file mode 100644 index 00000000000..195bbc44832 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/output.txt @@ -0,0 +1,90 @@ + +=== Deploy a schema with a lowercase name +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Change only the case of the name. UC lowercases names, so it is the same name +>>> update_file.py databricks.yml test_schema_ Test_Schema_ + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "Test_Schema_[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "test_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_schema_[UNIQUE_NAME]", + "new": "Test_Schema_[UNIQUE_NAME]", + "remote": "test_schema_[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + } +} + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test_schema_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "Test_Schema_[UNIQUE_NAME]" + } +} + +>>> [CLI] schemas get main.test_schema_[UNIQUE_NAME] +{ + "full_name": "main.test_schema_[UNIQUE_NAME]", + "name": "test_schema_[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +=== Redeploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> errcode [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename_case_only/script b/acceptance/bundle/resources/schemas/rename_case_only/script new file mode 100644 index 00000000000..a15de5ccca5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a lowercase name" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "main.test_schema_${UNIQUE_NAME}" | jq -r .schema_id)" SCHEMA_ID + +title "Change only the case of the name. UC lowercases names, so it is the same name" +trace update_file.py databricks.yml test_schema_ Test_Schema_ +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by effective_predictive_optimization_flag +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI schemas get "main.test_schema_${UNIQUE_NAME}" | jq '{full_name, name, schema_id}' + +title "Redeploy" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_case_only/test.toml b/acceptance/bundle/resources/schemas/rename_case_only/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_case_only/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl new file mode 100644 index 00000000000..2964856d08e --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/output.txt b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt new file mode 100644 index 00000000000..6c768de425a --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/output.txt @@ -0,0 +1,143 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> errcode [CLI] bundle deploy -q +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +[PROCESS_KILLED] + +Exit code: [KILLED] + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The volume is gone from its recorded path, so the plan looks for it at its path in config +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "comment": "c1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "storage_location": { + "action": "skip", + "reason": "immutable_backend_value", + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[VOLUME_ID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } +} + +=== The next deploy updates the volume at its new path +>>> [CLI] bundle deploy --force-lock +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2" + } +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "comment": "c2", + "volume_id": "[VOLUME_ID]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve --force-lock +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/script b/acceptance/bundle/resources/schemas/rename_interrupted/script new file mode 100644 index 00000000000..76ee184840a --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/script @@ -0,0 +1,36 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve --force-lock + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "main.test-schema-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema and change the volume's comment. The CLI is killed on the volume's update, after the schema's rename is saved" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "comment: c1" "comment: c2" +kill_after.py "PATCH /api/2.1/unity-catalog/volumes/$NEW_VOLUME" 0 1 +# -q: the per-resource lines race the kill signal. +trace errcode $CLI bundle deploy -q +trace print_requests.py //unity-catalog + +title "The volume is gone from its recorded path, so the plan looks for it at its path in config" +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' +trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"]' + +title "The next deploy updates the volume at its new path" +trace $CLI bundle deploy --force-lock +trace print_requests.py //unity-catalog +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, comment, volume_id}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_interrupted/test.toml b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml new file mode 100644 index 00000000000..443e541f555 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_interrupted/test.toml @@ -0,0 +1,35 @@ +# Local-only: the test server kills the CLI mid-deploy (kill_after.py), which a real workspace cannot. +Cloud = false + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# KillCaller needs the CLI's pid. A kill leaves no state in DMS to recover from, so DMS is not run. +Env.DATABRICKS_CLI_TEST_PID = "1" +EnvMatrix.DMS = [""] + +# Normalize how a killed process shows up on each OS, as in bundle/deploy/wal. +[[Repls]] +Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' +New = '[PROCESS_KILLED]' + +[[Repls]] +Old = '(\n>>> errcode [^\n]+\n)\nExit code:' +New = "${1}[PROCESS_KILLED]\n\nExit code:" + +[[Repls]] +Old = 'Exit code: 137' +New = 'Exit code: [KILLED]' + +[[Repls]] +Old = '(\[PROCESS_KILLED\]\n\nExit code: )1' +New = '${1}[KILLED]' + +# On Windows a killed CLI leaves no bash "Killed" line. The kill lands right after the +# "Uploading bundle files" line, so it is followed directly by "Exit code: 1". +[[Repls]] +Old = '(Uploading bundle files to [^\n]+)\n\nExit code: 1' +New = "${1}\n[PROCESS_KILLED]\n\nExit code: [KILLED]" diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt new file mode 100644 index 00000000000..d648ff373ce --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/output.txt @@ -0,0 +1,164 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band. UC moves the volume with it +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +=== With the config unchanged, the plan creates the schema and the volume again under their configured names +>>> [CLI] bundle plan +create schemas.schema1 +create volumes.volume1 + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +=== With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "name": "test-schema-renamed-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "id": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "skip", + "reason": "remote_already_set", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-renamed-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== Deploy only records the new names in the state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +{ + "schemas": { + "schema1": "main.test-schema-renamed-[UNIQUE_NAME]" + }, + "volumes": { + "volume1": "main.test-schema-renamed-[UNIQUE_NAME].myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/script b/acceptance/bundle/resources/schemas/rename_out_of_band/script new file mode 100644 index 00000000000..48f43a5ebc4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "$OLD_SCHEMA" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "$OLD_SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema out of band. UC moves the volume with it" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, schema_id}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config unchanged, the plan creates the schema and the volume again under their configured names" +trace $CLI bundle plan + +title "With the config renamed to match, the plan finds both at their new names, with the same schema_id and volume_id" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +plan + +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml b/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl new file mode 100644 index 00000000000..663acec3be0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema1-$UNIQUE_NAME + schema2: + catalog_name: main + name: test-schema2-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml b/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_then_change/output.txt b/acceptance/bundle/resources/schemas/rename_then_change/output.txt new file mode 100644 index 00000000000..a3f6f27efc7 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/output.txt @@ -0,0 +1,273 @@ + +=== Deploy two schemas and a volume in the first one +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created schemas.schema2 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --sort //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema1-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema2-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c1", + "name": "myvolume", + "schema_name": "test-schema1-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the volume's schema: the volume is re-keyed without a request +>>> update_file.py databricks.yml test-schema1- test-schema1-renamed- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-renamed-[UNIQUE_NAME]" + } +} + +=== After the rename, update, update_id and recreate of the volume all reach it at its new path +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2" + } +} + +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume", + "body": { + "comment": "c2", + "new_name": "myvolume2" + } +} + +>>> update_file.py databricks.yml schemas.schema1.name schemas.schema2.name + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema1-renamed-[UNIQUE_NAME].myvolume2" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c2", + "name": "myvolume2", + "schema_name": "test-schema2-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the volume's new schema: the volume is re-keyed again +>>> update_file.py databricks.yml test-schema2- test-schema2-renamed- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema2 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema2-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema2-renamed-[UNIQUE_NAME]" + } +} + +=== Update, update_id and recreate work again, moving the volume back to the first schema +>>> update_file.py databricks.yml comment: c2 comment: c3 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", + "body": { + "comment": "c3" + } +} + +>>> update_file.py databricks.yml name: myvolume2 name: myvolume + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume2", + "body": { + "comment": "c3", + "new_name": "myvolume" + } +} + +>>> update_file.py databricks.yml schemas.schema2.name schemas.schema1.name + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.volume1 +Recreated volumes.volume1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/volumes/main.test-schema2-renamed-[UNIQUE_NAME].myvolume" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "comment": "c3", + "name": "myvolume", + "schema_name": "test-schema1-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the first schema back: the volume is re-keyed to its original path +>>> update_file.py databricks.yml test-schema1-renamed- test-schema1- + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema1-renamed-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema1-[UNIQUE_NAME]" + } +} + +=== The state tracks the current paths and the plan is a no-op +>>> [CLI] bundle summary --output json +{ + "schemas": { + "schema1": "main.test-schema1-[UNIQUE_NAME]", + "schema2": "main.test-schema2-renamed-[UNIQUE_NAME]" + }, + "volumes": { + "volume1": "main.test-schema1-[UNIQUE_NAME].myvolume" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] volumes read main.test-schema1-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema1-[UNIQUE_NAME].myvolume", + "comment": "c3" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + delete resources.schemas.schema2 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_then_change/script b/acceptance/bundle/resources/schemas/rename_then_change/script new file mode 100644 index 00000000000..452af92148c --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/script @@ -0,0 +1,52 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +deploy() { + trace $CLI bundle deploy "$@" + trace print_requests.py //unity-catalog +} + +title "Deploy two schemas and a volume in the first one" +trace $CLI bundle deploy +# The two schemas are created in parallel. +trace print_requests.py --sort //unity-catalog + +title "Rename the volume's schema: the volume is re-keyed without a request" +trace update_file.py databricks.yml test-schema1- test-schema1-renamed- +deploy + +title "After the rename, update, update_id and recreate of the volume all reach it at its new path" +trace update_file.py databricks.yml "comment: c1" "comment: c2" +deploy +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +deploy +trace update_file.py databricks.yml schemas.schema1.name schemas.schema2.name +deploy --auto-approve + +title "Rename the volume's new schema: the volume is re-keyed again" +trace update_file.py databricks.yml test-schema2- test-schema2-renamed- +deploy + +title "Update, update_id and recreate work again, moving the volume back to the first schema" +trace update_file.py databricks.yml "comment: c2" "comment: c3" +deploy +trace update_file.py databricks.yml "name: myvolume2" "name: myvolume" +deploy +trace update_file.py databricks.yml schemas.schema2.name schemas.schema1.name +deploy --auto-approve + +title "Rename the first schema back: the volume is re-keyed to its original path" +trace update_file.py databricks.yml test-schema1-renamed- test-schema1- +deploy + +title "The state tracks the current paths and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan +trace $CLI volumes read "main.test-schema1-${UNIQUE_NAME}.myvolume" | jq '{full_name, comment}' diff --git a/acceptance/bundle/resources/schemas/rename_then_change/test.toml b/acceptance/bundle/resources/schemas/rename_then_change/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_then_change/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl new file mode 100644 index 00000000000..d73aacf0d14 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: c1 diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/output.txt b/acceptance/bundle/resources/schemas/rename_with_comment/output.txt new file mode 100644 index 00000000000..f685df69adc --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/output.txt @@ -0,0 +1,51 @@ + +=== Deploy a schema with a comment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and change its comment in one deploy: both go in one PATCH +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "c2", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== Both landed, so the redeploy is a no-op +>>> [CLI] schemas get main.test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]", + "comment": "c2", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/script b/acceptance/bundle/resources/schemas/rename_with_comment/script new file mode 100644 index 00000000000..96cdcd68160 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/script @@ -0,0 +1,24 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a comment" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "main.test-schema-${UNIQUE_NAME}" | jq -r .schema_id)" SCHEMA_ID + +title "Rename the schema and change its comment in one deploy: both go in one PATCH" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace update_file.py databricks.yml "comment: c1" "comment: c2" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Both landed, so the redeploy is a no-op" +trace $CLI schemas get "main.test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name, comment, schema_id}' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_with_comment/test.toml b/acceptance/bundle/resources/schemas/rename_with_comment/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_comment/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt new file mode 100644 index 00000000000..bb058ef40fd --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/output.txt @@ -0,0 +1,178 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "test-schema-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } +} + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "test-schema-renamed-[UNIQUE_NAME]" + } + }, + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test-schema-[UNIQUE_NAME]", + "new": "test-schema-renamed-[UNIQUE_NAME]", + "remote": "test-schema-[UNIQUE_NAME]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== Deploy renames the schema. UC moves the volume with it, so the volume gets no request +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +=== The old schema is gone and the volume is under the new one, with its original volume_id +>>> musterr [CLI] schemas get main.test-schema-[UNIQUE_NAME] +Error: Schema 'main.test-schema-[UNIQUE_NAME]' does not exist. + +>>> [CLI] volumes read main.test-schema-renamed-[UNIQUE_NAME].myvolume +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]" +} + +=== The state tracks the new path +>>> [CLI] bundle summary --output json +"main.test-schema-renamed-[UNIQUE_NAME].myvolume" + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/script b/acceptance/bundle/resources/schemas/rename_with_volume/script new file mode 100644 index 00000000000..fd5830bf18a --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/script @@ -0,0 +1,41 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +OLD_VOLUME="$OLD_SCHEMA.myvolume" +NEW_VOLUME="main.test-schema-renamed-${UNIQUE_NAME}.myvolume" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI volumes read "$OLD_VOLUME" | jq -r .volume_id)" VOLUME_ID + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "UC moves the volume with the schema, so the plan re-keys the volume instead of recreating it" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location + +title "Deploy renames the schema. UC moves the volume with it, so the volume gets no request" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The old schema is gone and the volume is under the new one, with its original volume_id" +trace musterr $CLI schemas get "$OLD_SCHEMA" +trace $CLI volumes read "$NEW_VOLUME" | jq '{full_name, schema_name, volume_id}' + +title "The state tracks the new path" +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' + +title "Redeploy is a no-op" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/rename_with_volume/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl new file mode 100644 index 00000000000..bddda4baff0 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume + jobs: + job1: + name: test-job-$UNIQUE_NAME + parameters: + - name: volume_full_name + default: ${resources.volumes.volume1.full_name} + - name: volume_id + default: ${resources.volumes.volume1.id} + - name: volume_uuid + default: ${resources.volumes.volume1.volume_id} diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml b/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt new file mode 100644 index 00000000000..8807308a705 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/output.txt @@ -0,0 +1,93 @@ + +=== Deploy a schema, a volume in it and a job that reads the volume's full_name and id +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.job1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [JOB_ID] +[ + { + "default": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "volume_full_name" + }, + { + "default": "main.test-schema-[UNIQUE_NAME].myvolume", + "name": "volume_id" + }, + { + "default": "[VOLUME_ID]", + "name": "volume_uuid" + } +] + +=== Rename the schema +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +=== The volume's full_name and id change with the schema, so the job is updated too +>>> [CLI] bundle plan +update jobs.job1 +update schemas.schema1 + +Plan: 0 to add, 2 to change, 0 to delete, 1 unchanged + +=== Deploy passes the volume's new full_name and id to the job +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated jobs.job1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test-schema-renamed-[UNIQUE_NAME]" + } +} + +>>> [CLI] jobs get [JOB_ID] +[ + { + "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "volume_full_name" + }, + { + "default": "main.test-schema-renamed-[UNIQUE_NAME].myvolume", + "name": "volume_id" + }, + { + "default": "[VOLUME_ID]", + "name": "volume_uuid" + } +] + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/script b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script new file mode 100644 index 00000000000..6418f29011d --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/script @@ -0,0 +1,31 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema, a volume in it and a job that reads the volume's full_name and id" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +JOB_ID=$($CLI bundle summary --output json | jq -r .resources.jobs.job1.id) +add_repl "$JOB_ID" JOB_ID +add_repl "$($CLI volumes read "main.test-schema-${UNIQUE_NAME}.myvolume" | jq -r .volume_id)" VOLUME_ID +trace $CLI jobs get "$JOB_ID" | jq .settings.parameters + +title "Rename the schema" +trace update_file.py databricks.yml test-schema- test-schema-renamed- + +title "The volume's full_name and id change with the schema, so the job is updated too" +trace $CLI bundle plan + +title "Deploy passes the volume's new full_name and id to the job" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI jobs get "$JOB_ID" | jq .settings.parameters + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml b/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_with_volume_ref/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.json b/acceptance/bundle/resources/volumes/change-name/out.plan.json index eb9308f572b..702a7ec3d5c 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.json @@ -41,6 +41,12 @@ "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } } diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt b/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt index b6619370518..3875fc83ad6 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt @@ -19,6 +19,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "action": "skip", "reason": "immutable_backend_value", "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt new file mode 100644 index 00000000000..b8b36ce7130 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/output.txt @@ -0,0 +1,94 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Out of band, delete the volume and create another one under a new name +>>> [CLI] volumes delete main.test-schema-[UNIQUE_NAME].myvolume + +=== With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> print_state.py +{ + "__id__": "main.test-schema-[UNIQUE_NAME].myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[NEW_VOLUME_ID]" +} + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume2", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myvolume", + "new": "myvolume2" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]" + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/script b/acceptance/bundle/resources/volumes/recreate_out_of_band/script new file mode 100644 index 00000000000..eaaa5d0268a --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + # Deleting the schema deletes the volumes in it, the one created out of band too. + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Out of band, delete the volume and create another one under a new name" +trace $CLI volumes delete "$SCHEMA.myvolume" +NEW_VOLUME_ID=$($CLI volumes create main "test-schema-${UNIQUE_NAME}" myvolume2 MANAGED | jq -r .volume_id) +add_repl "$NEW_VOLUME_ID" NEW_VOLUME_ID +print_requests.py //unity-catalog &> LOG.recreate_requests + +title "With the config renamed to match, the plan finds a volume at the new name, but its volume_id differs, so it plans a create" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +trace print_state.py | jq '.state["resources.volumes.volume1"] | {__id__, volume_id: .state.volume_id}' +trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' +plan diff --git a/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml b/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/recreate_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl new file mode 100644 index 00000000000..ca71c34c3aa --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test_schema_$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_case_only/out.test.toml b/acceptance/bundle/resources/volumes/rename_case_only/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_case_only/output.txt b/acceptance/bundle/resources/volumes/rename_case_only/output.txt new file mode 100644 index 00000000000..cdbdd33c65e --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/output.txt @@ -0,0 +1,126 @@ + +=== Deploy a volume with a lowercase name +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Change only the case of the name. UC lowercases names, so it is the same name +>>> update_file.py databricks.yml name: myvolume name: MyVolume + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test_schema_[UNIQUE_NAME]", + "name": "test_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "main", + "name": "MyVolume", + "schema_name": "test_schema_[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test_schema_[UNIQUE_NAME].myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "test_schema_[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myvolume", + "new": "MyVolume", + "remote": "myvolume" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/volumes/main.test_schema_[UNIQUE_NAME].myvolume", + "body": { + "comment": "", + "new_name": "MyVolume" + } +} + +>>> [CLI] volumes read main.test_schema_[UNIQUE_NAME].myvolume +{ + "full_name": "main.test_schema_[UNIQUE_NAME].myvolume", + "name": "myvolume", + "volume_id": "[VOLUME_ID]" +} + +=== Redeploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> errcode [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/volumes/rename_case_only/script b/acceptance/bundle/resources/volumes/rename_case_only/script new file mode 100644 index 00000000000..16608975431 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +VOLUME="main.test_schema_${UNIQUE_NAME}.myvolume" + +cleanup() { + trace errcode $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a volume with a lowercase name" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$VOLUME" | jq -r .volume_id)" VOLUME_ID + +title "Change only the case of the name. UC lowercases names, so it is the same name" +trace update_file.py databricks.yml "name: myvolume" "name: MyVolume" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI volumes read "$VOLUME" | jq '{full_name, name, volume_id}' + +title "Redeploy" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_case_only/test.toml b/acceptance/bundle/resources/volumes/rename_case_only/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_case_only/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..e45eccddc44 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + volumes: + volume1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt new file mode 100644 index 00000000000..e8fd1ecfdde --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -0,0 +1,223 @@ + +=== Deploy a catalog with a schema and a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the catalog and the schema in one deploy +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +=== Both of the volume's parents are renamed, so both its catalog_name and schema_name follow them +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[SCHEMA_ID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[SCHEMA_ID]", + "remote": "[SCHEMA_ID]" + } + } + }, + "resources.volumes.volume1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myvolume", + "schema_name": "myschema2", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "created_at": [UNIX_TIME_MILLIS][2], + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume", + "name": "myvolume", + "owner": "[USERNAME]", + "schema_name": "myschema", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== Deploy renames the catalog, then the schema under it. UC moves the volume both times, so it gets no request +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_renamed_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +=== The schema and the volume are under the new names, with their original ids +>>> [CLI] schemas get test_catalog_renamed_[UNIQUE_NAME].myschema2 +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", + "schema_id": "[SCHEMA_ID]" +} + +>>> [CLI] volumes read test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume" + } +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script new file mode 100644 index 00000000000..7b2ea667360 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/script @@ -0,0 +1,37 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +NEW_SCHEMA="test_catalog_renamed_${UNIQUE_NAME}.myschema2" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a catalog with a schema and a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI schemas get "test_catalog_${UNIQUE_NAME}.myschema" | jq -r .schema_id)" SCHEMA_ID +add_repl "$($CLI volumes read "test_catalog_${UNIQUE_NAME}.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the catalog and the schema in one deploy" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "name: myschema" "name: myschema2" + +title "Both of the volume's parents are renamed, so both its catalog_name and schema_name follow them" +# Drop the remote_state fields that differ between clouds or from the test server. +trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + +title "Deploy renames the catalog, then the schema under it. UC moves the volume both times, so it gets no request" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "The schema and the volume are under the new names, with their original ids" +trace $CLI schemas get "$NEW_SCHEMA" | jq '{full_name, schema_id}' +trace $CLI volumes read "$NEW_SCHEMA.myvolume" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..2234fdf2e39 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + volumes: + volume1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: myvolume diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt new file mode 100644 index 00000000000..f62a0c3920a --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/output.txt @@ -0,0 +1,135 @@ + +=== Deploy a schema with a volume in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created volumes.volume1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume out of band +>>> [CLI] volumes update main.test-schema-[UNIQUE_NAME].myvolume --new-name myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[VOLUME_ID]" +} + +=== With the config unchanged, the plan creates the volume again under its configured name +>>> [CLI] bundle plan +create volumes.volume1 + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== With the config renamed to match, the plan finds the volume at its new name, with the same volume_id +>>> update_file.py databricks.yml name: myvolume name: myvolume2 + +>>> [CLI] bundle plan --output json +{ + "resources.schemas.schema1": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "enable_predictive_optimization": "INHERIT", + "full_name": "main.test-schema-[UNIQUE_NAME]", + "name": "test-schema-[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.volumes.volume1": { + "id": "main.test-schema-[UNIQUE_NAME].myvolume2", + "depends_on": [ + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "main", + "name": "myvolume2", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_type": "MANAGED" + } + }, + "remote_state": { + "catalog_name": "main", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "name": "myvolume2", + "owner": "[USERNAME]", + "schema_name": "test-schema-[UNIQUE_NAME]", + "volume_id": "[VOLUME_ID]", + "volume_type": "MANAGED" + }, + "changes": { + "name": { + "action": "skip", + "reason": "remote_already_set", + "old": "myvolume", + "new": "myvolume2", + "remote": "myvolume2" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id", + "old": "[VOLUME_ID]", + "remote": "[VOLUME_ID]" + } + } + } +} + +=== Deploy only records the new name in the state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog + +>>> [CLI] volumes read main.test-schema-[UNIQUE_NAME].myvolume2 +{ + "full_name": "main.test-schema-[UNIQUE_NAME].myvolume2", + "volume_id": "[VOLUME_ID]" +} + +>>> [CLI] bundle summary --output json +"main.test-schema-[UNIQUE_NAME].myvolume2" + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + delete resources.volumes.volume1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/script b/acceptance/bundle/resources/volumes/rename_out_of_band/script new file mode 100644 index 00000000000..c16fde22dbc --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/script @@ -0,0 +1,41 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +SCHEMA="main.test-schema-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id effective_predictive_optimization_flag storage_location +} + +title "Deploy a schema with a volume in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI volumes read "$SCHEMA.myvolume" | jq -r .volume_id)" VOLUME_ID + +title "Rename the volume out of band" +trace $CLI volumes update "$SCHEMA.myvolume" --new-name myvolume2 | jq '{full_name, volume_id}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config unchanged, the plan creates the volume again under its configured name" +trace $CLI bundle plan + +title "With the config renamed to match, the plan finds the volume at its new name, with the same volume_id" +trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" +plan + +title "Deploy only records the new name in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI volumes read "$SCHEMA.myvolume2" | jq '{full_name, volume_id}' +trace $CLI bundle summary --output json | jq '.resources.volumes.volume1.id' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/volumes/rename_out_of_band/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt index bba6a759059..f58c45bb68f 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.txt @@ -33,7 +33,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "changes": { "name": { "action": "skip", - "reason": "id_field" + "reason": "id_changes" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id" } } }, @@ -51,6 +55,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "storage_location": { "action": "skip", "reason": "immutable_backend_value" + }, + "volume_id": { + "action": "skip", + "reason": "internal_id" } } } diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.json index 31bc7b0724a..e31832f7f4a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.json @@ -249,7 +249,8 @@ "__id__": "mycatalog.myschema", "state": { "catalog_name": "mycatalog", - "name": "myschema" + "name": "myschema", + "schema_id": "[UUID]" } } } diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index e53ef0d84bd..408e2a143f5 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -12,8 +12,8 @@ type Action struct { // Gone mirrors PlanEntry.Gone: the delete is a state-only cleanup because the // resource no longer exists remotely. Gone bool - // StateOnly mirrors PlanEntry.StateOnly: the delete is a state-only cleanup - // because the resource implements no DoDelete (deleting it has no backend effect). + // StateOnly mirrors PlanEntry.StateOnly: applying the action only changes the state + // (a delete of a resource without DoDelete, or a re-key). StateOnly bool } @@ -21,12 +21,10 @@ func (a Action) String() string { return fmt.Sprintf(" %s %s", a.ActionType.StringShort(), a.ResourceKey) } -// IsStateOnlyDelete reports whether applying this delete only drops the state entry -// without any backend call: the resource is already gone remotely (Gone) or has no -// delete operation (StateOnly). Such deletes are omitted from human output, excluded -// from the resource counts, and need no destructive-action approval. See the same -// method on PlanEntry. -func (a Action) IsStateOnlyDelete() bool { +// IsStateOnly reports whether applying this action makes no backend call and only changes the +// state (see PlanEntry.IsStateOnly). Such actions are omitted from human output and need no +// approval. +func (a Action) IsStateOnly() bool { return a.Gone || a.StateOnly } diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 8446b200b12..af154ab8740 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -60,13 +60,18 @@ func (p *Plan) CountActions() ActionCounts { case Create: c.Create++ case Update, UpdateWithID, Resize: + if entry.StateOnly { + // A re-key changes nothing in the backend. + c.Unchanged++ + continue + } c.Change++ case Delete: // A state-only delete touches nothing in the backend and only drops the // state entry, so it is not a real action: leave it out of the tally // entirely rather than misreport it as deleted or unchanged. This matches // how `bundle destroy` counts its own deletions. - if entry.IsStateOnlyDelete() { + if entry.IsStateOnly() { continue } c.Delete++ @@ -119,6 +124,8 @@ func LoadPlanFromFile(path string) (*Plan, error) { } type PlanEntry struct { + // ID is set when the resource was found at another ID than the one in state: a rename + // moved it there (see findMoved). Applying the entry addresses the resource at ID. ID string `json:"id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` @@ -126,23 +133,25 @@ type PlanEntry struct { // exists remotely. Applying such an entry only removes it from the state, without // calling the delete API, and approval prompts do not list it as a deletion. Gone bool `json:"gone,omitempty"` - // StateOnly is set on Delete entries for resources that implement no DoDelete: - // deleting them has no backend effect. Like Gone, applying such an entry only - // removes it from the state and it is excluded from destructive-action prompts, - // textual plan output and the deleted count — but unlike Gone it is a property of - // the resource type, not of the current remote state, so planning skips the - // remote read that Gone detection needs. + // StateOnly marks an entry whose apply has no backend effect and only changes the state: + // - a Delete of a resource that implements no DoDelete. Like Gone, applying it only + // removes it from the state and it is excluded from destructive-action prompts, + // textual plan output and the deleted count — but unlike Gone it is a property of + // the resource type, not of the current remote state, so planning skips the remote + // read that Gone detection needs. + // - an UpdateWithID that only follows a parent's rename (or a rename found by the plan): + // the backend already moved the resource, so applying it records the state under the + // ID IDFromState gives it. It is left out of the textual plan and counted as unchanged. StateOnly bool `json:"state_only,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` RemoteState any `json:"remote_state,omitempty"` Changes Changes `json:"changes,omitempty"` } -// IsStateOnlyDelete reports whether applying this delete only drops the state entry -// without any backend call: the resource is already gone remotely (Gone) or has no -// delete operation (StateOnly). Such deletes are omitted from human output, excluded -// from the resource counts, and need no destructive-action approval. -func (e *PlanEntry) IsStateOnlyDelete() bool { +// IsStateOnly reports whether applying this entry makes no backend call and only changes the +// state: a delete of a resource already gone remotely (Gone) or without a delete operation, or a +// re-key (StateOnly). Such entries are omitted from human output and need no approval. +func (e *PlanEntry) IsStateOnly() bool { return e.Gone || e.StateOnly } @@ -184,6 +193,14 @@ const ( // it and whatever the remote reports is not drift (e.g. cluster libraries without a libraries // section, where job runs install their task libraries cluster-wide). ReasonUnmanaged = "unmanaged" + // ReasonInternalID: the resource's internal ID field (see ResourceLifecycleConfig.InternalIDField), + // which config never sets. + ReasonInternalID = "internal_id" + + // ReasonParentRenamed: an ID field that references the name of a parent being renamed in + // place. The backend moves the resource along with its parent, so it is re-keyed rather + // than recreated. + ReasonParentRenamed = "parent_renamed" // Special reason that results in removing this change from the plan ReasonDrop = "!drop" @@ -231,6 +248,17 @@ func (c *Changes) HasChangeExcept(prefixes ...string) bool { return false } +// HasOnlySingleReason reports whether every actionable change has the given reason. It also holds +// when no change is actionable. +func (c *Changes) HasOnlySingleReason(reason string) bool { + for _, change := range *c { + if change.Action != Skip && change.Reason != reason { + return false + } + } + return true +} + func (p *Plan) GetActions() []Action { actions := make([]Action, 0, len(p.Plan)) for key, entry := range p.Plan { diff --git a/bundle/deployplan/plan_test.go b/bundle/deployplan/plan_test.go index cee00f42c44..1d93583514a 100644 --- a/bundle/deployplan/plan_test.go +++ b/bundle/deployplan/plan_test.go @@ -65,3 +65,51 @@ func TestHasChange(t *testing.T) { }) } } + +func TestHasOnlySingleReason(t *testing.T) { + parentRenamed := &deployplan.ChangeDesc{Action: deployplan.UpdateWithID, Reason: deployplan.ReasonParentRenamed} + tests := []struct { + name string + changes deployplan.Changes + want bool + }{ + { + name: "parent renamed", + changes: deployplan.Changes{"schema_name": parentRenamed}, + want: true, + }, + { + name: "skip change is ignored", + changes: deployplan.Changes{ + "schema_name": parentRenamed, + "storage_location": {Action: deployplan.Skip, Reason: "immutable_backend_value"}, + }, + want: true, + }, + { + name: "own field also changes", + changes: deployplan.Changes{ + "schema_name": parentRenamed, + "comment": {Action: deployplan.Update}, + }, + want: false, + }, + { + name: "own rename", + changes: deployplan.Changes{"name": {Action: deployplan.UpdateWithID, Reason: "id_changes"}}, + want: false, + }, + { + name: "only skipped changes", + changes: deployplan.Changes{ + "name": {Action: deployplan.Skip, Reason: deployplan.ReasonRemoteAlreadySet}, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.changes.HasOnlySingleReason(deployplan.ReasonParentRenamed)) + }) + } +} diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 9cdd5a4f505..543712f9afb 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -44,6 +44,10 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, if oldID == "" { return errors.New("state entry not found") } + if planEntry.ID != "" { + // The plan found the resource at the ID its config gives it: a rename moved it. + oldID = planEntry.ID + } switch actionType { case deployplan.Recreate: @@ -51,6 +55,9 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, case deployplan.Update: return d.Update(ctx, db, oldID, newState, planEntry) case deployplan.UpdateWithID: + if planEntry.StateOnly { + return d.Rekey(ctx, db, oldID, newState, planEntry) + } return d.UpdateWithID(ctx, db, oldID, newState, planEntry) case deployplan.Resize: return d.Resize(ctx, db, oldID, newState, planEntry) @@ -85,7 +92,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, nil) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -183,7 +190,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -228,7 +235,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn, planEntry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -249,6 +256,22 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return nil } +// Rekey records the resource under the ID its new state gives it, without calling the backend: a +// rename already moved it there (see PlanEntry.StateOnly). +func (d *DeploymentUnit) Rekey(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any, planEntry *deployplan.PlanEntry) error { + newID, err := d.Adapter.IDFromState(newState) + if err != nil { + return err + } + log.Infof(ctx, "Re-keyed %s id=%#v (previously %#v)", d.ResourceKey, newID, oldID) + + err = d.saveState(ctx, db, newID, newState, d.DependsOn, planEntry.RemoteState) + if err != nil { + return fmt.Errorf("saving state id=%s: %w", newID, err) + } + return nil +} + func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, oldID string) error { oldState, err := d.loadPersistedState(db) if err != nil { @@ -321,7 +344,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn, entry.RemoteState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -333,7 +356,19 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // with content hashes, see dresources.CompactState) before persisting it. Fields already // smaller than a hash placeholder are persisted as is. Sensitive fields are replaced by a // placeholder value so secrets are never written to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// +// It first records the internal ID in state (see ResourceLifecycleConfig.InternalIDField), from the +// remote state the operation returned or else from planRemote, the object as the plan read it (nil +// for a new object). +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, planRemote any) error { + remote := d.RemoteState + if remote == nil { + remote = planRemote + } + err := d.Adapter.RecordInternalID(state, remote) + if err != nil { + return fmt.Errorf("recording internal id: %w", err) + } compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), state) if err != nil { return fmt.Errorf("compacting state: %w", err) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index add38bf99f6..60e1b66bb47 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -115,7 +115,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - if entry.IsStateOnlyDelete() { + if entry.IsStateOnly() { // The resource is already deleted remotely (Gone) or has no delete // operation (StateOnly); either way only remove it from the state, // without calling the delete API. @@ -130,7 +130,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // A state-only delete performs no backend operation, so don't report it, // consistent with the summary (CountActions excludes it) and the terraform // path in logDeploySummary. - if reportApplied && !entry.IsStateOnlyDelete() { + if reportApplied && !entry.IsStateOnly() { cmdio.LogString(ctx, deployplan.AppliedLine(resourceKey, action)) } return true @@ -173,7 +173,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Reported before the remote-state refresh below: the resource is already // deployed at this point, so the line is accurate even if the refresh fails. - if reportApplied { + // A re-key changes nothing in the backend, so it is not reported (see PlanEntry.StateOnly). + if reportApplied && !entry.StateOnly { cmdio.LogString(ctx, deployplan.AppliedLine(resourceKey, action)) } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 8f8336beaa5..99826f48fa1 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -263,6 +263,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return true } + // Collected before resolveReferences, which drops the references it resolves. + parentRenames, err := b.parentRenames(resourceKey) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + // Process all references in the resource using Refs map // Refs maps path inside resource to references e.g. "${resources.jobs.foo.id} ${resources.jobs.foo.name}" if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, true) { @@ -327,6 +334,13 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } } + if remoteState == nil { + entry.ID, remoteState, err = findMoved(ctx, adapter, sv, dbentry.ID, savedState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } // We have a choice whether to include remoteState or remoteStateComparable from below. // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable @@ -364,7 +378,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - err = addPerFieldActions(ctx, adapter, entry.Changes, sv.Value, remoteState) + err = addPerFieldActions(ctx, adapter, entry.Changes, sv.Value, remoteState, parentRenames) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false @@ -377,6 +391,15 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } else { action = getMaxAction(entry.Changes) } + if entry.ID != "" { + // Found at a new ID: the state has to follow it there. + action = deployplan.GetHigherAction(action, deployplan.UpdateWithID) + } + if action == deployplan.UpdateWithID && adapter.HasIDFromState() && entry.Changes.HasOnlySingleReason(deployplan.ReasonParentRenamed) { + // The backend already moved the resource with its renamed parent, or the plan found it + // at its new ID: applying only records the state under the ID IDFromState gives it. + entry.StateOnly = true + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -465,7 +488,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, return m, nil } -func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, newState, remoteState any) error { +func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, newState, remoteState any, parentRenames map[string]*deployplan.ChangeDesc) error { cfg := adapter.ResourceConfig() generatedCfg := adapter.GeneratedResourceConfig() @@ -477,6 +500,13 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change return err } + // Config never sets the internal ID; the framework records it from the remote state. + if adapter.IsInternalIDField(path) { + ch.Action = deployplan.Skip + ch.Reason = deployplan.ReasonInternalID + continue + } + // RemoteAlreadySet only holds when ch.Remote is a real remote value we can compare // against ch.New. Skip it for fields whose remote value is fabricated and thus // meaningless: declared ignore_remote_changes (present in RemoteType but read back @@ -498,6 +528,10 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if action, reason, ok := classifyIDField(cfg, path, ch); ok { ch.Action = action ch.Reason = reason + if action == deployplan.Recreate && followsParentRename(adapter, parentRenames[pathString], ch) { + ch.Action = deployplan.UpdateWithID + ch.Reason = deployplan.ReasonParentRenamed + } } else if action, reason, ok := classifyIDField(generatedCfg, path, ch); ok { ch.Action = action ch.Reason = reason @@ -666,6 +700,122 @@ func classifyIDField(cfg *dresources.ResourceLifecycleConfig, path *structpath.P return deployplan.Undefined, "", false } +// parentRenames returns the fields of resourceKey whose whole value is a reference to a +// field that the referenced resource renames in this plan (an update_id change), keyed by +// field path, with the referenced change as the value. +func (b *DeploymentBundle) parentRenames(resourceKey string) (map[string]*deployplan.ChangeDesc, error) { + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + return nil, fmt.Errorf("internal error: no state cache entry found for %q", resourceKey) + } + + result := make(map[string]*deployplan.ChangeDesc) + for fieldPath, refString := range sv.Refs { + ref, ok := dynvar.NewRef(dyn.V(refString)) + if !ok || !ref.IsPure() { + continue + } + targetPath, err := structpath.ParsePath(ref.References()[0]) + if err != nil { + return nil, fmt.Errorf("cannot parse reference %q: %w", refString, err) + } + targetKey, targetField := splitResourcePath(targetPath) + change, err := b.renameChange(targetKey, targetField.String()) + if err != nil { + return nil, err + } + if change != nil { + result[fieldPath] = change + } + } + return result, nil +} + +// renameChange returns the change to field of resourceKey if it is part of an update_id, nil otherwise. +func (b *DeploymentBundle) renameChange(resourceKey, field string) (*deployplan.ChangeDesc, error) { + entry, err := b.Plan.ReadLockEntry(resourceKey) + if err != nil { + return nil, err + } + defer b.Plan.ReadUnlockEntry(resourceKey) + + // A missing entry is reported by resolveReferences. + if entry == nil || entry.Action != deployplan.UpdateWithID { + return nil, nil + } + change := entry.Changes[field] + if change == nil || change.Action != deployplan.UpdateWithID { + return nil, nil + } + return change, nil +} + +// findMoved looks for a resource missing from its recorded ID at the ID its config gives it, where a +// rename can have moved it: the resource or its parent renamed out of band, or by a deploy that +// stopped before re-keying it. What is found there is taken only if its internal ID matches the one +// in savedState, so an unrelated object with the same name is never adopted; the create then fails +// on it instead. +func findMoved(ctx context.Context, adapter *dresources.Adapter, sv *structvar.StructVar, storedID string, savedState any) (string, any, error) { + if !adapter.HasInternalID() || hasUnresolvedIDField(adapter, sv) { + return "", nil, nil + } + savedInternalID, err := adapter.InternalID(savedState) + if err != nil || savedInternalID == "" { + return "", nil, err + } + id, err := adapter.IDFromState(sv.Value) + if err != nil || id == storedID { + return "", nil, err + } + remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, id) + }) + if apierr.IsMissing(err) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("reading id=%q: %w", id, err) + } + found, err := adapter.RemapState(remote) + if err != nil { + return "", nil, err + } + foundInternalID, err := adapter.InternalID(found) + if err != nil || foundInternalID != savedInternalID { + return "", nil, err + } + return id, remote, nil +} + +// hasUnresolvedIDField reports whether a field the ID is built from is still an unresolved reference. +func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) bool { + cfg := adapter.ResourceConfig() + for pathString := range sv.Refs { + path, err := structpath.ParsePath(pathString) + if err != nil { + return true + } + if _, ok := findMatchingRule(path, cfg.ProvidedIDFields); ok { + return true + } + if _, ok := findMatchingRule(path, cfg.UpdatableIDFields); ok { + return true + } + } + return false +} + +// followsParentRename reports whether a change to a provided ID field only follows parentChange, +// the rename of the parent field it references (see parentRenames). UC moves the children of a +// renamed catalog or schema along with it, so recreating such a child fails with "already +// exists"; it is planned as an update_id instead, which addresses it under the new parent. +// +// The old values must match too: a child moved to the renamed parent from somewhere else +// still has to be recreated. +func followsParentRename(adapter *dresources.Adapter, parentChange, ch *deployplan.ChangeDesc) bool { + return parentChange != nil && adapter.HasDoUpdateWithID() && structdiff.IsEqual(ch.Old, parentChange.Old) +} + // shouldSkipNormalized skips a change that is a false diff caused by UC API // normalization: the API strips trailing slashes from storage URLs // (normalize_slash). The direct engine saves local config to state, so without diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 2d051dc21b9..42101800f21 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -55,8 +55,8 @@ func TestDynPathToStructPath(t *testing.T) { func TestExtractReferences_ExcludesReadonlyFields(t *testing.T) { adapters, err := dresources.InitAll(nil) require.NoError(t, err) - // The volume state type is catalog.CreateVolumeRequestContent, which has - // comment but not volume_path. + // The volume state type is VolumeState, which has comment but not + // volume_path. stateType := adapters["volumes"].StateType() const yml = ` @@ -259,7 +259,7 @@ func TestRemoteAlreadySetGuards(t *testing.T) { adapter, ok := adapters[tt.resource] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - err := addPerFieldActions(t.Context(), adapter, changes, nil, nil) + err := addPerFieldActions(t.Context(), adapter, changes, nil, nil, nil) require.NoError(t, err) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { @@ -271,6 +271,64 @@ func TestRemoteAlreadySetGuards(t *testing.T) { } } +// TestFollowsParentRename pins when an ID field that references a renamed parent is +// re-keyed (update_id) instead of recreated: only a provided_id_field of a resource with +// DoUpdateWithID, and only if it was under that parent before the rename. +func TestFollowsParentRename(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + tests := []struct { + name string + resource string + field string + old string + expectedAction deployplan.ActionType + }{ + { + name: "child of the renamed parent follows it", + resource: "volumes", + field: "schema_name", + old: "s_old", + expectedAction: deployplan.UpdateWithID, + }, + { + name: "child moved from another parent is recreated", + resource: "volumes", + field: "schema_name", + old: "other", + expectedAction: deployplan.Recreate, + }, + { + name: "resource without DoUpdateWithID is recreated", + resource: "registered_models", + field: "schema_name", + old: "s_old", + expectedAction: deployplan.Recreate, + }, + { + name: "recreate_on_changes field is recreated", + resource: "volumes", + field: "storage_location", + old: "s_old", + expectedAction: deployplan.Recreate, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + adapter, ok := adapters[tt.resource] + require.True(t, ok) + ch := &deployplan.ChangeDesc{Old: tt.old, New: "s_new", Remote: tt.old} + parentRenames := map[string]*deployplan.ChangeDesc{ + tt.field: {Action: deployplan.UpdateWithID, Reason: "id_changes", Old: "s_old", New: "s_new", Remote: "s_old"}, + } + require.NoError(t, addPerFieldActions(t.Context(), adapter, deployplan.Changes{tt.field: ch}, nil, nil, parentRenames)) + assert.Equal(t, tt.expectedAction, ch.Action) + }) + } +} + // TestRecreateBackendDefault pins the implicit backend-default behavior for // recreate_on_changes fields (recreateOrBackendDefault): an immutable field the // config never set (old and new nil) but the backend populated (remote set) is left @@ -322,7 +380,7 @@ func TestRecreateBackendDefault(t *testing.T) { adapter, ok := adapters[tt.resource] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil, nil)) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { assert.Equal(t, tt.expectedReason, tt.ch.Reason) @@ -453,7 +511,7 @@ func TestScalarZeroChange(t *testing.T) { adapter, ok := adapters["clusters"] require.True(t, ok) changes := deployplan.Changes{tt.field: tt.ch} - require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, nil, nil, nil)) assert.Equal(t, tt.expectedAction, tt.ch.Action) if tt.expectedReason != "" { assert.Equal(t, tt.expectedReason, tt.ch.Reason) @@ -517,7 +575,7 @@ func jobRunResultStateAction(t *testing.T, state *jobs.RunState) *deployplan.Cha Remote: state.ResultState, }} - require.NoError(t, addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil, remote)) + require.NoError(t, addPerFieldActions(t.Context(), adapters["job_runs"], changes, nil, remote, nil)) return changes["result_state"] } @@ -791,7 +849,30 @@ func TestLadderBackendDefaultBeforeRemoteAddition(t *testing.T) { // config-absent (Old==New==nil), present in remote: matches both rules. "enable_elastic_disk": &deployplan.ChangeDesc{Remote: true}, } - require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, newState, nil)) + require.NoError(t, addPerFieldActions(t.Context(), adapter, changes, newState, nil, nil)) assert.Equal(t, deployplan.Skip, changes["enable_elastic_disk"].Action) assert.Equal(t, deployplan.ReasonBackendDefault, changes["enable_elastic_disk"].Reason) } + +// TestInternalIDChangesAreSkipped: volume_id and schema_id are recorded in state but never set by +// config, so neither the always-different local value nor a state recorded before them plans an update. +func TestInternalIDChangesAreSkipped(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + for _, tt := range []struct{ resource, field string }{ + {"volumes", "volume_id"}, + {"schemas", "schema_id"}, + } { + for name, ch := range map[string]*deployplan.ChangeDesc{ + "recorded": {Old: "uuid-1", New: nil, Remote: "uuid-1"}, + "not recorded": {Old: nil, New: nil, Remote: "uuid-1"}, + } { + t.Run(tt.resource+" "+name, func(t *testing.T) { + changes := deployplan.Changes{tt.field: ch} + require.NoError(t, addPerFieldActions(t.Context(), adapters[tt.resource], changes, nil, nil, nil)) + assert.Equal(t, deployplan.Skip, changes[tt.field].Action) + }) + } + } +} diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 78952537943..373af02b13f 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -47,6 +47,12 @@ For resources whose create or update is asynchronous (the resource is not immedi If the API may return a slice's elements in a different order between calls (e.g., `depends_on` in job tasks, `privileges` in grants), implement `KeyedSlices` to compare elements by a natural key rather than by index. Without this, every deploy after any reordering shows phantom diffs. +## Renamed parents and moved resources: IDFromState / internal_id_field + +If the ID is built from the resource's own fields (a UC full name), implement `IDFromState`. Then an `update_id` that only follows a parent's rename is applied by the framework without calling `DoUpdateWithID` (the plan marks it `state_only`), so `DoUpdateWithID` only handles real updates. + +If the backend also has its own ID that a rename keeps (a volume's `volume_id`), add it to the state type, copy it in `RemapState`, and name it in the config as `internal_id_field`. The framework records it on every save and skips its changes in the plan. When nothing is at the recorded ID, the plan reads at the ID the config gives the resource and takes what it finds only if the internal ID matches. This covers a rename done out of band, or by a deploy that stopped before re-keying the resource. + ## Empty states: IsEmptyState If a desired state describes no resource at all (e.g. an empty grants list), implement `IsEmptyState`. The planner omits such a node instead of planning a create, and apply drops its state entry instead of persisting one. diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 80ece24b2be..64c7fc22d8d 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/calladapt" + "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" @@ -118,6 +119,12 @@ type IResource interface { // (returned by GET, not 404) that rejects a second DELETE. // Example: func (*ResourceApp) IsGone(remote *AppRemote) bool IsGone(remoteState any) bool + + // [Optional] IDFromState returns the ID the state gives the resource, for resource types whose ID + // is built from their own fields (a UC full name). With it, an update_id that only follows a + // parent's rename is applied without calling DoUpdateWithID (see PlanEntry.StateOnly). + // Example: func (*ResourceVolume) IDFromState(state *VolumeState) string + IDFromState(state any) string } // Adapter wraps resource implementation, validates signatures and type consistency across methods @@ -141,10 +148,14 @@ type Adapter struct { overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller isGone *calladapt.BoundCaller + idFromState *calladapt.BoundCaller resourceConfig *ResourceLifecycleConfig generatedResourceConfig *ResourceLifecycleConfig keyedSlices map[string]any + + // internalIDPath is InternalIDField, parsed; nil for a resource type without one. + internalIDPath *structpath.PathNode } func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceClient) (*Adapter, error) { @@ -182,6 +193,8 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC waitAfterDelete: nil, overrideChangeDesc: nil, isGone: nil, + idFromState: nil, + internalIDPath: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), keyedSlices: nil, @@ -308,6 +321,11 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.idFromState, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "IDFromState") + if err != nil { + return err + } + keyedSlicesCall, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "KeyedSlices") if err != nil { return err @@ -402,6 +420,24 @@ func (a *Adapter) validate() error { validations = append(validations, "IsGone remoteState", a.isGone.InTypes[0], remoteType) } + if a.idFromState != nil { + validations = append(validations, "IDFromState state", a.idFromState.InTypes[0], stateType) + } + + if a.resourceConfig != nil && a.resourceConfig.InternalIDField != "" { + if a.idFromState == nil || a.doUpdateWithID == nil { + return errors.New("internal_id_field requires IDFromState and DoUpdateWithID") + } + path, err := structpath.ParsePath(a.resourceConfig.InternalIDField) + if err != nil { + return fmt.Errorf("internal_id_field: %w", err) + } + if err := structaccess.ValidatePath(stateType, path); err != nil { + return fmt.Errorf("internal_id_field: %w", err) + } + a.internalIDPath = path + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) @@ -727,6 +763,58 @@ func (a *Adapter) IsGone(remoteState any) bool { return outs[0].(bool) } +// HasIDFromState reports whether the resource implements IDFromState. +func (a *Adapter) HasIDFromState() bool { + return a.idFromState != nil +} + +// HasInternalID reports whether the resource records an internal ID (see InternalIDField). +func (a *Adapter) HasInternalID() bool { + return a.internalIDPath != nil +} + +// IsInternalIDField reports whether path is the resource's internal ID field. It matches by +// prefix, like the field rules do. +func (a *Adapter) IsInternalIDField(path *structpath.PathNode) bool { + return a.internalIDPath != nil && path.HasPrefix(a.internalIDPath) +} + +// IDFromState returns the ID the state gives the resource; see IResource.IDFromState. +func (a *Adapter) IDFromState(state any) (string, error) { + outs, err := a.idFromState.Call(state) + if err != nil { + return "", err + } + return outs[0].(string), nil +} + +// InternalID returns the internal ID recorded in state (see InternalIDField). +func (a *Adapter) InternalID(state any) (string, error) { + value, err := structaccess.Get(state, a.internalIDPath) + if err != nil || value == nil { + return "", err + } + id, _ := value.(string) + return id, nil +} + +// RecordInternalID copies the internal ID from remoteState into state, which is about to be saved. +// Config never sets it, so without this the saved state would lose it. +func (a *Adapter) RecordInternalID(state, remoteState any) error { + if a.internalIDPath == nil || remoteState == nil || reflect.ValueOf(remoteState).IsNil() { + return nil + } + remapped, err := a.RemapState(remoteState) + if err != nil { + return err + } + id, err := a.InternalID(remapped) + if err != nil || id == "" { + return err + } + return structaccess.Set(state, a.internalIDPath, id) +} + // prepareCallRequired prepares a call and ensures the method is found. func prepareCallRequired(resource any, methodName string) (*calladapt.BoundCaller, error) { caller, err := calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), methodName) diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index d3a230b347d..b8652d74120 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -68,7 +68,7 @@ quality_monitors: catalog.CreateMonitor registered_models: catalog.RegisteredModelInfo -schemas: catalog.CreateSchema +schemas: catalog.SchemaInfo secret_scopes: workspace.CreateScope @@ -82,4 +82,4 @@ vector_search_endpoints: vectorsearch.CreateEndpoint vector_search_indexes: vectorsearch.CreateVectorIndexRequest -volumes: catalog.CreateVolumeRequestContent +volumes: catalog.VolumeInfo diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 9b3ae9080b8..7adcc36086a 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -68,6 +68,13 @@ type ResourceLifecycleConfig struct { // IgnoreLocalChanges: field patterns where local changes are ignored (can't be updated via API). IgnoreLocalChanges []FieldRule `yaml:"ignore_local_changes,omitempty"` + // InternalIDField: the state field holding the backend's own ID for the resource, which a rename + // keeps (a volume's volume_id). Config never sets it: the framework records it from the remote + // state on every save and skips its changes in the plan. When the resource is missing from its + // recorded ID, the plan looks for it at the ID IDFromState gives it and takes what it finds only + // if this field matches. Requires IDFromState and DoUpdateWithID. + InternalIDField string `yaml:"internal_id_field,omitempty"` + // RecreateOnChanges: field patterns that trigger delete + create when changed. RecreateOnChanges []FieldRule `yaml:"recreate_on_changes,omitempty"` @@ -155,6 +162,7 @@ const ( var empty = ResourceLifecycleConfig{ IgnoreRemoteChanges: nil, IgnoreLocalChanges: nil, + InternalIDField: "", RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 8ef27e71d15..5bb8f161038 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -1,9 +1,10 @@ provided_id_fields: - # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - - field: name - reason: id_field - field: catalog_name reason: id_field +updatable_id_fields: + # UC lowercases identifier names; remote returns "myschema" for config "MySchema". + - field: name + reason: id_changes recreate_on_changes: - field: storage_root reason: immutable @@ -15,3 +16,6 @@ backend_defaults: # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - field: properties['unity.catalog.managed.*.defaults.*'] +# Assigned by the backend and kept across renames: lets the plan find a schema renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: schema_id diff --git a/bundle/direct/dresources/configs/volumes.generated.yml b/bundle/direct/dresources/configs/volumes.generated.yml new file mode 100644 index 00000000000..3faf600a2dd --- /dev/null +++ b/bundle/direct/dresources/configs/volumes.generated.yml @@ -0,0 +1,5 @@ +# Generated, do not edit. + +ignore_remote_changes: + - field: volume_id + reason: spec:output_only diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index d4b68e95875..9e570a1f7c2 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -16,3 +16,6 @@ normalize_slash: # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 - field: storage_location reason: uc_strips_trailing_slash +# Assigned by the backend and kept across renames: lets the plan find a volume renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: volume_id diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index d80dac584bc..76a619ee8ad 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -8,9 +8,26 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// SchemaState is the schema's config plus its schema_id. The backend assigns schema_id and a +// rename keeps it, so recording it lets the plan recognize the schema under a new name (see +// internal_id_field in schemas.yml). +type SchemaState struct { + catalog.CreateSchema + SchemaId string `json:"schema_id,omitempty"` +} + +func (s *SchemaState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s SchemaState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceSchema struct { client *databricks.WorkspaceClient } @@ -19,12 +36,15 @@ func (*ResourceSchema) New(client *databricks.WorkspaceClient) *ResourceSchema { return &ResourceSchema{client: client} } -func (*ResourceSchema) PrepareState(input *resources.Schema) *catalog.CreateSchema { - return &input.CreateSchema +func (*ResourceSchema) PrepareState(input *resources.Schema) *SchemaState { + return &SchemaState{ + CreateSchema: input.CreateSchema, + SchemaId: "", // Recorded by the framework from the remote state (internal_id_field). + } } -func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchema { - return &catalog.CreateSchema{ +func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *SchemaState { + return &SchemaState{ CatalogName: info.CatalogName, Comment: info.Comment, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -32,6 +52,7 @@ func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchem Properties: info.Properties, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateSchema](info.ForceSendFields), + SchemaId: info.SchemaId, } } @@ -39,8 +60,8 @@ func (r *ResourceSchema) DoRead(ctx context.Context, id string) (*catalog.Schema return r.client.Schemas.GetByFullName(ctx, id) } -func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSchema) (string, *catalog.SchemaInfo, error) { - response, err := r.client.Schemas.Create(ctx, *config) +func (r *ResourceSchema) DoCreate(ctx context.Context, config *SchemaState) (string, *catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Create(ctx, config.CreateSchema) if err != nil || response == nil { return "", nil, err } @@ -64,32 +85,63 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSch // migrate invariant (schema.yml is excluded there, and schema_tf_compat.yml covers migrate instead). var schemaForceSend = []string{"Comment", "CustomMaxRetentionHours"} -// DoUpdate updates the schema in place and returns remote state. -func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *PlanEntry) (*catalog.SchemaInfo, error) { - updateRequest := catalog.UpdateSchema{ +func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog.UpdateSchema { + return catalog.UpdateSchema{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, EnablePredictiveOptimization: "", // Not supported by DABs FullName: id, - NewName: "", // We recreate schemas on name change intentionally. + NewName: newName, Owner: "", // Not supported by DABs Properties: config.Properties, ForceSendFields: utils.FilterFields[catalog.UpdateSchema](append(slices.Clone(schemaForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "NewName", "Owner"), } +} - response, err := r.client.Schemas.Update(ctx, updateRequest) - if err != nil { +// DoUpdate updates the schema in place and returns remote state. +func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *SchemaState, _ *PlanEntry) (*catalog.SchemaInfo, error) { + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(id, "", &config.CreateSchema)) + if err != nil || response == nil { return nil, err } - if response != nil && response.FullName != id { + if response.FullName != id { log.Warnf(ctx, "schemas: response contains unexpected full_name=%#v (expected %#v)", response.FullName, id) } return response, nil } -func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *catalog.CreateSchema) error { +// DoUpdateWithID renames the schema in place when the name changes; the ID is the full name, so it changes too. +func (r *ResourceSchema) DoUpdateWithID(ctx context.Context, id string, config *SchemaState, _ *PlanEntry) (string, *catalog.SchemaInfo, error) { + nameFromID, err := getNameFromID(id) + if err != nil { + return "", nil, err + } + + newName := "" + if config.Name != nameFromID { + newName = config.Name + } + + // The catalog comes from config: a renamed catalog moves the schema with it, leaving id + // stale (see followsParentRename). + fullName := config.CatalogName + "." + nameFromID + + response, err := r.client.Schemas.Update(ctx, makeUpdateSchema(fullName, newName, &config.CreateSchema)) + if err != nil || response == nil { + return "", nil, err + } + + return response.FullName, response, nil +} + +// IDFromState returns the full name, which is the ID. +func (*ResourceSchema) IDFromState(state *SchemaState) string { + return state.CatalogName + "." + state.Name +} + +func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *SchemaState) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ FullName: id, Force: true, diff --git a/bundle/direct/dresources/schema_test.go b/bundle/direct/dresources/schema_test.go index ee6397afc6e..5c486cab1f1 100644 --- a/bundle/direct/dresources/schema_test.go +++ b/bundle/direct/dresources/schema_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,13 +14,14 @@ func TestResourceSchema_DoUpdate_WithUnsupportedForceSendFields(t *testing.T) { adapter := (*ResourceSchema)(nil).New(client) ctx := t.Context() - config := &catalog.CreateSchema{ + config := &SchemaState{ CatalogName: "main", Name: "test_schema", Comment: "original comment", Properties: map[string]string{"key": "value"}, StorageRoot: "", ForceSendFields: nil, + SchemaId: "", } id, _, err := adapter.DoCreate(ctx, config) diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 7cd1eaba1ce..63a978c01c2 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -10,9 +10,26 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// VolumeState is the volume's config plus its volume_id. The backend assigns volume_id and a +// rename keeps it, so recording it lets the plan recognize the volume under a new name (see +// internal_id_field in volumes.yml). +type VolumeState struct { + catalog.CreateVolumeRequestContent + VolumeId string `json:"volume_id,omitempty"` +} + +func (s *VolumeState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s VolumeState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceVolume struct { client *databricks.WorkspaceClient } @@ -21,12 +38,15 @@ func (*ResourceVolume) New(client *databricks.WorkspaceClient) *ResourceVolume { return &ResourceVolume{client: client} } -func (*ResourceVolume) PrepareState(input *resources.Volume) *catalog.CreateVolumeRequestContent { - return &input.CreateVolumeRequestContent +func (*ResourceVolume) PrepareState(input *resources.Volume) *VolumeState { + return &VolumeState{ + CreateVolumeRequestContent: input.CreateVolumeRequestContent, + VolumeId: "", // Recorded by the framework from the remote state (internal_id_field). + } } -func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolumeRequestContent { - return &catalog.CreateVolumeRequestContent{ +func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *VolumeState { + return &VolumeState{ CatalogName: info.CatalogName, Comment: info.Comment, Name: info.Name, @@ -34,6 +54,7 @@ func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolum StorageLocation: info.StorageLocation, VolumeType: info.VolumeType, ForceSendFields: utils.FilterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields), + VolumeId: info.VolumeId, } } @@ -41,8 +62,8 @@ func (r *ResourceVolume) DoRead(ctx context.Context, id string) (*catalog.Volume return r.client.Volumes.ReadByName(ctx, id) } -func (r *ResourceVolume) DoCreate(ctx context.Context, config *catalog.CreateVolumeRequestContent) (string, *catalog.VolumeInfo, error) { - response, err := r.client.Volumes.Create(ctx, *config) +func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (string, *catalog.VolumeInfo, error) { + response, err := r.client.Volumes.Create(ctx, config.CreateVolumeRequestContent) if err != nil { return "", nil, err } @@ -52,7 +73,7 @@ func (r *ResourceVolume) DoCreate(ctx context.Context, config *catalog.CreateVol // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. var volumeForceSend = []string{"Comment"} -func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (*catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (*catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: id, @@ -83,10 +104,10 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalo return response, err } -func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, - Name: id, + Name: "", // Initialized below NewName: "", // Initialized below if needed Owner: "", // Not supported by DABs @@ -100,6 +121,10 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * } nameFromID := items[len(items)-1] + // The parents come from config: a renamed catalog or schema moves the volume with it, + // leaving id stale (see followsParentRename). + updateRequest.Name = config.CatalogName + "." + config.SchemaName + "." + nameFromID + if config.Name != nameFromID { updateRequest.NewName = config.Name } @@ -112,7 +137,12 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * return response.FullName, response, nil } -func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *catalog.CreateVolumeRequestContent) error { +// IDFromState returns the full name, which is the ID. +func (*ResourceVolume) IDFromState(state *VolumeState) string { + return state.CatalogName + "." + state.SchemaName + "." + state.Name +} + +func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *VolumeState) error { return r.client.Volumes.DeleteByName(ctx, id) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index c3c3306130e..92ceae95f9a 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -49,7 +49,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P // Deletes that only clean up the state (already gone remotely, or no delete // operation) are not destructive and need no approval. - actions = slices.DeleteFunc(actions, func(a deployplan.Action) bool { return a.IsStateOnlyDelete() }) + actions = slices.DeleteFunc(actions, func(a deployplan.Action) bool { return a.IsStateOnly() }) err := checkForPreventDestroy(b, actions) if err != nil { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index b3e885e54d4..ee9d6cdb305 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -92,7 +92,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. // operation) are not destructive, so they are not listed as deletions and need no // approval. In particular this makes prevent_destroy inert for state-only // resources: nothing is destroyed. - deleteActions = slices.DeleteFunc(deleteActions, func(a deployplan.Action) bool { return a.IsStateOnlyDelete() }) + deleteActions = slices.DeleteFunc(deleteActions, func(a deployplan.Action) bool { return a.IsStateOnly() }) err := checkForPreventDestroy(b, deleteActions) if err != nil { @@ -192,7 +192,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) { // a destruction to report. deleted := 0 for _, a := range plan.GetActions() { - if a.ActionType == deployplan.Delete && !a.IsChildResource() && !a.IsStateOnlyDelete() { + if a.ActionType == deployplan.Delete && !a.IsChildResource() && !a.IsStateOnly() { deleted++ } } diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 1132834bc4d..48d3be6a783 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -91,7 +91,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, } // A state-only delete has no backend effect; keep it in the JSON // plan but omit it from the human-readable action list. - if action.IsStateOnlyDelete() { + if action.IsStateOnly() { continue } key := strings.TrimPrefix(action.ResourceKey, "resources.")