direct: revoke grants and permissions when the block is deleted or emptied

.nextchanges/notable-changes/grants-and-permissions-revoked-on-delete.md

@@ -0,0 +1,12 @@
+Deleting a `grants` or `permissions` block, or emptying it to `[]`, now revokes what it
+granted. Both were previously ignored: the grant or permission stayed in place, and so did
+anything added outside the bundle. An empty list and a removed block are the same request,
+and both are now applied as a delete ([#6474](https://github.com/databricks/cli/pull/6474)).
+
+`grants: []` revokes everything. `permissions: []` revokes everything but the object owner,
+which the API requires on every update. The owner is read from the object rather than assumed
+to be whoever deploys, so an object owned by a service principal, or handed over since it was
+deployed, keeps the owner it has.
+
+Nothing changes for a bundle that has always had an empty list, and deleting the resource a
+block belongs to is unaffected.

acceptance/bundle/resources/grants/schemas/delete_semantics/databricks.empty.yml.tmpl

@@ -0,0 +1,2 @@
+bundle:
+  name: schema-grants-delete-semantics-$UNIQUE_NAME

acceptance/bundle/resources/grants/schemas/delete_semantics/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: schema-grants-delete-semantics-$UNIQUE_NAME
+
+resources:
+  schemas:
+    grants_schema:
+      name: schema_delete_semantics_$UNIQUE_NAME
+      catalog_name: main
+      grants: [{ principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA] }] # GRANTS

acceptance/bundle/resources/grants/schemas/delete_semantics/out.direct.txt

@@ -0,0 +1,50 @@
+=== only the grants node is deleted
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "ALL_PRIVILEGES"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the grants list is empty
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "ALL_PRIVILEGES"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the parent is deleted too
+delete schemas.grants_schema
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged
+{
+  "method": "DELETE",
+  "path": "/api/2.1/unity-catalog/schemas/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "q": {
+    "force": "true"
+  }
+}

acceptance/bundle/resources/grants/schemas/delete_semantics/out.terraform.txt

@@ -0,0 +1,64 @@
+=== only the grants node is deleted
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "USE_SCHEMA"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the grants list is empty
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "USE_SCHEMA"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the parent is deleted too
+delete schemas.grants_schema
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged
+{
+  "method": "DELETE",
+  "path": "/api/2.1/unity-catalog/schemas/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "q": {
+    "force": "true"
+  }
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "USE_SCHEMA"
+        ]
+      }
+    ]
+  }
+}

acceptance/bundle/resources/grants/schemas/delete_semantics/output.txt

@@ -0,0 +1,27 @@
+
+=== Deploy the schema with one grant
+>>> [CLI] bundle deploy -qq
+
+=== Case 1: the grants block is removed, the schema stays
+>>> [CLI] bundle deploy -qq
+
+=== Put the grant back
+>>> [CLI] bundle deploy -qq
+
+=== Case 2: the grants list is set to []
+>>> [CLI] bundle deploy -qq
+
+=== Put the grant back
+>>> [CLI] bundle deploy -qq
+
+=== Case 3: the schema and its grants are deleted together
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.schemas.grants_schema
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.grants_schema
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-delete-semantics-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/grants/schemas/delete_semantics/script

@@ -0,0 +1,54 @@
+SCHEMA_FULL_NAME=main.schema_delete_semantics_$UNIQUE_NAME
+GRANTS_LINE='grants: [{ principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA] }] # GRANTS'
+
+# The plan, the requests and the leftover grants all go to one per-engine file: the engines
+# disagree on how a dropped grants node is classified and on what they send for it.
+ENGINE_OUT=out.$DATABRICKS_BUNDLE_ENGINE.txt
+
+envsubst < databricks.yml.tmpl > databricks.yml
+cp databricks.yml databricks.yml.saved
+
+cleanup() {
+    errcode $CLI bundle destroy --auto-approve &> LOG.cleanup
+    rm -f out.requests.txt databricks.yml.saved tmp.yml
+}
+trap cleanup EXIT
+
+record() {
+    echo "=== $1" >> $ENGINE_OUT
+    $CLI bundle plan >> $ENGINE_OUT
+    trace $CLI bundle deploy -qq
+    print_requests.py //unity-catalog --sort >> $ENGINE_OUT
+    $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort_lines.py --repl >> $ENGINE_OUT
+}
+
+restore_grant() {
+    title "Put the grant back"
+    cp databricks.yml.saved databricks.yml
+    trace $CLI bundle deploy -qq
+    rm -f out.requests.txt
+}
+
+title "Deploy the schema with one grant"
+trace $CLI bundle deploy -qq
+rm -f out.requests.txt
+
+title "Case 1: the grants block is removed, the schema stays"
+grep -v GRANTS databricks.yml > tmp.yml && mv tmp.yml databricks.yml
+record "only the grants node is deleted"
+
+restore_grant
+title "Case 2: the grants list is set to []"
+update_file.py databricks.yml "$GRANTS_LINE" 'grants: []'
+record "the grants list is empty"
+
+restore_grant
+title "Case 3: the schema and its grants are deleted together"
+echo "=== the parent is deleted too" >> $ENGINE_OUT
+# Planned against a config with the schema gone, so the plan names both nodes and the
+# action each one gets. `bundle destroy` applies the same shape but its banner hides
+# child nodes, so it never shows what happens to the grants.
+envsubst < databricks.empty.yml.tmpl > databricks.yml
+$CLI bundle plan >> $ENGINE_OUT
+trace $CLI bundle destroy --auto-approve
+print_requests.py //unity-catalog --sort >> $ENGINE_OUT

acceptance/bundle/resources/grants/schemas/delete_semantics/test.toml

@@ -0,0 +1,2 @@
+# Six deploys plus a destroy; extra headroom for heavy parallel runs.
+Timeout = '2m'

acceptance/bundle/resources/permissions/jobs/revoke_semantics/databricks.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: job-permissions-revoke-$UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: job_revoke_semantics_$UNIQUE_NAME
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: ./notebook.py
+      permissions: # PERMISSIONS
+        - level: CAN_VIEW # PERMISSIONS
+          user_name: deco-test-user@databricks.com # PERMISSIONS

acceptance/bundle/resources/permissions/jobs/revoke_semantics/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("hello")

acceptance/bundle/resources/permissions/jobs/revoke_semantics/output.txt

@@ -0,0 +1,57 @@
+
+=== Deploy with one grant to a non-owner
+>>> [CLI] bundle deploy -qq
+
+>>> acl
+[
+  {
+    "levels": [
+      "CAN_VIEW"
+    ],
+    "name": "deco-test-user@databricks.com"
+  },
+  {
+    "levels": [
+      "IS_OWNER"
+    ],
+    "name": "[USERNAME]"
+  }
+]
+
+=== Empty the list: everything but the owner is revoked
+>>> [CLI] bundle plan
+delete jobs.foo.permissions
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy -qq
+
+>>> acl
+[
+  {
+    "levels": [
+      "IS_OWNER"
+    ],
+    "name": "[USERNAME]"
+  }
+]
+
+=== Put the grant back, then remove the block entirely: same outcome
+>>> [CLI] bundle deploy -qq
+
+>>> [CLI] bundle plan
+delete jobs.foo.permissions
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy -qq
+
+>>> acl
+[
+  {
+    "levels": [
+      "IS_OWNER"
+    ],
+    "name": "[USERNAME]"
+  }
+]

acceptance/bundle/resources/permissions/jobs/revoke_semantics/script

@@ -0,0 +1,39 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+cp databricks.yml databricks.yml.saved
+
+cleanup() {
+    errcode $CLI bundle destroy --auto-approve &> LOG.cleanup
+    rm -f out.requests.txt databricks.yml.saved tmp.yml
+}
+trap cleanup EXIT
+
+# The API rejects a Set without exactly one IS_OWNER, so an emptied list revokes down to the
+# owner rather than to nothing. The testserver does not enforce that, which is why this runs
+# on cloud: the read below is the assertion.
+acl() {
+    $CLI api get "/api/2.0/permissions/jobs/$JOB_ID" |
+        jq -S '[.access_control_list[]
+                | select(any(.all_permissions[]; .inherited == false))
+                | {name: (.user_name // .service_principal_name // .group_name),
+                   levels: [.all_permissions[] | select(.inherited == false) | .permission_level]}]'
+}
+
+title "Deploy with one grant to a non-owner"
+trace $CLI bundle deploy -qq
+JOB_ID=$(read_id.py foo)
+trace acl
+
+title "Empty the list: everything but the owner is revoked"
+update_file.py databricks.yml 'permissions: # PERMISSIONS' 'permissions: []'
+sed -i '/# PERMISSIONS/d' databricks.yml
+trace $CLI bundle plan
+trace $CLI bundle deploy -qq
+trace acl
+
+title "Put the grant back, then remove the block entirely: same outcome"
+cp databricks.yml.saved databricks.yml
+trace $CLI bundle deploy -qq
+sed -i '/# PERMISSIONS/d' databricks.yml
+trace $CLI bundle plan
+trace $CLI bundle deploy -qq
+trace acl

acceptance/bundle/resources/permissions/jobs/revoke_semantics/test.toml

@@ -0,0 +1,4 @@
+# The API requires exactly one IS_OWNER on every Set, so the revoke floor is owner-only.
+# The testserver does not enforce that, so this only means anything against a real workspace.
+Cloud = true
+EnvMatrix.DMS = [""]

bundle/deployplan/action.go

@@ -25,6 +25,17 @@ func (a Action) IsChildResource() bool {
 	return len(items) == 4
 }
 
+// ParentKey returns the key of the resource a child node hangs off, e.g.
+// "resources.schemas.foo" for "resources.schemas.foo.grants". Returns "" if
+// resourceKey is not a child node.
+func ParentKey(resourceKey string) string {
+	items := strings.Split(resourceKey, ".")
+	if len(items) != 4 {
+		return ""
+	}
+	return strings.Join(items[:3], ".")
+}
+
 type ActionType string
 
 // Actions are ordered in increasing severity.