direct: revoke grants and permissions when the block is deleted or emptied

.nextchanges/notable-changes/grants-and-permissions-revoked-on-delete.md

@@ -0,0 +1,12 @@
+Deleting a `grants` or `permissions` block, or emptying it to `[]`, now revokes what it
+granted. Both were previously ignored: the grant or permission stayed in place, and so did
+anything added outside the bundle. An empty list and a removed block are the same request,
+and both are now applied as a delete ([#6474](https://github.com/databricks/cli/pull/6474)).
+
+`grants: []` revokes everything, and so does `permissions: []` - except on jobs, pipelines and
+SQL warehouses, where the API requires exactly one owner on every update, so the owner is kept.
+That owner is read from the object rather than assumed to be whoever deploys, so an object
+owned by a service principal, or handed over since it was deployed, keeps the owner it has.
+
+Unaffected: a bundle that has always had an empty list, deleting the resource a block belongs
+to, and `secret_scopes` permissions, which already revoked down to the deploying user.

acceptance/bundle/resources/cluster_policies/permissions/revoke_semantics/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: cluster-policy-permissions-revoke-$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    test_cluster_policy:
+      name: my_cluster_policy-$UNIQUE_NAME
+      definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}'
+      permissions: # PERMISSIONS
+        - level: CAN_USE # PERMISSIONS
+          group_name: users # PERMISSIONS

acceptance/bundle/resources/cluster_policies/permissions/revoke_semantics/output.txt

@@ -0,0 +1,37 @@
+
+=== Deploy with one grant
+>>> [CLI] bundle deploy -qq
+
+>>> acl
+[
+  {
+    "levels": [
+      "CAN_USE"
+    ],
+    "name": "users"
+  }
+]
+
+=== Empty the list: everything is revoked, there is no owner to keep
+>>> [CLI] bundle plan
+delete cluster_policies.test_cluster_policy.permissions
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy -qq
+
+>>> acl
+[]
+
+=== Put the grant back, then remove the block entirely: same outcome
+>>> [CLI] bundle deploy -qq
+
+>>> [CLI] bundle plan
+delete cluster_policies.test_cluster_policy.permissions
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy -qq
+
+>>> acl
+[]

acceptance/bundle/resources/cluster_policies/permissions/revoke_semantics/script

@@ -0,0 +1,39 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+cp databricks.yml databricks.yml.saved
+
+cleanup() {
+    errcode $CLI bundle destroy --auto-approve &> LOG.cleanup
+    rm -f out.requests.txt databricks.yml.saved
+}
+trap cleanup EXIT
+
+# A cluster policy has no owner, so an emptied list revokes to nothing at all - unlike a job,
+# which keeps one IS_OWNER (see ../../../permissions/jobs/revoke_semantics). FixPermissions skips
+# cluster policies, so the deploying user is never added either.
+acl() {
+    $CLI permissions get cluster-policies "$POLICY_ID" |
+        jq -S '[(.access_control_list // [])[]
+                | select(any(.all_permissions[]; .inherited == false))
+                | {name: (.user_name // .service_principal_name // .group_name),
+                   levels: [.all_permissions[] | select(.inherited == false) | .permission_level]}]'
+}
+
+title "Deploy with one grant"
+trace $CLI bundle deploy -qq
+POLICY_ID=$(read_id.py test_cluster_policy)
+trace acl
+
+title "Empty the list: everything is revoked, there is no owner to keep"
+update_file.py databricks.yml 'permissions: # PERMISSIONS' 'permissions: []'
+sed -i '/# PERMISSIONS/d' databricks.yml
+trace $CLI bundle plan
+trace $CLI bundle deploy -qq
+trace acl
+
+title "Put the grant back, then remove the block entirely: same outcome"
+cp databricks.yml.saved databricks.yml
+trace $CLI bundle deploy -qq
+sed -i '/# PERMISSIONS/d' databricks.yml
+trace $CLI bundle plan
+trace $CLI bundle deploy -qq
+trace acl

acceptance/bundle/resources/cluster_policies/permissions/revoke_semantics/test.toml

@@ -0,0 +1,3 @@
+# A cluster policy has no owner, so the revoke floor is an empty ACL rather than owner-only.
+# The testserver enforces neither rule, so this only means anything against a real workspace.
+Ignore = [".databricks", "databricks.yml", "databricks.yml.saved"]

acceptance/bundle/resources/grants/schemas/delete_semantics/databricks.empty.yml.tmpl

@@ -0,0 +1,2 @@
+bundle:
+  name: schema-grants-delete-semantics-$UNIQUE_NAME

acceptance/bundle/resources/grants/schemas/delete_semantics/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: schema-grants-delete-semantics-$UNIQUE_NAME
+
+resources:
+  schemas:
+    grants_schema:
+      name: schema_delete_semantics_$UNIQUE_NAME
+      catalog_name: main
+      grants: [{ principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA] }] # GRANTS

acceptance/bundle/resources/grants/schemas/delete_semantics/out.direct.txt

@@ -0,0 +1,50 @@
+=== only the grants node is deleted
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "ALL_PRIVILEGES"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the grants list is empty
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "ALL_PRIVILEGES"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the parent is deleted too
+delete schemas.grants_schema
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged
+{
+  "method": "DELETE",
+  "path": "/api/2.1/unity-catalog/schemas/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "q": {
+    "force": "true"
+  }
+}

acceptance/bundle/resources/grants/schemas/delete_semantics/out.terraform.txt

@@ -0,0 +1,64 @@
+=== only the grants node is deleted
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "USE_SCHEMA"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the grants list is empty
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "USE_SCHEMA"
+        ]
+      }
+    ]
+  }
+}
+json = {};
+=== the parent is deleted too
+delete schemas.grants_schema
+delete schemas.grants_schema.grants
+
+Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged
+{
+  "method": "DELETE",
+  "path": "/api/2.1/unity-catalog/schemas/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "q": {
+    "force": "true"
+  }
+}
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/schema/main.schema_delete_semantics_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "USE_SCHEMA"
+        ]
+      }
+    ]
+  }
+}

acceptance/bundle/resources/grants/schemas/delete_semantics/output.txt

@@ -0,0 +1,27 @@
+
+=== Deploy the schema with one grant
+>>> [CLI] bundle deploy -qq
+
+=== Case 1: the grants block is removed, the schema stays
+>>> [CLI] bundle deploy -qq
+
+=== Put the grant back
+>>> [CLI] bundle deploy -qq
+
+=== Case 2: the grants list is set to []
+>>> [CLI] bundle deploy -qq
+
+=== Put the grant back
+>>> [CLI] bundle deploy -qq
+
+=== Case 3: the schema and its grants are deleted together
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.schemas.grants_schema
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.grants_schema
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-delete-semantics-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/grants/schemas/delete_semantics/script

@@ -0,0 +1,54 @@
+SCHEMA_FULL_NAME=main.schema_delete_semantics_$UNIQUE_NAME
+GRANTS_LINE='grants: [{ principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA] }] # GRANTS'
+
+# The plan, the requests and the leftover grants all go to one per-engine file: the engines
+# disagree on how a dropped grants node is classified and on what they send for it.
+ENGINE_OUT=out.$DATABRICKS_BUNDLE_ENGINE.txt
+
+envsubst < databricks.yml.tmpl > databricks.yml
+cp databricks.yml databricks.yml.saved
+
+cleanup() {
+    errcode $CLI bundle destroy --auto-approve &> LOG.cleanup
+    rm -f out.requests.txt databricks.yml.saved tmp.yml
+}
+trap cleanup EXIT
+
+record() {
+    echo "=== $1" >> $ENGINE_OUT
+    $CLI bundle plan >> $ENGINE_OUT
+    trace $CLI bundle deploy -qq
+    print_requests.py //unity-catalog --sort >> $ENGINE_OUT
+    $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort_lines.py --repl >> $ENGINE_OUT
+}
+
+restore_grant() {
+    title "Put the grant back"
+    cp databricks.yml.saved databricks.yml
+    trace $CLI bundle deploy -qq
+    rm -f out.requests.txt
+}
+
+title "Deploy the schema with one grant"
+trace $CLI bundle deploy -qq
+rm -f out.requests.txt
+
+title "Case 1: the grants block is removed, the schema stays"
+grep -v GRANTS databricks.yml > tmp.yml && mv tmp.yml databricks.yml
+record "only the grants node is deleted"
+
+restore_grant
+title "Case 2: the grants list is set to []"
+update_file.py databricks.yml "$GRANTS_LINE" 'grants: []'
+record "the grants list is empty"
+
+restore_grant
+title "Case 3: the schema and its grants are deleted together"
+echo "=== the parent is deleted too" >> $ENGINE_OUT
+# Planned against a config with the schema gone, so the plan names both nodes and the
+# action each one gets. `bundle destroy` applies the same shape but its banner hides
+# child nodes, so it never shows what happens to the grants.
+envsubst < databricks.empty.yml.tmpl > databricks.yml
+$CLI bundle plan >> $ENGINE_OUT
+trace $CLI bundle destroy --auto-approve
+print_requests.py //unity-catalog --sort >> $ENGINE_OUT

acceptance/bundle/resources/grants/schemas/delete_semantics/test.toml

@@ -0,0 +1,2 @@
+# Six deploys plus a destroy; extra headroom for heavy parallel runs.
+Timeout = '2m'

acceptance/bundle/resources/permissions/jobs/revoke_semantics/databricks.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: job-permissions-revoke-$UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: job_revoke_semantics_$UNIQUE_NAME
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: ./notebook.py
+      permissions: # PERMISSIONS
+        - level: CAN_VIEW # PERMISSIONS
+          user_name: deco-test-user@databricks.com # PERMISSIONS

acceptance/bundle/resources/permissions/jobs/revoke_semantics/notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("hello")