Fix secret scope permissions migration from Terraform to Direct engine

NEXT_CHANGELOG.md

@@ -14,6 +14,7 @@
 * engine/direct: Fix unwanted recreation of secret scopes when scope_backend_type is not set ([#4834](https://github.com/databricks/cli/pull/4834))
 * engine/direct: Fix bind and unbind for non-Terraform resources ([#4850](https://github.com/databricks/cli/pull/4850))
 * engine/direct: Fix deploying removed principals ([#4824](https://github.com/databricks/cli/pull/4824))
+* engine/direct: Fix secret scope permissions migration from Terraform to Direct engine ([#4866](https://github.com/databricks/cli/pull/4866))
 
 ### Dependency updates
 

acceptance/bundle/invariant/configs/secret_scope_with_permissions.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  secret_scopes:
+    foo:
+      name: test-scope-$UNIQUE_NAME
+      backend_type: DATABRICKS
+      permissions:
+        - level: READ
+          group_name: users
+        - level: WRITE
+          group_name: admins

acceptance/bundle/invariant/migrate/test.toml

@@ -2,10 +2,6 @@
 EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"]
 EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"]
 
-# Unexpected action='create' for resources.secret_scopes.foo.permissions
-EnvMatrixExclude.no_secret_scope = ["INPUT_CONFIG=secret_scope.yml.tmpl"]
-EnvMatrixExclude.no_secret_scope2 = ["INPUT_CONFIG=secret_scope_default_backend_type.yml.tmpl"]
-
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to
 # ${databricks_job.job_b.permissions[0].level}, but Terraform's databricks_job resource

acceptance/bundle/invariant/test.toml

@@ -49,6 +49,7 @@ EnvMatrix.INPUT_CONFIG = [
     "schema_with_grants.yml.tmpl",
     "secret_scope.yml.tmpl",
     "secret_scope_default_backend_type.yml.tmpl",
+    "secret_scope_with_permissions.yml.tmpl",
     "synced_database_table.yml.tmpl",
     "volume.yml.tmpl",
 ]

acceptance/bundle/resources/secret_scopes/basic/out.plan1.terraform.json

@@ -3,6 +3,9 @@
   "plan": {
     "resources.secret_scopes.my_scope": {
       "action": "create"
+    },
+    "resources.secret_scopes.my_scope.permissions": {
+      "action": "create"
     }
   }
 }

acceptance/bundle/resources/secret_scopes/basic/out.plan2.terraform.json

@@ -3,6 +3,9 @@
   "plan": {
     "resources.secret_scopes.my_scope": {
       "action": "recreate"
+    },
+    "resources.secret_scopes.my_scope.permissions": {
+      "action": "recreate"
     }
   }
 }

acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.terraform.json

@@ -3,6 +3,9 @@
   "plan": {
     "resources.secret_scopes.my_scope": {
       "action": "skip"
+    },
+    "resources.secret_scopes.my_scope.permissions": {
+      "action": "skip"
     }
   }
 }

acceptance/bundle/resources/secret_scopes/delete_scope/out.plan.terraform.txt

@@ -1,5 +1,6 @@
 
 >>> [CLI] bundle plan
 delete secret_scopes.second
+delete secret_scopes.second.permissions
 
-Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+Plan: 0 to add, 0 to change, 2 to delete, 1 unchanged

acceptance/bundle/resources/secret_scopes/permissions/out.plan.recreate.terraform.txt

@@ -1,3 +1,4 @@
 recreate secret_scopes.my_scope
+recreate secret_scopes.my_scope.permissions
 
-Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
+Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged

acceptance/bundle/resources/secret_scopes/permissions/script

@@ -36,7 +36,7 @@ cleanup() {
 trap cleanup EXIT
 
 title "create secret scope with permissions"
-trace $CLI bundle plan > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.txt
+trace $CLI bundle plan > out.plan.create.txt
 trace $CLI bundle deploy
 scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name')
 
@@ -60,7 +60,7 @@ resources:
           level: MANAGE
 EOF
 envsubst < databricks.yml.tmpl > databricks.yml
-trace $CLI bundle plan > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.txt
+trace $CLI bundle plan > out.plan.update.txt
 trace $CLI bundle deploy
 trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort
 

bundle/deploy/terraform/pkg.go

@@ -133,6 +133,7 @@ var GroupToTerraformName = map[string]string{
 	// 3 level groups: resources.*.GROUP
 	"permissions": "databricks_permissions",
 	"grants":      "databricks_grants",
+	"secret_acls": "databricks_secret_acl",
 }
 
 var TerraformToGroupName = func() map[string]string {

bundle/deploy/terraform/showplanfile.go

@@ -10,12 +10,6 @@ import (
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
-// silentlyUpdatedResources contains resource types that are automatically created by DABs,
-// no need to show them in the plan
-var silentlyUpdatedResources = map[string]bool{
-	"databricks_secret_acl": true,
-}
-
 var prefixToGroup = []struct{ prefix, group string }{
 	{"job_", "jobs"},
 	{"pipeline_", "pipelines"},
@@ -63,6 +57,17 @@ func convertGrantsResourceNameToKey(terraformName string) string {
 	return ""
 }
 
+// convertSecretAclNameToScopeKey converts terraform secret ACL resource names to scope permission keys.
+// ACL names have format "secret_acl_<scope_key>_<idx>" (see convert_secret_scope.go).
+// e.g., "secret_acl_my_scope_0" -> "resources.secret_scopes.my_scope.permissions"
+func convertSecretAclNameToScopeKey(name string) string {
+	name, _ = strings.CutPrefix(name, "secret_acl_")
+	if i := strings.LastIndex(name, "_"); i >= 0 {
+		name = name[:i]
+	}
+	return "resources.secret_scopes." + name + ".permissions"
+}
+
 // populatePlan populates a deployplan.Plan from Terraform resource changes.
 func populatePlan(ctx context.Context, plan *deployplan.Plan, changes []*tfjson.ResourceChange) {
 	for _, rc := range changes {
@@ -88,25 +93,35 @@ func populatePlan(ctx context.Context, plan *deployplan.Plan, changes []*tfjson.
 
 		group, ok := TerraformToGroupName[rc.Type]
 		if !ok {
-			if !silentlyUpdatedResources[rc.Type] {
-				log.Warnf(ctx, "unknown resource type '%s'", rc.Type)
-			}
+			log.Warnf(ctx, "unknown resource type '%s'", rc.Type)
 			continue
 		}
 
 		var key string
 		switch group {
 		case "permissions":
-			// Convert terraform permission resource name back to hierarchical resource key
 			key = convertPermissionsResourceNameToKey(rc.Name)
 		case "grants":
-			// Convert terraform grants resource name back to hierarchical resource key
 			key = convertGrantsResourceNameToKey(rc.Name)
+		case "secret_acls":
+			key = convertSecretAclNameToScopeKey(rc.Name)
 		default:
 			key = "resources." + group + "." + rc.Name
 		}
 
-		plan.Plan[key] = &deployplan.PlanEntry{Action: actionType}
+		if existing, ok := plan.Plan[key]; ok {
+			// For secret ACLs, multiple individual ACL changes are merged into a single
+			// scope-level permissions entry. When the actions differ (e.g., some ACLs are
+			// recreated while others are deleted), it means permissions are being updated,
+			// not deleted entirely.
+			if group == "secret_acls" && existing.Action != actionType {
+				existing.Action = deployplan.Update
+			} else {
+				existing.Action = deployplan.GetHigherAction(existing.Action, actionType)
+			}
+		} else {
+			plan.Plan[key] = &deployplan.PlanEntry{Action: actionType}
+		}
 	}
 }
 

bundle/deploy/terraform/showplanfile_test.go

@@ -76,3 +76,89 @@ func TestPopulatePlan(t *testing.T) {
 	// Unknown resource type should not be in the plan
 	assert.NotContains(t, plan.Plan, "resources.recreate whatever")
 }
+
+func TestPopulatePlanSecretAcl(t *testing.T) {
+	ctx := t.Context()
+	changes := []*tfjson.ResourceChange{
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionCreate}},
+			Name:   "secret_acl_my_scope_0",
+		},
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}},
+			Name:   "secret_acl_my_scope_1",
+		},
+	}
+
+	plan := deployplan.NewPlanTerraform()
+	populatePlan(ctx, plan, changes)
+
+	// Multiple ACL changes for the same scope with different actions are merged as Update.
+	assert.Equal(t, map[string]*deployplan.PlanEntry{
+		"resources.secret_scopes.my_scope.permissions": {Action: deployplan.Update},
+	}, plan.Plan)
+}
+
+func TestPopulatePlanSecretAclMixedCreateDelete(t *testing.T) {
+	ctx := t.Context()
+	changes := []*tfjson.ResourceChange{
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionDelete}},
+			Name:   "secret_acl_my_scope_0",
+		},
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionCreate}},
+			Name:   "secret_acl_my_scope_1",
+		},
+	}
+
+	plan := deployplan.NewPlanTerraform()
+	populatePlan(ctx, plan, changes)
+
+	assert.Equal(t, map[string]*deployplan.PlanEntry{
+		"resources.secret_scopes.my_scope.permissions": {Action: deployplan.Update},
+	}, plan.Plan)
+}
+
+func TestPopulatePlanSecretAclMixedRecreateDelete(t *testing.T) {
+	ctx := t.Context()
+	// Simulates a permission update where some ACLs are recreated (principal changed)
+	// and some are deleted (principal removed). This is the typical Terraform plan shape
+	// when updating secret scope permissions.
+	changes := []*tfjson.ResourceChange{
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}},
+			Name:   "secret_acl_my_scope_0",
+		},
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}},
+			Name:   "secret_acl_my_scope_1",
+		},
+		{
+			Type:   "databricks_secret_acl",
+			Change: &tfjson.Change{Actions: tfjson.Actions{tfjson.ActionDelete}},
+			Name:   "secret_acl_my_scope_2",
+		},
+	}
+
+	plan := deployplan.NewPlanTerraform()
+	populatePlan(ctx, plan, changes)
+
+	// When permissions are being updated (some ACLs recreated, some deleted),
+	// the aggregated action should be Update, not Delete.
+	assert.Equal(t, map[string]*deployplan.PlanEntry{
+		"resources.secret_scopes.my_scope.permissions": {Action: deployplan.Update},
+	}, plan.Plan)
+}
+
+func TestConvertSecretAclNameToScopeKey(t *testing.T) {
+	assert.Equal(t, "resources.secret_scopes.my_scope.permissions", convertSecretAclNameToScopeKey("secret_acl_my_scope_0"))
+	assert.Equal(t, "resources.secret_scopes.my_scope.permissions", convertSecretAclNameToScopeKey("secret_acl_my_scope_1"))
+	assert.Equal(t, "resources.secret_scopes.scope_123.permissions", convertSecretAclNameToScopeKey("secret_acl_scope_123_2"))
+}

bundle/deploy/terraform/util.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/statemgmt/resourcestate"
+	"github.com/databricks/cli/libs/log"
 	tfjson "github.com/hashicorp/terraform-json"
 )
 
@@ -75,18 +76,27 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap
 			continue
 		}
 		for _, instance := range resource.Instances {
-			groupName, ok := TerraformToGroupName[resource.Type]
+			var resourceKey string
+			var resourceState ResourceState
 
+			groupName, ok := TerraformToGroupName[resource.Type]
 			if !ok {
-				// secret_acls
+				log.Warnf(ctx, "Unknown Terraform resource type: %s", resource.Type)
 				continue
 			}
 
-			var resourceKey string
-			var resourceState ResourceState
-
 			switch groupName {
-			case "apps", "secret_scopes", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints":
+			case "secret_acls":
+				// Secret ACLs don't have their own state entries; permissions are
+				// created alongside the scope in the "secret_scopes" case below.
+				continue
+			case "secret_scopes":
+				resourceKey = "resources." + groupName + "." + resource.Name
+				resourceState = ResourceState{ID: instance.Attributes.Name}
+				// The direct engine manages permissions as a sub-resource
+				// (SecretScopeFixups adds MANAGE ACL for the current user).
+				result[resourceKey+".permissions"] = ResourceState{ID: instance.Attributes.Name}
+			case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints":
 				resourceKey = "resources." + groupName + "." + resource.Name
 				resourceState = ResourceState{ID: instance.Attributes.Name}
 			case "dashboards":