direct: ignore remote-only changes on name-based ID fields

NEXT_CHANGELOG.md

@@ -18,6 +18,7 @@
 * Remove hidden, never-functional `--existing-dashboard-id`, `--existing-dashboard-path`, `--existing-alert-id`, and `--existing-genie-space-id` alias flags from `bundle generate`; use the documented `--existing-id` / `--existing-path` flags instead ([#5591](https://github.com/databricks/cli/pull/5591)).
 * engine/direct: Fix WAL corruption after two consecutive failed deploys ([#5606](https://github.com/databricks/cli/pull/5606)).
 * engine/direct: Don't open the deployment state WAL when a deploy's plan fails ([#5607](https://github.com/databricks/cli/pull/5607)).
+* direct: Stop spurious recreate/rename on redeploy when the backend normalizes a resource's name-based ID (e.g. Unity Catalog lowercasing a schema or volume name) ([#5599](https://github.com/databricks/cli/pull/5599)).
 
 ### Dependency updates
 

acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.json

@@ -36,7 +36,7 @@
         },
         "name": {
           "action": "recreate",
-          "reason": "immutable",
+          "reason": "id_field",
           "old": "[ENDPOINT_NAME_1]",
           "new": "[ENDPOINT_NAME_2]",
           "remote": "[ENDPOINT_NAME_1]"

acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json

@@ -58,7 +58,7 @@
       "changes": {
         "name": {
           "action": "recreate",
-          "reason": "immutable",
+          "reason": "id_field",
           "old": "[ORIGINAL_ENDPOINT_ID]",
           "new": "[NEW_ENDPOINT_ID]",
           "remote": "[ORIGINAL_ENDPOINT_ID]"

acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json

@@ -31,7 +31,7 @@
       "changes": {
         "table_name": {
           "action": "recreate",
-          "reason": "immutable",
+          "reason": "id_field",
           "old": "main.qm_test_[UNIQUE_NAME].test_table",
           "new": "main.qm_test_[UNIQUE_NAME].test_table_2",
           "remote": "main.qm_test_[UNIQUE_NAME].test_table"

acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt

@@ -8,7 +8,7 @@ json.plan.resources.secret_scopes.my_scope.new_state.value.scope_backend_type =
 json.plan.resources.secret_scopes.my_scope.remote_state.backend_type = "DATABRICKS";
 json.plan.resources.secret_scopes.my_scope.remote_state.name = "test-scope-[UNIQUE_NAME]-1";
 json.plan.resources.secret_scopes.my_scope.changes.scope.action = "recreate";
-json.plan.resources.secret_scopes.my_scope.changes.scope.reason = "immutable";
+json.plan.resources.secret_scopes.my_scope.changes.scope.reason = "id_field";
 json.plan.resources.secret_scopes.my_scope.changes.scope.old = "test-scope-[UNIQUE_NAME]-1";
 json.plan.resources.secret_scopes.my_scope.changes.scope.new = "test-scope-[UNIQUE_NAME]-2";
 json.plan.resources.secret_scopes.my_scope.changes.scope.remote = "test-scope-[UNIQUE_NAME]-1";

acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt

@@ -32,7 +32,7 @@ Deployment complete!
     "changes": {
       "name": {
         "action": "skip",
-        "reason": "uc_case"
+        "reason": "id_field"
       }
     }
   },
@@ -41,11 +41,11 @@ Deployment complete!
     "changes": {
       "name": {
         "action": "skip",
-        "reason": "uc_case"
+        "reason": "id_changes"
       },
       "schema_name": {
         "action": "skip",
-        "reason": "uc_case"
+        "reason": "id_field"
       },
       "storage_location": {
         "action": "skip",

bundle/direct/bundle_plan.go

@@ -369,6 +369,12 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 		} else if reason, ok := shouldSkip(generatedCfg, path, ch); ok {
 			ch.Action = deployplan.Skip
 			ch.Reason = reason
+		} else if action, reason, ok := classifyIDField(cfg, path, ch); ok {
+			ch.Action = action
+			ch.Reason = reason
+		} else if action, reason, ok := classifyIDField(generatedCfg, path, ch); ok {
+			ch.Action = action
+			ch.Reason = reason
 		} else if reason, ok := shouldSkipBackendDefault(cfg, path, ch); ok {
 			ch.Action = deployplan.Skip
 			ch.Reason = reason
@@ -381,11 +387,11 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change
 		} else if reason, ok := shouldSkipNormalized(generatedCfg, path, ch); ok {
 			ch.Action = deployplan.Skip
 			ch.Reason = reason
-		} else if action, reason := shouldUpdateOrRecreate(cfg, path); action != deployplan.Undefined {
-			ch.Action = action
+		} else if reason, ok := findMatchingRule(path, cfg.RecreateOnChanges); ok {
+			ch.Action = deployplan.Recreate
 			ch.Reason = reason
-		} else if action, reason := shouldUpdateOrRecreate(generatedCfg, path); action != deployplan.Undefined {
-			ch.Action = action
+		} else if reason, ok := findMatchingRule(path, generatedCfg.RecreateOnChanges); ok {
+			ch.Action = deployplan.Recreate
 			ch.Reason = reason
 		} else {
 			ch.Action = deployplan.Update
@@ -440,11 +446,41 @@ func shouldSkip(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNo
 	return "", false
 }
 
+// classifyIDField decides the action for a field that composes the resource's
+// name-based ID, in one place so the remote-only-vs-local rule does not depend on
+// ordering elsewhere in the ladder. Returns ok=false if the path is not such a field.
+//
+//   - Remote-only difference (Old==New): the resource was just fetched by that ID,
+//     so a differing remote value can only be backend normalization (e.g. UC
+//     lowercasing) — a real out-of-band rename would 404 and is handled as
+//     resource-gone. Skip.
+//   - Local change: provided_id_fields recreate (delete + create); updatable_id_fields
+//     rename via UpdateWithID.
+func classifyIDField(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, ch *deployplan.ChangeDesc) (deployplan.ActionType, string, bool) {
+	if cfg == nil {
+		return deployplan.Undefined, "", false
+	}
+	localChange := !structdiff.IsEqual(ch.Old, ch.New)
+	if reason, ok := findMatchingRule(path, cfg.ProvidedIDFields); ok {
+		if localChange {
+			return deployplan.Recreate, reason, true
+		}
+		return deployplan.Skip, reason, true
+	}
+	if reason, ok := findMatchingRule(path, cfg.UpdatableIDFields); ok {
+		if localChange {
+			return deployplan.UpdateWithID, reason, true
+		}
+		return deployplan.Skip, reason, true
+	}
+	return deployplan.Undefined, "", false
+}
+
 // shouldSkipNormalized skips a change that is a false diff caused by UC API
-// normalization: the API lowercases identifier names (normalize_case) and strips
-// trailing slashes from storage URLs (normalize_slash). The direct engine saves
-// local config to state, so without this the next plan sees the original value
-// against the normalized remote value and triggers a spurious recreate/update.
+// normalization: the API strips trailing slashes from storage URLs
+// (normalize_slash). The direct engine saves local config to state, so without
+// this the next plan sees the original value against the normalized remote value
+// and triggers a spurious recreate/update.
 func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, ch *deployplan.ChangeDesc) (string, bool) {
 	if cfg == nil {
 		return "", false
@@ -454,28 +490,12 @@ func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structp
 	if !newOk || !remoteOk {
 		return "", false
 	}
-	if reason, ok := findMatchingRule(path, cfg.NormalizeCase); ok && strings.EqualFold(newStr, remoteStr) {
-		return reason, true
-	}
 	if reason, ok := findMatchingRule(path, cfg.NormalizeSlash); ok && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") {
 		return reason, true
 	}
 	return "", false
 }
 
-func shouldUpdateOrRecreate(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) (deployplan.ActionType, string) {
-	if cfg == nil {
-		return deployplan.Undefined, ""
-	}
-	if reason, ok := findMatchingRule(path, cfg.RecreateOnChanges); ok {
-		return deployplan.Recreate, reason
-	}
-	if reason, ok := findMatchingRule(path, cfg.UpdateIDOnChanges); ok {
-		return deployplan.UpdateWithID, reason
-	}
-	return deployplan.Undefined, ""
-}
-
 // shouldSkipBackendDefault checks if a change should be skipped because the remote value
 // is a known backend default. Applies when old and new are nil but remote is set.
 // If the rule has allowed values, the remote value must match one of them.
@@ -679,7 +699,7 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s
 		return value, nil
 	}
 
-	canReadRemoteCache := targetAction == deployplan.Skip || (targetAction.KeepsID() && adapter.IsFieldInRecreateOnChanges(fieldPath))
+	canReadRemoteCache := targetAction == deployplan.Skip || (targetAction.KeepsID() && adapter.FieldTriggersRecreate(fieldPath))
 
 	if configValidErr != nil && remoteValidErr == nil {
 		// The field is only present in remote state schema.

bundle/direct/dresources/README.md

@@ -24,7 +24,7 @@ Each field with special plan/deploy behavior must be declared in `resources.yml`
    - `managed` — managed by the cloud provider or platform, not by the user config
  - **`ignore_local_changes`**: Ignore changes the user makes to this field. Use for fields that cannot be updated via API — either they are immutable after creation or require a separate API that is not yet implemented. Must have a comment in resources.yml explaining why.
  - **`recreate_on_changes`**: Changing this field requires delete + create. Use for truly immutable fields (name, type, location). The reason should reference API docs or TF provider.
- - **`update_id_on_changes`**: Changing this field changes the resource's ID. Requires `DoUpdateWithID` to be implemented.
+ - **`updatable_id_fields`**: Changing this field changes the resource's ID. Requires `DoUpdateWithID` to be implemented.
 
 ## Update mask
 

bundle/direct/dresources/adapter.go

@@ -346,12 +346,12 @@ func (a *Adapter) validate() error {
 
 	// Validate resourceConfig consistency with DoUpdateWithID
 	if a.overrideChangeDesc == nil {
-		hasUpdateWithIDTrigger := a.resourceConfig != nil && len(a.resourceConfig.UpdateIDOnChanges) > 0
+		hasUpdateWithIDTrigger := a.resourceConfig != nil && len(a.resourceConfig.UpdatableIDFields) > 0
 		if hasUpdateWithIDTrigger && a.doUpdateWithID == nil {
-			return errors.New("resourceConfig has update_id_on_changes but DoUpdateWithID is not implemented")
+			return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented")
 		}
 		if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger {
-			return errors.New("DoUpdateWithID is implemented but resourceConfig lacks update_id_on_changes")
+			return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields")
 		}
 	}
 
@@ -378,12 +378,20 @@ func (a *Adapter) GeneratedResourceConfig() *ResourceLifecycleConfig {
 	return a.generatedResourceConfig
 }
 
-func (a *Adapter) IsFieldInRecreateOnChanges(path *structpath.PathNode) bool {
+// FieldTriggersRecreate reports whether a local change to the field forces a
+// delete + create. Both recreate_on_changes and provided_id_fields do this, so a
+// caller that knows the ID is preserved can conclude the field is unchanged.
+func (a *Adapter) FieldTriggersRecreate(path *structpath.PathNode) bool {
 	for _, p := range a.resourceConfig.RecreateOnChanges {
 		if path.HasPatternPrefix(p.Field) {
 			return true
 		}
 	}
+	for _, p := range a.resourceConfig.ProvidedIDFields {
+		if path.HasPatternPrefix(p.Field) {
+			return true
+		}
+	}
 	return false
 }
 

bundle/direct/dresources/all_test.go

@@ -1003,8 +1003,11 @@ func validateResourceConfig(t *testing.T, stateType reflect.Type, cfg *ResourceL
 	for _, p := range cfg.RecreateOnChanges {
 		assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "RecreateOnChanges: %s", p.Field)
 	}
-	for _, p := range cfg.UpdateIDOnChanges {
-		assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "UpdateIDOnChanges: %s", p.Field)
+	for _, p := range cfg.ProvidedIDFields {
+		assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "ProvidedIDFields: %s", p.Field)
+	}
+	for _, p := range cfg.UpdatableIDFields {
+		assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "UpdatableIDFields: %s", p.Field)
 	}
 	for _, p := range cfg.IgnoreRemoteChanges {
 		assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "IgnoreRemoteChanges: %s", p.Field)

bundle/direct/dresources/app_test.go

@@ -147,6 +147,11 @@ func TestAppDoUpdate_UpdateMaskHasAllFields(t *testing.T) {
 		nonUpdatableFields = append(nonUpdatableFields, field.Field.String())
 	}
 
+	// provided_id_fields recreate on local changes, so they are not updatable either.
+	for _, field := range config.ProvidedIDFields {
+		nonUpdatableFields = append(nonUpdatableFields, field.Field.String())
+	}
+
 	fields := reflect.TypeFor[apps.App]()
 	var allFields []string
 	for field := range fields.Fields() {

bundle/direct/dresources/config.go

@@ -56,12 +56,20 @@ type ResourceLifecycleConfig struct {
 	// RecreateOnChanges: field patterns that trigger delete + create when changed.
 	RecreateOnChanges []FieldRule `yaml:"recreate_on_changes,omitempty"`
 
-	// UpdateIDOnChanges: field patterns that trigger UpdateWithID when changed.
-	UpdateIDOnChanges []FieldRule `yaml:"update_id_on_changes,omitempty"`
-
-	// NormalizeCase: string field patterns the UC API lowercases on write.
-	// A change is skipped when local and remote differ only by case.
-	NormalizeCase []FieldRule `yaml:"normalize_case,omitempty"`
+	// ProvidedIDFields: field patterns that compose the resource's ID — a name the
+	// user provides (not a server-generated id), which DoRead fetches by. Local
+	// changes trigger delete + create. Remote-only differences are skipped: since the
+	// user supplies the value and we just fetched by it, a differing remote value can
+	// only be backend normalization (e.g. UC lowercasing) — a real out-of-band rename
+	// would 404 and is handled as resource-gone.
+	ProvidedIDFields []FieldRule `yaml:"provided_id_fields,omitempty"`
+
+	// UpdatableIDFields: like ProvidedIDFields, these compose the resource's
+	// user-provided ID, but the backend supports renaming them in place. A local
+	// change triggers UpdateWithID (a rename; the ID changes) instead of delete +
+	// create. A remote-only difference is still skipped (see classifyIDField) — it
+	// can only be backend normalization, since we just fetched by this ID.
+	UpdatableIDFields []FieldRule `yaml:"updatable_id_fields,omitempty"`
 
 	// NormalizeSlash: string field patterns the UC API strips trailing slashes from.
 	// A change is skipped when local and remote differ only by trailing slashes.
@@ -87,8 +95,8 @@ var empty = ResourceLifecycleConfig{
 	IgnoreRemoteChanges: nil,
 	IgnoreLocalChanges:  nil,
 	RecreateOnChanges:   nil,
-	UpdateIDOnChanges:   nil,
-	NormalizeCase:       nil,
+	ProvidedIDFields:    nil,
+	UpdatableIDFields:   nil,
 	NormalizeSlash:      nil,
 	BackendDefaults:     nil,
 }

bundle/direct/dresources/config_test.go

@@ -16,7 +16,7 @@ func TestGetResourceConfig(t *testing.T) {
 	assert.Empty(t, GetResourceConfig("nonexistent").RecreateOnChanges)
 }
 
-// categoryRules projects ResourceLifecycleConfig's five categories onto a
+// categoryRules projects ResourceLifecycleConfig's categories onto a
 // uniform [name, []FieldRule] shape so the redundancy check can iterate them.
 func categoryRules(c ResourceLifecycleConfig) []struct {
 	name  string
@@ -33,7 +33,8 @@ func categoryRules(c ResourceLifecycleConfig) []struct {
 		{"ignore_remote_changes", c.IgnoreRemoteChanges},
 		{"ignore_local_changes", c.IgnoreLocalChanges},
 		{"recreate_on_changes", c.RecreateOnChanges},
-		{"update_id_on_changes", c.UpdateIDOnChanges},
+		{"provided_id_fields", c.ProvidedIDFields},
+		{"updatable_id_fields", c.UpdatableIDFields},
 		{"backend_defaults", backendAsFieldRules},
 	}
 }
@@ -72,3 +73,35 @@ func TestResourcesYMLNoRedundantRules(t *testing.T) {
 		}
 	}
 }
+
+// TestResourcesYMLActionCategoriesExclusive guards that a field is in at most one
+// of the action categories that decide a change's action. They are not
+// independent: classifyIDField (provided_id_fields, updatable_id_fields) runs
+// before recreate_on_changes in the ladder and short-circuits, so a field listed
+// in more than one would have all but the first entry silently dead — and the
+// categories disagree (e.g. provided_id_fields skips a remote-only diff that
+// recreate_on_changes would recreate).
+func TestResourcesYMLActionCategoriesExclusive(t *testing.T) {
+	cfg := MustLoadConfig()
+	for resourceType, rc := range cfg.Resources {
+		actionCats := []struct {
+			name  string
+			rules []FieldRule
+		}{
+			{"recreate_on_changes", rc.RecreateOnChanges},
+			{"provided_id_fields", rc.ProvidedIDFields},
+			{"updatable_id_fields", rc.UpdatableIDFields},
+		}
+		firstCat := map[string]string{}
+		for _, c := range actionCats {
+			for _, r := range c.rules {
+				field := r.Field.String()
+				if prev, ok := firstCat[field]; ok {
+					t.Errorf("bundle/direct/dresources/resources.yml: %s lists %q in both %s and %s; a field's action belongs to exactly one category", resourceType, field, prev, c.name)
+				} else {
+					firstCat[field] = c.name
+				}
+			}
+		}
+	}
+}