From b6d6f855bde4e1c12326cac29b5b745882fd35ed Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 12 Jun 2026 20:44:08 +0200 Subject: [PATCH] Add --select flag to bundle config-remote-sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an opt-in --select flag to the experimental `bundle config-remote-sync` command that restricts detected and saved changes to specific resources. Each selector is ":" (e.g. jobs:123456789), using the resource type and its deployed resource ID — the pair the workspace UI knows from a resource's page, which has the ID but not the bundle "type.name" key. The type is required because a resource ID is only unique within a type, so an ID that collides across types would otherwise select the wrong resource. Selection is therefore independent from `bundle deploy --select`, which matches keys. Selectors are resolved to plan keys against the deployment state after planning, so ${resources.*} references still resolve; only the emitted change set is restricted. A selector that matches no deployed resource is an error. Default behavior without the flag is unchanged. This limits the blast radius of a sync run: syncing one resource can no longer rewrite an unrelated drifted resource's configuration. --- .../select_basic/databricks.yml.tmpl | 30 ++++++ .../select_basic/out.test.toml | 4 + .../select_basic/output.txt | 66 +++++++++++++ .../config-remote-sync/select_basic/script | 48 ++++++++++ .../config-remote-sync/select_basic/test.toml | 10 ++ .../select_multiple/databricks.yml.tmpl | 41 ++++++++ .../select_multiple/out.test.toml | 4 + .../select_multiple/output.txt | 74 +++++++++++++++ .../config-remote-sync/select_multiple/script | 42 +++++++++ .../select_multiple/test.toml | 10 ++ bundle/configsync/diff.go | 39 ++++---- bundle/configsync/select.go | 94 +++++++++++++++++++ bundle/configsync/select_test.go | 94 +++++++++++++++++++ cmd/bundle/config_remote_sync.go | 29 +++++- 14 files changed, 567 insertions(+), 18 deletions(-) create mode 100644 acceptance/bundle/config-remote-sync/select_basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/select_basic/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/select_basic/output.txt create mode 100644 acceptance/bundle/config-remote-sync/select_basic/script create mode 100644 acceptance/bundle/config-remote-sync/select_basic/test.toml create mode 100644 acceptance/bundle/config-remote-sync/select_multiple/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/select_multiple/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/select_multiple/output.txt create mode 100644 acceptance/bundle/config-remote-sync/select_multiple/script create mode 100644 acceptance/bundle/config-remote-sync/select_multiple/test.toml create mode 100644 bundle/configsync/select.go create mode 100644 bundle/configsync/select_test.go diff --git a/acceptance/bundle/config-remote-sync/select_basic/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/select_basic/databricks.yml.tmpl new file mode 100644 index 00000000000..8301740fedd --- /dev/null +++ b/acceptance/bundle/config-remote-sync/select_basic/databricks.yml.tmpl @@ -0,0 +1,30 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + job_one: + max_concurrent_runs: 1 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/job1 + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + + job_two: + max_concurrent_runs: 2 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/job2 + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + +targets: + default: + mode: development diff --git a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml new file mode 100644 index 00000000000..579b1e4a3c9 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_basic/output.txt b/acceptance/bundle/config-remote-sync/select_basic/output.txt new file mode 100644 index 00000000000..9357a819faa --- /dev/null +++ b/acceptance/bundle/config-remote-sync/select_basic/output.txt @@ -0,0 +1,66 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Modify both jobs remotely +=== Sync only job_one, selected by its type and deployed resource id +Detected changes in 1 resource(s): + +Resource: resources.jobs.job_one + max_concurrent_runs: replace + + + +=== Only job_one is updated; job_two is left untouched + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -5,5 +5,5 @@ + jobs: + job_one: +- max_concurrent_runs: 1 ++ max_concurrent_runs: 5 + tasks: + - task_key: main + +=== Selecting job_one again is idempotent +No changes detected. + + +=== Unfiltered sync still detects the job_two drift (no lost updates) +Detected changes in 1 resource(s): + +Resource: resources.jobs.job_two + max_concurrent_runs: replace + + + +=== An unknown resource id is rejected +>>> [CLI] bundle config-remote-sync --select jobs:no-such-id-123 +Error: no deployed jobs resource with id no-such-id-123 + +Exit code: 1 + +=== A selector without a type is rejected +>>> [CLI] bundle config-remote-sync --select no-such-id-123 +Error: invalid --select value "no-such-id-123", expected : (e.g. jobs:[NUMID]) + +Exit code: 1 + +=== An id that exists under a different type is rejected (no cross-type collision) +>>> [CLI] bundle config-remote-sync --select pipelines:[JOB_ONE_ID] +Error: no deployed pipelines resource with id [JOB_ONE_ID] + +Exit code: 1 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job_one + delete resources.jobs.job_two + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/select_basic/script b/acceptance/bundle/config-remote-sync/select_basic/script new file mode 100644 index 00000000000..cb3fadd9d40 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/select_basic/script @@ -0,0 +1,48 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_one_id="$(read_id.py job_one)" +job_two_id="$(read_id.py job_two)" + +title "Modify both jobs remotely" +edit_resource.py jobs $job_one_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -5,5 +5,5 @@ + jobs: + job_a: +- max_concurrent_runs: 1 ++ max_concurrent_runs: 5 + tasks: + - task_key: main +@@ -16,5 +16,5 @@ + + job_b: +- max_concurrent_runs: 1 ++ max_concurrent_runs: 5 + tasks: + - task_key: main + +=== Unfiltered sync still detects the job_c drift +Detected changes in 1 resource(s): + +Resource: resources.jobs.job_c + max_concurrent_runs: replace + + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job_a + delete resources.jobs.job_b + delete resources.jobs.job_c + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/select_multiple/script b/acceptance/bundle/config-remote-sync/select_multiple/script new file mode 100644 index 00000000000..c2f4a89d25c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/select_multiple/script @@ -0,0 +1,42 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_a_id="$(read_id.py job_a)" +job_b_id="$(read_id.py job_b)" +job_c_id="$(read_id.py job_c)" + +title "Modify all three jobs remotely" +for id in $job_a_id $job_b_id $job_c_id; do +edit_resource.py jobs $id <:" selectors to their plan keys +// ("resources..") using the open deployment state (see +// OpenDeploymentState). +// +// Selection uses both the resource type and the deployed resource id (a job id, +// pipeline id, dashboard id, ...) — the pair the workspace UI knows from a +// resource's page. The type is required because a resource id is only unique +// within a type: an id that happens to collide across types (e.g. a job and a +// warehouse) would otherwise select the wrong resource. is the bundle +// resource type as it appears in the plan key, e.g. "jobs" or "pipelines". This +// is also why selection is independent from `bundle deploy --select`, which +// matches "type.name" keys. +// +// A selector that matches no deployed resource is an error: only deployed +// resources have an id, so a selector matching nothing is a caller mistake. +// Duplicate selectors are deduplicated; the returned keys preserve the order in +// which their selectors first appear. +func ResolveResourceSelectors(deployBundle *direct.DeploymentBundle, selectors []string) ([]string, error) { + // Index deployed resources by ":". State keys have the form + // "resources.."; indexing by the component means a + // selector can only ever match a resource of that exact type, never an id + // that happens to collide across types. + byTypeID := make(map[string]string) + for key := range deployBundle.StateDB.Data.State { + id := deployBundle.StateDB.GetResourceID(key) + if id == "" { + continue + } + typeAndName, ok := strings.CutPrefix(key, "resources.") + if !ok { + continue + } + resourceType, _, ok := strings.Cut(typeAndName, ".") + if !ok { + continue + } + byTypeID[resourceType+":"+id] = key + } + + keys := make([]string, 0, len(selectors)) + for _, selector := range selectors { + resourceType, id, ok := strings.Cut(selector, ":") + if !ok || resourceType == "" || id == "" { + return nil, fmt.Errorf("invalid --select value %q, expected : (e.g. jobs:123456789)", selector) + } + key, ok := byTypeID[selector] + if !ok { + return nil, fmt.Errorf("no deployed %s resource with id %s", resourceType, id) + } + if !slices.Contains(keys, key) { + keys = append(keys, key) + } + } + return keys, nil +} + +// FilterChanges returns the subset of changes that belong to the resources in +// selected, a list of plan keys ("resources..") as returned by +// ResolveResourceSelectors. Change keys are plan keys too. +// +// Selection is at the resource level: a change node is kept when it is the +// selected resource itself or any node beneath it, matched by prefix on the "." +// path boundary. This groups a resource's permissions/grants sub-nodes +// ("resources...permissions") with their parent, so selecting a +// resource never silently skips its permissions. The "." boundary stops +// "resources.jobs.foo" from also matching an unrelated "resources.jobs.foobar". +// +// Only the selected resources' own nodes are kept: unlike deploy's plan +// filtering, the selection is not expanded to transitive dependencies, because +// dependencies matter only when planning (DetectChanges always plans the full +// resource set so ${resources.*} references resolve), not when deciding which +// resources' configuration may be rewritten. +func FilterChanges(changes Changes, selected []string) Changes { + filtered := make(Changes) + for key, resourceChanges := range changes { + for _, sel := range selected { + if key == sel || strings.HasPrefix(key, sel+".") { + filtered[key] = resourceChanges + break + } + } + } + return filtered +} diff --git a/bundle/configsync/select_test.go b/bundle/configsync/select_test.go new file mode 100644 index 00000000000..7b6b45e3f38 --- /dev/null +++ b/bundle/configsync/select_test.go @@ -0,0 +1,94 @@ +package configsync + +import ( + "maps" + "slices" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFilterChanges(t *testing.T) { + changes := Changes{ + "resources.jobs.foo": { + "max_concurrent_runs": {Operation: OperationReplace, Value: 5}, + }, + "resources.jobs.foo.permissions": { + "[0].level": {Operation: OperationReplace, Value: "CAN_MANAGE"}, + }, + // Boundary: shares the "resources.jobs.foo" prefix but is a different + // resource, so selecting "resources.jobs.foo" must not pull it in. + "resources.jobs.foobar": { + "name": {Operation: OperationReplace, Value: "foobar"}, + }, + "resources.jobs.bar": { + "name": {Operation: OperationReplace, Value: "bar"}, + }, + "resources.schemas.baz": { + "comment": {Operation: OperationAdd, Value: "c"}, + }, + "resources.schemas.baz.grants": { + "[0].principal": {Operation: OperationAdd, Value: "users"}, + }, + // A resource whose name is literally "permissions" is the resource itself, + // not a sub-node, and is kept only when selected by its own key. + "resources.jobs.permissions": { + "name": {Operation: OperationReplace, Value: "p"}, + }, + } + + tests := []struct { + name string + selected []string + wantKeys []string + }{ + { + name: "resource groups its permissions sub-node by prefix, excludes the foobar sibling", + selected: []string{"resources.jobs.foo"}, + wantKeys: []string{"resources.jobs.foo", "resources.jobs.foo.permissions"}, + }, + { + name: "resource without sub-nodes", + selected: []string{"resources.jobs.bar"}, + wantKeys: []string{"resources.jobs.bar"}, + }, + { + name: "grants sub-node follows its parent", + selected: []string{"resources.schemas.baz"}, + wantKeys: []string{"resources.schemas.baz", "resources.schemas.baz.grants"}, + }, + { + name: "multiple selections are a union", + selected: []string{"resources.jobs.bar", "resources.schemas.baz"}, + wantKeys: []string{"resources.jobs.bar", "resources.schemas.baz", "resources.schemas.baz.grants"}, + }, + { + name: "resource with no detected changes yields empty result", + selected: []string{"resources.jobs.never_drifted"}, + wantKeys: []string{}, + }, + { + name: "resource named permissions is kept only by its own key", + selected: []string{"resources.jobs.permissions"}, + wantKeys: []string{"resources.jobs.permissions"}, + }, + { + name: "empty selection keeps nothing", + selected: nil, + wantKeys: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FilterChanges(changes, tt.selected) + assert.ElementsMatch(t, tt.wantKeys, slices.Collect(maps.Keys(got))) + for _, key := range tt.wantKeys { + assert.Equal(t, changes[key], got[key]) + } + }) + } + + // The input map is never mutated. + assert.Len(t, changes, 7) +} diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index c50e613d71a..39b6da88024 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -19,6 +19,7 @@ import ( func newConfigRemoteSyncCommand() *cobra.Command { var save bool + var selectResources []string cmd := &cobra.Command{ Use: "config-remote-sync", @@ -35,11 +36,15 @@ Examples: databricks bundle config-remote-sync # Show diff and save to files - databricks bundle config-remote-sync --save`, + databricks bundle config-remote-sync --save + + # Restrict the sync to a single resource by its type and deployed resource ID + databricks bundle config-remote-sync --select jobs:123456789 --save`, Hidden: true, // Used by DABs in the Workspace only } cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk") + cmd.Flags().StringSliceVar(&selectResources, "select", nil, "Sync only the given resources, each as : (e.g. jobs:123456789). Can be repeated or comma-separated.") cmd.RunE = func(cmd *cobra.Command, args []string) error { if runtime.GOOS == "windows" { @@ -54,11 +59,31 @@ Examples: b.SkipLocalFileValidation = true }, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { - changes, err := configsync.DetectChanges(ctx, b, stateDesc.Engine) + // Open the deployment state once and reuse it for both planning and + // selector resolution (avoids reading the terraform snapshot twice). + deployBundle, err := configsync.OpenDeploymentState(ctx, b, stateDesc.Engine) + if err != nil { + return err + } + + changes, err := configsync.DetectChanges(ctx, b, deployBundle) if err != nil { return fmt.Errorf("failed to detect changes: %w", err) } + if len(selectResources) > 0 { + // --select takes : selectors (what the workspace UI knows), + // resolved to plan keys against the same state DetectChanges planned. + // Filter after planning, never before: the plan must cover every + // resource so ${resources.*} references resolve; only the emitted + // changes are restricted to the selected resources. + selected, err := configsync.ResolveResourceSelectors(deployBundle, selectResources) + if err != nil { + return err + } + changes = configsync.FilterChanges(changes, selected) + } + fieldChanges, err := configsync.ResolveChanges(ctx, b, changes) if err != nil { return fmt.Errorf("failed to resolve field changes: %w", err)