From edb6a1a665a3b3e536c50f375d97aecfbe6d28c6 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 3 Mar 2026 15:54:20 +0100 Subject: [PATCH 1/4] Test to track incorrect behavior --- .../validation_errors/databricks.yml.tmpl | 26 ++++++++++ .../validation_errors/out.test.toml | 8 ++++ .../validation_errors/output.txt | 26 ++++++++++ .../validation_errors/script | 48 +++++++++++++++++++ .../validation_errors/test.toml | 8 ++++ 5 files changed, 116 insertions(+) create mode 100644 acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/validation_errors/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/validation_errors/output.txt create mode 100644 acceptance/bundle/config-remote-sync/validation_errors/script create mode 100644 acceptance/bundle/config-remote-sync/validation_errors/test.toml diff --git a/acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl new file mode 100644 index 0000000000..c63034835c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + pipelines: + my_pipeline: + name: test-pipeline-$UNIQUE_NAME + root_path: ./pipeline_root + libraries: + - notebook: + path: /Users/{{workspace_user_name}}/notebook + + jobs: + my_job: + tasks: + - task_key: main + notebook_task: + notebook_path: ./src/notebook.py + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + +targets: + default: + mode: development diff --git a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml new file mode 100644 index 0000000000..c4500f378d --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false + +[GOOS] + windows = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt new file mode 100644 index 0000000000..02421b6050 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -0,0 +1,26 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Set correct paths and add git info remotely +=== Break local paths to simulate stale config +=== Sync with broken local paths +Error: notebook src/notebook.py not found + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +Exit code: 1 diff --git a/acceptance/bundle/config-remote-sync/validation_errors/script b/acceptance/bundle/config-remote-sync/validation_errors/script new file mode 100644 index 0000000000..be62c798c4 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/script @@ -0,0 +1,48 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# Create valid paths so that initial deploy succeeds +mkdir -p pipeline_root src +echo '# Databricks notebook source' > src/notebook.py + +cleanup() { + # Restore valid paths for destroy to work + mkdir -p pipeline_root src + echo '# Databricks notebook source' > src/notebook.py + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" +pipeline_id="$(read_id.py my_pipeline)" + + +title "Set correct paths and add git info remotely" +edit_resource.py pipelines $pipeline_id < Date: Tue, 3 Mar 2026 18:43:08 +0100 Subject: [PATCH 2/4] Skip validation --- .../validation_errors/output.txt | 40 +++++++- .../validation_errors/script | 4 +- bundle/bundle.go | 5 + bundle/config/mutator/translate_paths.go | 28 +++++- bundle/config/mutator/translate_paths_test.go | 98 +++++++++++++++++++ cmd/bundle/config_remote_sync.go | 4 + 6 files changed, 170 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt index 02421b6050..c4ce29efa0 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/output.txt +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -6,8 +6,44 @@ Deployment complete! === Set correct paths and add git info remotely === Break local paths to simulate stale config === Sync with broken local paths -Error: notebook src/notebook.py not found +Detected changes in 2 resource(s): +Resource: resources.jobs.my_job + git_source: add + tasks[task_key='main'].notebook_task.notebook_path: replace + +Resource: resources.pipelines.my_pipeline + root_path: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -6,5 +6,5 @@ + my_pipeline: + name: test-pipeline-[UNIQUE_NAME] +- root_path: ./pipeline_root ++ root_path: ./pipeline_root_v2 + libraries: + - notebook: +@@ -16,9 +16,13 @@ + - task_key: main + notebook_task: +- notebook_path: ./src/notebook.py ++ notebook_path: /Users/[USERNAME]/notebook + new_cluster: + spark_version: 13.3.x-snapshot-scala2.12 + node_type_id: [NODE_TYPE_ID] + num_workers: 1 ++ git_source: ++ git_branch: main ++ git_provider: gitHub ++ git_url: https://github.com/databricks/databricks-sdk-go.git + + targets: >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -22,5 +58,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - -Exit code: 1 diff --git a/acceptance/bundle/config-remote-sync/validation_errors/script b/acceptance/bundle/config-remote-sync/validation_errors/script index be62c798c4..828ec20644 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/script +++ b/acceptance/bundle/config-remote-sync/validation_errors/script @@ -5,8 +5,8 @@ mkdir -p pipeline_root src echo '# Databricks notebook source' > src/notebook.py cleanup() { - # Restore valid paths for destroy to work - mkdir -p pipeline_root src + # Restore valid paths for destroy to work (includes pipeline_root_v2 from sync) + mkdir -p pipeline_root pipeline_root_v2 src echo '# Databricks notebook source' > src/notebook.py trace $CLI bundle destroy --auto-approve } diff --git a/bundle/bundle.go b/bundle/bundle.go index b2dae07554..588c573ca4 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -149,6 +149,11 @@ type Bundle struct { // files AutoApprove bool + // SkipLocalFileValidation makes path translation tolerant of missing local files. + // When set, TranslatePaths computes workspace paths without verifying files exist. + // Used by config-remote-sync which may run when referenced local files are stale. + SkipLocalFileValidation bool + // Tagging is used to normalize tag keys and values. // The implementation depends on the cloud being targeted. Tagging tags.Cloud diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 213bf37fad..8b72f6942a 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -84,6 +84,10 @@ type translateContext struct { // It is equal to ${workspace.file_path} for regular deployments. // It points to the source root path for source-linked deployments. remoteRoot string + + // skipLocalFileValidation makes path translation tolerant of missing local files. + // When set, paths are translated without verifying files exist on the local filesystem. + skipLocalFileValidation bool } // rewritePath converts a given relative path from the loaded config to a new path based on the passed rewriting function @@ -180,6 +184,11 @@ func (t *translateContext) rewritePath( func (t *translateContext) translateNotebookPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath) if errors.Is(err, fs.ErrNotExist) { + if t.skipLocalFileValidation { + localRelPathNoExt := strings.TrimSuffix(localRelPath, path.Ext(localRelPath)) + return path.Join(t.remoteRoot, localRelPathNoExt), nil + } + if path.Ext(localFullPath) != notebook.ExtensionNone { return "", fmt.Errorf("notebook %s not found", literal) } @@ -215,6 +224,9 @@ to contain one of the following file extensions: [%s]`, literal, strings.Join(no func (t *translateContext) translateFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath) if errors.Is(err, fs.ErrNotExist) { + if t.skipLocalFileValidation { + return path.Join(t.remoteRoot, localRelPath), nil + } return "", fmt.Errorf("file %s not found", literal) } if err != nil { @@ -229,6 +241,9 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { info, err := t.b.SyncRoot.Stat(localRelPath) if err != nil { + if t.skipLocalFileValidation { + return path.Join(t.remoteRoot, localRelPath), nil + } return "", err } if !info.IsDir() { @@ -244,6 +259,9 @@ func (t *translateContext) translateGlobPath(ctx context.Context, literal, local func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) { info, err := os.Stat(filepath.FromSlash(localFullPath)) if errors.Is(err, fs.ErrNotExist) { + if t.skipLocalFileValidation { + return localFullPath, nil + } return "", fmt.Errorf("directory %s not found", literal) } if err != nil { @@ -311,8 +329,9 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { t := &translateContext{ - b: b, - seen: make(map[string]string), + b: b, + seen: make(map[string]string), + skipLocalFileValidation: b.SkipLocalFileValidation, } return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ @@ -327,8 +346,9 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { t := &translateContext{ - b: b, - seen: make(map[string]string), + b: b, + seen: make(map[string]string), + skipLocalFileValidation: b.SkipLocalFileValidation, } return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 299aa45046..396a628770 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -1015,3 +1015,101 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) { b.Config.Resources.Pipelines["pipeline"].Libraries[1].Notebook.Path, ) } + +func TestTranslatePathsWithSkipLocalFileValidation(t *testing.T) { + dir := t.TempDir() + // Intentionally do NOT create any files — paths are stale/missing. + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + SkipLocalFileValidation: true, + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: jobs.JobSettings{ + Tasks: []jobs.Task{ + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "./src/notebook.py", + }, + }, + { + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "./src/main.py", + }, + }, + }, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Libraries: []pipelines.PipelineLibrary{ + { + Notebook: &pipelines.NotebookLibrary{ + Path: "./src/pipeline_notebook.py", + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + + diags := bundle.ApplySeq(context.Background(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + require.NoError(t, diags.Error()) + + // Notebook path should be translated (extension stripped) even though file doesn't exist. + assert.Equal(t, "/bundle/src/notebook", b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath) + + // File path should be translated even though file doesn't exist. + assert.Equal(t, "/bundle/src/main.py", b.Config.Resources.Jobs["job"].Tasks[1].SparkPythonTask.PythonFile) + + // Pipeline notebook path should be translated even though file doesn't exist. + assert.Equal(t, "/bundle/src/pipeline_notebook", b.Config.Resources.Pipelines["pipeline"].Libraries[0].Notebook.Path) +} + +func TestTranslatePathsWithSkipLocalFileValidationDirectory(t *testing.T) { + dir := t.TempDir() + // Intentionally do NOT create pipeline_root directory. + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + SkipLocalFileValidation: true, + Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/bundle", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + RootPath: "./pipeline_root", + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) + + diags := bundle.ApplySeq(context.Background(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + require.NoError(t, diags.Error()) + + // Directory path should be translated even though directory doesn't exist. + assert.Equal(t, "/bundle/pipeline_root", b.Config.Resources.Pipelines["pipeline"].RootPath) +} diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index f131bf4f10..8b99088824 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -6,6 +6,7 @@ import ( "fmt" "runtime" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/configsync" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" @@ -46,6 +47,9 @@ Examples: ReadState: true, Build: true, AlwaysPull: true, + InitFunc: func(b *bundle.Bundle) { + b.SkipLocalFileValidation = true + }, }) if err != nil { return err From cf35a7e36ce5e5e262038f27016d7eeb8d7d6dd8 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Sat, 7 Mar 2026 12:10:52 +0100 Subject: [PATCH 3/4] Review feedback --- bundle/bundle.go | 5 +++- bundle/config/mutator/translate_paths.go | 35 ++++++++++++++---------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index 588c573ca4..55a1275b7e 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -151,7 +151,10 @@ type Bundle struct { // SkipLocalFileValidation makes path translation tolerant of missing local files. // When set, TranslatePaths computes workspace paths without verifying files exist. - // Used by config-remote-sync which may run when referenced local files are stale. + // Used by config-remote-sync: a user may modify resource paths remotely (e.g., + // rename a pipeline root folder in the UI), and the updated paths may not exist + // locally. Path translation is still needed to produce fully resolved paths for + // comparison with remote state, but local file validation would incorrectly fail. SkipLocalFileValidation bool // Tagging is used to normalize tag keys and values. diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 8b72f6942a..cd35dfa042 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -87,6 +87,10 @@ type translateContext struct { // skipLocalFileValidation makes path translation tolerant of missing local files. // When set, paths are translated without verifying files exist on the local filesystem. + // This is used by config-remote-sync: a user may rename a resource's root folder + // in the workspace UI, and the updated path may not exist locally. Path translation + // is still needed to produce fully resolved paths for comparison with remote state, + // but local file validation would incorrectly fail. skipLocalFileValidation bool } @@ -182,13 +186,13 @@ func (t *translateContext) rewritePath( } func (t *translateContext) translateNotebookPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { + if t.skipLocalFileValidation { + localRelPathNoExt := strings.TrimSuffix(localRelPath, path.Ext(localRelPath)) + return path.Join(t.remoteRoot, localRelPathNoExt), nil + } + nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath) if errors.Is(err, fs.ErrNotExist) { - if t.skipLocalFileValidation { - localRelPathNoExt := strings.TrimSuffix(localRelPath, path.Ext(localRelPath)) - return path.Join(t.remoteRoot, localRelPathNoExt), nil - } - if path.Ext(localFullPath) != notebook.ExtensionNone { return "", fmt.Errorf("notebook %s not found", literal) } @@ -222,11 +226,12 @@ to contain one of the following file extensions: [%s]`, literal, strings.Join(no } func (t *translateContext) translateFilePath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { + if t.skipLocalFileValidation { + return path.Join(t.remoteRoot, localRelPath), nil + } + nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, localRelPath) if errors.Is(err, fs.ErrNotExist) { - if t.skipLocalFileValidation { - return path.Join(t.remoteRoot, localRelPath), nil - } return "", fmt.Errorf("file %s not found", literal) } if err != nil { @@ -239,11 +244,12 @@ func (t *translateContext) translateFilePath(ctx context.Context, literal, local } func (t *translateContext) translateDirectoryPath(ctx context.Context, literal, localFullPath, localRelPath string) (string, error) { + if t.skipLocalFileValidation { + return path.Join(t.remoteRoot, localRelPath), nil + } + info, err := t.b.SyncRoot.Stat(localRelPath) if err != nil { - if t.skipLocalFileValidation { - return path.Join(t.remoteRoot, localRelPath), nil - } return "", err } if !info.IsDir() { @@ -257,11 +263,12 @@ func (t *translateContext) translateGlobPath(ctx context.Context, literal, local } func (t *translateContext) translateLocalAbsoluteDirectoryPath(ctx context.Context, literal, localFullPath, _ string) (string, error) { + if t.skipLocalFileValidation { + return localFullPath, nil + } + info, err := os.Stat(filepath.FromSlash(localFullPath)) if errors.Is(err, fs.ErrNotExist) { - if t.skipLocalFileValidation { - return localFullPath, nil - } return "", fmt.Errorf("directory %s not found", literal) } if err != nil { From 11cc9361736de473b58ec9d4ec9b785fcf43f751 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 9 Mar 2026 00:20:22 +0100 Subject: [PATCH 4/4] Make lint --- bundle/config/mutator/translate_paths_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 796dcec7a8..8776459c57 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -1067,7 +1067,7 @@ func TestTranslatePathsWithSkipLocalFileValidation(t *testing.T) { bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) - diags := bundle.ApplySeq(context.Background(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) // Notebook path should be translated (extension stripped) even though file doesn't exist. @@ -1107,7 +1107,7 @@ func TestTranslatePathsWithSkipLocalFileValidationDirectory(t *testing.T) { bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "databricks.yml")}}) - diags := bundle.ApplySeq(context.Background(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePaths()) require.NoError(t, diags.Error()) // Directory path should be translated even though directory doesn't exist.