diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2cf027cb00..4ef3e9df24 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,6 +8,8 @@ ### Bundles +* Add declarative bind support for direct deployment engine ([#4630](https://github.com/databricks/cli/pull/4630)). + ### Dependency updates ### API Changes diff --git a/acceptance/bundle/deploy/bind/basic/databricks.yml b/acceptance/bundle/deploy/bind/basic/databricks.yml new file mode 100644 index 0000000000..a6b280560a --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-basic + +resources: + jobs: + foo: + name: test-bind-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/basic/hello.py b/acceptance/bundle/deploy/bind/basic/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/basic/out.test.toml b/acceptance/bundle/deploy/bind/basic/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/basic/output.txt b/acceptance/bundle/deploy/bind/basic/output.txt new file mode 100644 index 0000000000..8e4787d6c0 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/output.txt @@ -0,0 +1,66 @@ + +>>> [CLI] bundle plan +bind jobs.foo (id: [NEW_JOB_ID]) + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_state.py +{ + "state_version": 1, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.jobs.foo": { + "__id__": "[NEW_JOB_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "client": "1" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "test-bind-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "environment_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default/files/hello.py" + }, + "task_key": "my_task" + } + ] + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/basic/script b/acceptance/bundle/deploy/bind/basic/script new file mode 100644 index 0000000000..2252f3a630 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/script @@ -0,0 +1,21 @@ +# Create a job in the workspace +NEW_JOB_ID=$($CLI jobs create --json '{"name": "test-import-job", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id) +add_repl.py $NEW_JOB_ID NEW_JOB_ID + +# Update the databricks.yml with the actual job ID +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID" + +# Run plan - should show import action +trace $CLI bundle plan + +# Deploy with auto-approve +trace $CLI bundle deploy --auto-approve + +# Plan again - should show no changes (skip) +trace $CLI bundle plan + +# Verify state file contains the imported ID +trace print_state.py | contains.py "$NEW_JOB_ID" + +# Cleanup +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/bind-and-update/databricks.yml b/acceptance/bundle/deploy/bind/bind-and-update/databricks.yml new file mode 100644 index 0000000000..3b813772d0 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-update + +resources: + jobs: + foo: + name: updated-job-name + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/bind-and-update/hello.py b/acceptance/bundle/deploy/bind/bind-and-update/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/bind-and-update/out.test.toml b/acceptance/bundle/deploy/bind/bind-and-update/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/bind-and-update/output.txt b/acceptance/bundle/deploy/bind/bind-and-update/output.txt new file mode 100644 index 0000000000..b45e2607b8 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/output.txt @@ -0,0 +1,23 @@ + +>>> [CLI] bundle plan +bind jobs.foo (id: [NEW_JOB_ID]) + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] jobs get [NEW_JOB_ID] +updated-job-name + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-update/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/bind-and-update/script b/acceptance/bundle/deploy/bind/bind-and-update/script new file mode 100644 index 0000000000..b0a27071c8 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/script @@ -0,0 +1,18 @@ +# Create a job in the workspace with a different name +NEW_JOB_ID=$($CLI jobs create --json '{"name": "original-job-name", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id) +add_repl.py $NEW_JOB_ID NEW_JOB_ID + +# Update the databricks.yml with the actual job ID +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID" + +# Run plan - should show import_and_update action (name differs from config) +trace $CLI bundle plan + +# Deploy with auto-approve +trace $CLI bundle deploy --auto-approve + +# Verify the job was updated +trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name + +# Cleanup +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/block-migrate/databricks.yml b/acceptance/bundle/deploy/bind/block-migrate/databricks.yml new file mode 100644 index 0000000000..16cc8b2be6 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-block-migrate + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/block-migrate/hello.py b/acceptance/bundle/deploy/bind/block-migrate/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/block-migrate/out.test.toml b/acceptance/bundle/deploy/bind/block-migrate/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/block-migrate/output.txt b/acceptance/bundle/deploy/bind/block-migrate/output.txt new file mode 100644 index 0000000000..6bc78e4ace --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/output.txt @@ -0,0 +1,3 @@ + +>>> musterr [CLI] bundle deployment migrate +Error: cannot run 'bundle deployment migrate' when bind blocks are defined in the target configuration; bind blocks are only supported with the direct deployment engine diff --git a/acceptance/bundle/deploy/bind/block-migrate/script b/acceptance/bundle/deploy/bind/block-migrate/script new file mode 100644 index 0000000000..64c538ec31 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/script @@ -0,0 +1,2 @@ +# Try to run migration with bind blocks - should fail +trace musterr $CLI bundle deployment migrate diff --git a/acceptance/bundle/deploy/bind/block-migrate/test.toml b/acceptance/bundle/deploy/bind/block-migrate/test.toml new file mode 100644 index 0000000000..680c17c1e0 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/test.toml @@ -0,0 +1,2 @@ +# Migration test does not need engine matrix +[EnvMatrix] diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml new file mode 100644 index 0000000000..cbb7da54af --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bind-delete-conflict + +resources: + jobs: + foo: + name: test-bind-delete-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks_conflict.yml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks_conflict.yml new file mode 100644 index 0000000000..5975881706 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks_conflict.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-delete-conflict + +resources: + jobs: + bar: + name: test-bind-delete-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + bar: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/hello.py b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/out.test.toml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/output.txt b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/output.txt new file mode 100644 index 0000000000..57e63fae48 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-delete-conflict/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> musterr [CLI] bundle plan +Error: bind block for "resources.jobs.bar" has the same ID "[FOO_ID]" as existing resource "resources.jobs.foo"; remove the bind block or the conflicting resource + at targets.default.bind.jobs.bar + +Error: bind validation failed + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-delete-conflict/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/script b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/script new file mode 100644 index 0000000000..6547472f40 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/script @@ -0,0 +1,18 @@ +# Deploy foo to create it in state +trace $CLI bundle deploy --auto-approve + +# Get the job ID from state +JOB_ID=$(read_id.py foo) + +# Switch to a config that renames foo->bar and adds a bind block for bar +# with the same job ID. This creates a conflict: foo is being deleted +# (still in state) while bar is being bound with the same ID. +cp databricks.yml databricks.yml.bak +cp databricks_conflict.yml databricks.yml +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$JOB_ID" + +trace musterr $CLI bundle plan + +# Cleanup: restore original config and destroy +cp databricks.yml.bak databricks.yml +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/test.toml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/test.toml new file mode 100644 index 0000000000..a07a767561 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks", "databricks.yml.bak", "databricks_conflict.yml"] diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks.yml b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks.yml new file mode 100644 index 0000000000..9ae092f190 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bind-duplicate-id + +resources: + jobs: + foo: + name: test-bind-dup-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks_with_bind.yml b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks_with_bind.yml new file mode 100644 index 0000000000..0b3abd007f --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks_with_bind.yml @@ -0,0 +1,34 @@ +bundle: + name: test-bind-duplicate-id + +resources: + jobs: + foo: + name: test-bind-dup-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + bar: + name: test-bind-dup-bar + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + bar: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/hello.py b/acceptance/bundle/deploy/bind/duplicate-bind-id/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/out.test.toml b/acceptance/bundle/deploy/bind/duplicate-bind-id/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/output.txt b/acceptance/bundle/deploy/bind/duplicate-bind-id/output.txt new file mode 100644 index 0000000000..60a188806b --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-duplicate-id/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> musterr [CLI] bundle plan +Error: bind block for "resources.jobs.bar" has the same ID "[FOO_ID]" as existing resource "resources.jobs.foo"; remove the bind block or the conflicting resource + at targets.default.bind.jobs.bar + +Error: bind validation failed + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-duplicate-id/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/script b/acceptance/bundle/deploy/bind/duplicate-bind-id/script new file mode 100644 index 0000000000..57ab78fcf0 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/script @@ -0,0 +1,18 @@ +# Deploy foo to create it in state +trace $CLI bundle deploy --auto-approve + +# Get foo's job ID from state +JOB_ID=$(read_id.py foo) + +# Switch to a config that keeps foo AND adds bar with a bind block +# pointing to foo's ID. This is a conflict: foo is still managed in +# state and bar tries to bind the same resource ID. +cp databricks.yml databricks.yml.bak +cp databricks_with_bind.yml databricks.yml +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$JOB_ID" + +trace musterr $CLI bundle plan + +# Cleanup +cp databricks.yml.bak databricks.yml +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/test.toml b/acceptance/bundle/deploy/bind/duplicate-bind-id/test.toml new file mode 100644 index 0000000000..f70b5c78ff --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks", "databricks.yml.bak", "databricks_with_bind.yml"] diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/databricks.yml b/acceptance/bundle/deploy/bind/invalid-resource-type/databricks.yml new file mode 100644 index 0000000000..84e59cd7f5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-invalid-resource-type + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + foobar: + foo: + id: "123" diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/hello.py b/acceptance/bundle/deploy/bind/invalid-resource-type/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/out.test.toml b/acceptance/bundle/deploy/bind/invalid-resource-type/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/output.txt b/acceptance/bundle/deploy/bind/invalid-resource-type/output.txt new file mode 100644 index 0000000000..d06b74c8f1 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/output.txt @@ -0,0 +1,7 @@ + +>>> musterr [CLI] bundle plan +Error: bind block references undefined resource "resources.foobar.foo"; define it in the resources section or remove the bind block + at targets.default.bind.foobar.foo + +Error: bind validation failed + diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/script b/acceptance/bundle/deploy/bind/invalid-resource-type/script new file mode 100644 index 0000000000..9d9604578f --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/script @@ -0,0 +1 @@ +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/databricks.yml b/acceptance/bundle/deploy/bind/orphaned-bind/databricks.yml new file mode 100644 index 0000000000..26d0272b26 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-orphaned + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + bar: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/hello.py b/acceptance/bundle/deploy/bind/orphaned-bind/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/out.test.toml b/acceptance/bundle/deploy/bind/orphaned-bind/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/output.txt b/acceptance/bundle/deploy/bind/orphaned-bind/output.txt new file mode 100644 index 0000000000..6ae6816b73 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/output.txt @@ -0,0 +1,7 @@ + +>>> musterr [CLI] bundle plan +Error: bind block references undefined resource "resources.jobs.bar"; define it in the resources section or remove the bind block + at targets.default.bind.jobs.bar + +Error: bind validation failed + diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/script b/acceptance/bundle/deploy/bind/orphaned-bind/script new file mode 100644 index 0000000000..aeae9c12fa --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/script @@ -0,0 +1,2 @@ +# Import block references jobs.bar but only jobs.foo exists in resources +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/databricks.yml b/acceptance/bundle/deploy/bind/recreate-blocked/databricks.yml new file mode 100644 index 0000000000..ddad62da6a --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bind-recreate + +resources: + pipelines: + foo: + name: test-pipeline + storage: /new/storage/path + libraries: + - notebook: + path: ./nb.sql + +targets: + default: + bind: + pipelines: + foo: + id: "PLACEHOLDER_PIPELINE_ID" diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/nb.sql b/acceptance/bundle/deploy/bind/recreate-blocked/nb.sql new file mode 100644 index 0000000000..199ff50788 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/out.test.toml b/acceptance/bundle/deploy/bind/recreate-blocked/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/output.txt b/acceptance/bundle/deploy/bind/recreate-blocked/output.txt new file mode 100644 index 0000000000..4a39ec4441 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/output.txt @@ -0,0 +1,6 @@ + +>>> musterr [CLI] bundle plan +Error: cannot plan resources.pipelines.foo: cannot recreate resource with bind block; this would destroy the existing workspace resource. Remove the bind block to allow recreation + +Error: planning failed + diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/script b/acceptance/bundle/deploy/bind/recreate-blocked/script new file mode 100644 index 0000000000..f28f6b94ac --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/script @@ -0,0 +1,9 @@ +# Create a pipeline with a different storage path +NEW_PIPELINE_ID=$($CLI pipelines create --json '{"name": "test-pipeline", "storage": "/old/storage/path", "allow_duplicate_names": true, "libraries": [{"notebook": {"path": "/Workspace/test"}}]}' | jq -r .pipeline_id) +add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID + +# Update the databricks.yml with the actual pipeline ID +update_file.py databricks.yml 'PLACEHOLDER_PIPELINE_ID' "$NEW_PIPELINE_ID" + +# Run plan - should fail because changing storage requires recreate which is blocked for binds +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/resource-not-found/databricks.yml b/acceptance/bundle/deploy/bind/resource-not-found/databricks.yml new file mode 100644 index 0000000000..e02258511d --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-not-found + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "999999999" diff --git a/acceptance/bundle/deploy/bind/resource-not-found/hello.py b/acceptance/bundle/deploy/bind/resource-not-found/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/resource-not-found/out.test.toml b/acceptance/bundle/deploy/bind/resource-not-found/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/resource-not-found/output.txt b/acceptance/bundle/deploy/bind/resource-not-found/output.txt new file mode 100644 index 0000000000..958309e7d8 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/output.txt @@ -0,0 +1,6 @@ + +>>> musterr [CLI] bundle plan +Error: cannot plan resources.jobs.foo: resource with ID "[NUMID]" does not exist in workspace + +Error: planning failed + diff --git a/acceptance/bundle/deploy/bind/resource-not-found/script b/acceptance/bundle/deploy/bind/resource-not-found/script new file mode 100644 index 0000000000..6226cb8591 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/script @@ -0,0 +1,2 @@ +# Try to plan with a non-existent job ID - should fail +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/databricks.yml b/acceptance/bundle/deploy/bind/terraform-with-bind/databricks.yml new file mode 100644 index 0000000000..9845092834 --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-terraform + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/hello.py b/acceptance/bundle/deploy/bind/terraform-with-bind/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/out.test.toml b/acceptance/bundle/deploy/bind/terraform-with-bind/out.test.toml new file mode 100644 index 0000000000..a9f28de48a --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/output.txt b/acceptance/bundle/deploy/bind/terraform-with-bind/output.txt new file mode 100644 index 0000000000..62c0d2b6c5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/output.txt @@ -0,0 +1,4 @@ + +>>> musterr [CLI] bundle plan +Error: bind blocks in the target configuration are only supported with the direct deployment engine; set DATABRICKS_BUNDLE_ENGINE=direct or remove the bind blocks + diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/script b/acceptance/bundle/deploy/bind/terraform-with-bind/script new file mode 100644 index 0000000000..4b3c7a8ecd --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/script @@ -0,0 +1,2 @@ +# Import blocks should error with terraform engine +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/test.toml b/acceptance/bundle/deploy/bind/terraform-with-bind/test.toml new file mode 100644 index 0000000000..272dde4b9c --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/test.toml @@ -0,0 +1,3 @@ +# Override engine to terraform to test import block rejection +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/bind/test.toml b/acceptance/bundle/deploy/bind/test.toml new file mode 100644 index 0000000000..931833f6cc --- /dev/null +++ b/acceptance/bundle/deploy/bind/test.toml @@ -0,0 +1,5 @@ +Cloud = true +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/top-level-bind/databricks.yml b/acceptance/bundle/deploy/bind/top-level-bind/databricks.yml new file mode 100644 index 0000000000..6783311697 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: test-bind-top-level + +bind: + jobs: + foo: + id: "123" + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py diff --git a/acceptance/bundle/deploy/bind/top-level-bind/hello.py b/acceptance/bundle/deploy/bind/top-level-bind/hello.py new file mode 100644 index 0000000000..11b15b1a45 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/top-level-bind/out.test.toml b/acceptance/bundle/deploy/bind/top-level-bind/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/top-level-bind/output.txt b/acceptance/bundle/deploy/bind/top-level-bind/output.txt new file mode 100644 index 0000000000..ee8fd54ee7 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle validate +Warning: unknown field: bind + in databricks.yml:4:1 + +Name: test-bind-top-level +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bind-top-level/default + +Found 1 warning diff --git a/acceptance/bundle/deploy/bind/top-level-bind/script b/acceptance/bundle/deploy/bind/top-level-bind/script new file mode 100644 index 0000000000..5350876150 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/bundle/deploy/bind/top-level-bind/test.toml b/acceptance/bundle/deploy/bind/top-level-bind/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/bundle/config/bind.go b/bundle/config/bind.go new file mode 100644 index 0000000000..923b7830a1 --- /dev/null +++ b/bundle/config/bind.go @@ -0,0 +1,42 @@ +package config + +// BindResource represents a single resource to bind with its workspace ID. +type BindResource struct { + ID string `json:"id"` +} + +// Bind defines resources to bind at the target level. +// Resources listed here will be bound to the bundle at deploy time. +// This field is only valid for the direct deployment engine. +// +// The outer map key is the resource type (e.g., "jobs", "pipelines"), +// and the inner map key is the resource name in the bundle configuration. +type Bind map[string]map[string]BindResource + +// GetBindID returns the bind ID for a given resource type and name. +// Returns empty string if no bind is defined for the resource. +func (i Bind) GetBindID(resourceType, resourceName string) string { + if r, ok := i[resourceType][resourceName]; ok { + return r.ID + } + return "" +} + +// ForEach calls fn for each bind entry in the configuration. +func (i Bind) ForEach(fn func(resourceType, resourceName, bindID string)) { + for resourceType, resources := range i { + for name, r := range resources { + fn(resourceType, name, r.ID) + } + } +} + +// IsEmpty returns true if no binds are defined. +func (i Bind) IsEmpty() bool { + for _, resources := range i { + if len(resources) > 0 { + return false + } + } + return true +} diff --git a/bundle/config/target.go b/bundle/config/target.go index fae9c940b3..a58e5191ee 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -69,6 +69,11 @@ type Target struct { Sync *Sync `json:"sync,omitempty"` Permissions []resources.Permission `json:"permissions,omitempty"` + + // Bind specifies existing workspace resources to bind into bundle management. + // Resources listed here will be bound to the bundle at deploy time. + // This field is only valid for the direct deployment engine. + Bind Bind `json:"bind,omitempty"` } const ( diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 1e27da9150..ee1421c4b8 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -139,7 +139,7 @@ func DetectChanges(ctx context.Context, b *bundle.Bundle, engine engine.EngineTy _, statePath = b.StateFilenameConfigSnapshot(ctx) } - plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, statePath) + plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, statePath, nil) if err != nil { return nil, fmt.Errorf("failed to calculate plan: %w", err) } diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index e8e1427956..87e144dfa0 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -28,36 +28,45 @@ type ActionType string // If case of several options, action with highest severity wins. // Note, Create/Delete are handled explicitly and never compared. const ( - Undefined ActionType = "" - Skip ActionType = "skip" - Resize ActionType = "resize" - Update ActionType = "update" - UpdateWithID ActionType = "update_id" - Create ActionType = "create" - Recreate ActionType = "recreate" - Delete ActionType = "delete" + Undefined ActionType = "" + Skip ActionType = "skip" + Resize ActionType = "resize" + Update ActionType = "update" + UpdateWithID ActionType = "update_id" + Bind ActionType = "bind" + BindAndUpdate ActionType = "bind_and_update" + Create ActionType = "create" + Recreate ActionType = "recreate" + Delete ActionType = "delete" ) var actionOrder = map[ActionType]int{ - Undefined: 0, - Skip: 1, - Resize: 2, - Update: 3, - UpdateWithID: 4, - Create: 5, - Recreate: 6, - Delete: 7, + Undefined: 0, + Skip: 1, + Resize: 2, + Update: 3, + UpdateWithID: 4, + Bind: 5, + BindAndUpdate: 6, + Create: 7, + Recreate: 8, + Delete: 9, } func (a ActionType) KeepsID() bool { switch a { - case Create, UpdateWithID, Recreate, Delete: + case Create, UpdateWithID, Recreate, Delete, Bind, BindAndUpdate: return false default: return true } } +// IsBind returns true if the action is a bind action. +func (a ActionType) IsBind() bool { + return a == Bind || a == BindAndUpdate +} + // StringShort short version of action string, without suffix func (a ActionType) StringShort() string { items := strings.SplitN(string(a), "_", 2) diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 22f7db34c7..b9c7736596 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -74,6 +74,7 @@ func LoadPlanFromFile(path string) (*Plan, error) { type PlanEntry struct { ID string `json:"id,omitempty"` + BindID string `json:"bind_id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 04bb7054a2..c73104b402 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -56,6 +56,40 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } } +// DeclarativeBind handles binding an existing workspace resource into the bundle state. +// For Bind action, it just saves the state with the bind ID. +// For BindAndUpdate action, it also applies config changes to the resource. +func (d *DeploymentUnit) DeclarativeBind(ctx context.Context, db *dstate.DeploymentState, bindID string, newState any, actionType deployplan.ActionType, changes deployplan.Changes) error { + if actionType == deployplan.BindAndUpdate { + // Apply updates to the bound resource + if !d.Adapter.HasDoUpdate() { + return fmt.Errorf("internal error: DoUpdate not implemented for resource %s", d.ResourceKey) + } + + remoteState, err := d.Adapter.DoUpdate(ctx, bindID, newState, changes) + if err != nil { + return fmt.Errorf("updating bound resource id=%s: %w", bindID, err) + } + + err = d.SetRemoteState(remoteState) + if err != nil { + return err + } + + log.Infof(ctx, "Bound and updated %s id=%s", d.ResourceKey, bindID) + } else { + log.Infof(ctx, "Bound %s id=%s", d.ResourceKey, bindID) + } + + // Save state with the bound ID + err := db.SaveState(d.ResourceKey, bindID, newState, d.DependsOn) + if err != nil { + return fmt.Errorf("saving state id=%s: %w", bindID, err) + } + + return nil +} + func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { newID, remoteState, err := d.Adapter.DoCreate(ctx, newState) if err != nil { diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index b476319ed7..f16a5b2e0f 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -105,7 +105,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - plan, err := b.CalculatePlan(ctx, client, configRoot, tmpStatePath) + plan, err := b.CalculatePlan(ctx, client, configRoot, tmpStatePath, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -146,7 +146,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - plan, err = b.CalculatePlan(ctx, client, configRoot, tmpStatePath) + plan, err = b.CalculatePlan(ctx, client, configRoot, tmpStatePath, nil) if err != nil { os.Remove(tmpStatePath) return nil, err diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 4a0b9359ee..c5261b3ed8 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -120,6 +120,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } err = b.StateDB.SaveState(resourceKey, dbentry.ID, sv.Value, entry.DependsOn) + } else if action.IsBind() { + // Handle bind actions + err = d.DeclarativeBind(ctx, &b.StateDB, entry.BindID, sv.Value, action, entry.Changes) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry.Changes) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index cd2d8a18ab..6608eeca8f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" @@ -109,7 +110,7 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } -func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string) (*deployplan.Plan, error) { +func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string, bindConfig config.Bind) (*deployplan.Plan, error) { err := b.StateDB.Open(statePath) if err != nil { return nil, fmt.Errorf("reading state from %s: %w", statePath, err) @@ -120,11 +121,56 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, err } - plan, err := b.makePlan(ctx, configRoot, &b.StateDB.Data) + plan, err := b.makePlan(ctx, configRoot, &b.StateDB.Data, bindConfig) if err != nil { return nil, fmt.Errorf("reading config: %w", err) } + // Validate bind entries when a bind config is provided. + if !bindConfig.IsEmpty() { + var hasBindErrors bool + targetName := configRoot.Bundle.Target + + // Validate all bind entries reference resources defined in config. + bindConfig.ForEach(func(resourceType, resourceName, bindID string) { + key := "resources." + resourceType + "." + resourceName + if _, ok := plan.Plan[key]; !ok { + bindPath := dyn.NewPath(dyn.Key("targets"), dyn.Key(targetName), dyn.Key("bind"), dyn.Key(resourceType), dyn.Key(resourceName)) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("bind block references undefined resource %q; define it in the resources section or remove the bind block", key), + Locations: configRoot.GetLocations(bindPath.String()), + Paths: []dyn.Path{bindPath}, + }) + hasBindErrors = true + } + }) + + // Validate that no bind ID conflicts with an existing resource in state. + // Deletes, recreates, and update_ids are not allowed when a bind block + // references the same resource ID under a different resource key. + bindConfig.ForEach(func(resourceType, resourceName, bindID string) { + bindKey := "resources." + resourceType + "." + resourceName + for stateKey, stateEntry := range b.StateDB.Data.State { + if stateKey == bindKey || stateEntry.ID != bindID { + continue + } + bindPath := dyn.NewPath(dyn.Key("targets"), dyn.Key(targetName), dyn.Key("bind"), dyn.Key(resourceType), dyn.Key(resourceName)) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("bind block for %q has the same ID %q as existing resource %q; remove the bind block or the conflicting resource", bindKey, bindID, stateKey), + Locations: configRoot.GetLocations(bindPath.String()), + Paths: []dyn.Path{bindPath}, + }) + hasBindErrors = true + } + }) + + if hasBindErrors { + return nil, errors.New("bind validation failed") + } + } + b.Plan = plan g, err := makeGraph(plan) @@ -196,6 +242,22 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) + + // Handle bind block: if BindID is set, this resource should be bound + if entry.BindID != "" { + if hasEntry { + // Resource is already in state - check if IDs match + if dbentry.ID != entry.BindID { + logdiag.LogError(ctx, fmt.Errorf("%s: resource already bound to ID %q, cannot bind as %q; remove the bind block or unbind the existing resource", errorPrefix, dbentry.ID, entry.BindID)) + return false + } + // IDs match - proceed with normal planning (resource was previously bound) + } else { + // Not in state - this is a new bind + return b.handleBindPlan(ctx, resourceKey, entry, adapter, errorPrefix) + } + } + if !hasEntry { entry.Action = deployplan.Create return true @@ -709,7 +771,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st return true } -func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root, db *dstate.Database) (*deployplan.Plan, error) { +func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root, db *dstate.Database, bindConfig config.Bind) (*deployplan.Plan, error) { p := deployplan.NewPlanDirect() // Copy state metadata to plan for validation during apply @@ -871,9 +933,14 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) } + // Check if this resource has a bind block defined + resourceType, resourceName := getResourceTypeAndName(node) + bindID := bindConfig.GetBindID(resourceType, resourceName) + e := deployplan.PlanEntry{ DependsOn: dependsOn, NewState: newStateJSON, + BindID: bindID, } p.Plan[node] = &e @@ -944,3 +1011,93 @@ func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Ada return adapter, nil } + +// handleBindPlan handles planning for resources that should be bound from the workspace. +// This is called when a resource has a bind block defined and is not yet in the state. +func (b *DeploymentBundle) handleBindPlan(ctx context.Context, resourceKey string, entry *deployplan.PlanEntry, adapter *dresources.Adapter, errorPrefix string) bool { + bindID := entry.BindID + + // Read the remote resource to verify it exists and get current state + remoteState, err := adapter.DoRead(ctx, bindID) + if err != nil { + if isResourceGone(err) { + logdiag.LogError(ctx, fmt.Errorf("%s: resource with ID %q does not exist in workspace", errorPrefix, bindID)) + } else { + logdiag.LogError(ctx, fmt.Errorf("%s: reading remote resource id=%q: %w", errorPrefix, bindID, err)) + } + return false + } + + entry.RemoteState = remoteState + b.RemoteStateCache.Store(resourceKey, remoteState) + + // Get the new state config + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no state cache entry", errorPrefix)) + return false + } + + // Compare remote state with config to determine if update needed + remoteStateComparable, err := adapter.RemapState(remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: interpreting remote state: %w", errorPrefix, err)) + return false + } + + remoteDiff, err := structdiff.GetStructDiff(remoteStateComparable, sv.Value, adapter.KeyedSlices()) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: diffing remote state: %w", errorPrefix, err)) + return false + } + + // For binds, there's no "saved state" so we compare remote directly with config + entry.Changes, err = prepareChanges(ctx, adapter, nil, remoteDiff, nil, remoteStateComparable) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + + err = addPerFieldActions(ctx, adapter, entry.Changes, remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) + return false + } + + // Determine action based on changes + maxAction := getMaxAction(entry.Changes) + + // Block recreate action for bound resources + if maxAction == deployplan.Recreate { + logdiag.LogError(ctx, fmt.Errorf("%s: cannot recreate resource with bind block; this would destroy the existing workspace resource. Remove the bind block to allow recreation", errorPrefix)) + return false + } + + if maxAction == deployplan.Skip || maxAction == deployplan.Undefined { + entry.Action = deployplan.Bind + } else { + entry.Action = deployplan.BindAndUpdate + } + + return true +} + +// getResourceTypeAndName extracts the resource type and name from a resource key. +// For example, "resources.jobs.my_job" returns ("jobs", "my_job"). +// For child resources like "resources.jobs.my_job.permissions", returns ("jobs.permissions", "my_job"). +func getResourceTypeAndName(resourceKey string) (resourceType, resourceName string) { + dp, err := dyn.NewPathFromString(resourceKey) + if err != nil || len(dp) < 3 { + return "", "" + } + + resourceType = dp[1].Key() + resourceName = dp[2].Key() + + // Handle child resources (permissions, grants) + if len(dp) >= 4 && (dp[3].Key() == "permissions" || dp[3].Key() == "grants") { + resourceType = dp[1].Key() + "." + dp[3].Key() + } + + return resourceType, resourceName +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 6464e26cac..547e1c3426 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -23,6 +23,10 @@ github.com/databricks/cli/bundle/config.ArtifactFile: "source": "description": |- Required. The artifact source file. +github.com/databricks/cli/bundle/config.BindResource: + "id": + "description": |- + The ID of the existing workspace resource to bind. github.com/databricks/cli/bundle/config.Bundle: "cluster_id": "description": |- @@ -363,6 +367,9 @@ github.com/databricks/cli/bundle/config.Target: "artifacts": "description": |- The artifacts to include in the target deployment. + "bind": + "description": |- + The existing workspace resources to bind into bundle management for this target. "bundle": "description": |- The bundle attributes when deploying to this target. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 35c4161eae..1afdf046ed 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -213,9 +213,19 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *deployplan.Plan { + // Validate bind blocks are only used with the direct deployment engine. + if !engine.IsDirect() && b.Target != nil && !b.Target.Bind.IsEmpty() { + logdiag.LogError(ctx, errors.New("bind blocks in the target configuration are only supported with the direct deployment engine; set DATABRICKS_BUNDLE_ENGINE=direct or remove the bind blocks")) + return nil + } + if engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath) + var bindConfig config.Bind + if b.Target != nil { + bindConfig = b.Target.Bind + } + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath, bindConfig) if err != nil { logdiag.LogError(ctx, err) return nil diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 6c6657c8ce..68dca4f154 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -158,7 +158,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath) + plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath, nil) if err != nil { logdiag.LogError(ctx, err) return diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 0a25ac5a95..fc7c6bf183 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2490,6 +2490,27 @@ "config.ArtifactType": { "type": "string" }, + "config.BindResource": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "description": "The ID of the existing workspace resource to bind.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "config.Bundle": { "oneOf": [ { @@ -2906,6 +2927,10 @@ "description": "The artifacts to include in the target deployment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" }, + "bind": { + "description": "The existing workspace resources to bind into bundle management for this target.", + "$ref": "#/$defs/map/map/github.com/databricks/cli/bundle/config.BindResource" + }, "bundle": { "description": "The bundle attributes when deploying to this target.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" @@ -10623,6 +10648,20 @@ } ] }, + "config.BindResource": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.BindResource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "config.Command": { "oneOf": [ { @@ -10669,6 +10708,30 @@ } } }, + "map": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config.BindResource": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.BindResource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + } + }, "string": { "oneOf": [ { diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index a71fa241cb..b1eddafc2f 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -152,7 +152,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return diag.FromErr(fmt.Errorf("failed to create uninterpolated config: %w", err)) } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &uninterpolatedConfig, snapshotPath) + plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &uninterpolatedConfig, snapshotPath, nil) if err != nil { return diag.FromErr(err) } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 6f67f8ce43..d2a0ff4629 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -154,6 +154,11 @@ WARNING: Both direct deployment engine and this command are experimental and not } ctx := cmd.Context() + // Check for bind blocks - migration is not allowed with bind blocks defined + if b.Target != nil && !b.Target.Bind.IsEmpty() { + return errors.New("cannot run 'bundle deployment migrate' when bind blocks are defined in the target configuration; bind blocks are only supported with the direct deployment engine") + } + if stateDesc.Lineage == "" { // TODO: mention bundle.engine once it's there cmdio.LogString(ctx, `Error: This command migrates the existing Terraform state file (terraform.tfstate) to a direct deployment state file (resources.json). However, no existing local or remote state was found. @@ -231,7 +236,7 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va } }() - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, tempStatePath) + plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, tempStatePath, nil) if err != nil { return err } diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index e3dd63929e..a61e939e87 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -72,6 +72,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, updateCount := 0 deleteCount := 0 unchangedCount := 0 + bindCount := 0 for _, change := range plan.GetActions() { switch change.ActionType { @@ -87,6 +88,11 @@ It is useful for previewing changes before running 'bundle deploy'.`, createCount++ case deployplan.Skip, deployplan.Undefined: unchangedCount++ + case deployplan.Bind: + bindCount++ + case deployplan.BindAndUpdate: + bindCount++ + updateCount++ } } @@ -95,7 +101,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, switch root.OutputType(cmd) { case flags.OutputText: // Print summary line and actions to stdout - totalChanges := createCount + updateCount + deleteCount + totalChanges := createCount + updateCount + deleteCount + bindCount if totalChanges > 0 { // Print all actions in the order they were processed for _, action := range plan.GetActions() { @@ -103,12 +109,22 @@ It is useful for previewing changes before running 'bundle deploy'.`, continue } key := strings.TrimPrefix(action.ResourceKey, "resources.") - fmt.Fprintf(out, "%s %s\n", action.ActionType.StringShort(), key) + // For bind actions, include the bind ID + if action.ActionType.IsBind() { + entry := plan.Plan[action.ResourceKey] + fmt.Fprintf(out, "%s %s (id: %s)\n", action.ActionType.StringShort(), key, entry.BindID) + } else { + fmt.Fprintf(out, "%s %s\n", action.ActionType.StringShort(), key) + } } fmt.Fprintln(out) } // Note, this string should not be changed, "bundle deployment migrate" depends on this format: - fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged\n", createCount, updateCount, deleteCount, unchangedCount) + if bindCount > 0 { + fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged, %d to bind\n", createCount, updateCount, deleteCount, unchangedCount, bindCount) + } else { + fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged\n", createCount, updateCount, deleteCount, unchangedCount) + } case flags.OutputJSON: buf, err := json.MarshalIndent(plan, "", " ") if err != nil {