diff --git a/acceptance/bundle/lifecycle/started/databricks.yml b/acceptance/bundle/lifecycle/started/databricks.yml new file mode 100644 index 0000000000..313b1cb8de --- /dev/null +++ b/acceptance/bundle/lifecycle/started/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test_lifecycle_started + +resources: + jobs: + my_job: + name: my_job + lifecycle: + started: true diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/lifecycle/started/output.txt b/acceptance/bundle/lifecycle/started/output.txt new file mode 100644 index 0000000000..0ffc8f1d18 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/output.txt @@ -0,0 +1,5 @@ +Error: lifecycle.started is not supported for resources.jobs.my_job; it is only supported for apps, clusters, and sql_warehouses + in databricks.yml:9:18 + + +Exit code: 1 diff --git a/acceptance/bundle/lifecycle/started/script b/acceptance/bundle/lifecycle/started/script new file mode 100644 index 0000000000..4fbc2b517c --- /dev/null +++ b/acceptance/bundle/lifecycle/started/script @@ -0,0 +1 @@ +errcode $CLI bundle plan diff --git a/acceptance/bundle/lifecycle/started/test.toml b/acceptance/bundle/lifecycle/started/test.toml new file mode 100644 index 0000000000..d1240963e0 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/test.toml @@ -0,0 +1,7 @@ +Local = true +Cloud = false + +Ignore = [".databricks"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index f401945572..c817a477b0 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -34,6 +34,7 @@ resources.alerts.*.file_path string INPUT resources.alerts.*.id string ALL resources.alerts.*.lifecycle resources.Lifecycle INPUT resources.alerts.*.lifecycle.prevent_destroy bool INPUT +resources.alerts.*.lifecycle.started *bool INPUT resources.alerts.*.lifecycle_state sql.AlertLifecycleState ALL resources.alerts.*.modified_status string INPUT resources.alerts.*.owner_user_name string ALL @@ -100,14 +101,14 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.active_instances int ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL -resources.apps.*.config *resources.AppConfig INPUT -resources.apps.*.config.command []string INPUT -resources.apps.*.config.command[*] string INPUT -resources.apps.*.config.env []resources.AppEnvVar INPUT -resources.apps.*.config.env[*] resources.AppEnvVar INPUT -resources.apps.*.config.env[*].name string INPUT -resources.apps.*.config.env[*].value string INPUT -resources.apps.*.config.env[*].value_from string INPUT +resources.apps.*.config *resources.AppConfig INPUT STATE +resources.apps.*.config.command []string INPUT STATE +resources.apps.*.config.command[*] string INPUT STATE +resources.apps.*.config.env []resources.AppEnvVar INPUT STATE +resources.apps.*.config.env[*] resources.AppEnvVar INPUT STATE +resources.apps.*.config.env[*].name string INPUT STATE +resources.apps.*.config.env[*].value string INPUT STATE +resources.apps.*.config.env[*].value_from string INPUT STATE resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -119,18 +120,19 @@ resources.apps.*.effective_user_api_scopes[*] string ALL resources.apps.*.git_repository *apps.GitRepository ALL resources.apps.*.git_repository.provider string ALL resources.apps.*.git_repository.url string ALL -resources.apps.*.git_source *apps.GitSource INPUT -resources.apps.*.git_source.branch string INPUT -resources.apps.*.git_source.commit string INPUT -resources.apps.*.git_source.git_repository *apps.GitRepository INPUT -resources.apps.*.git_source.git_repository.provider string INPUT -resources.apps.*.git_source.git_repository.url string INPUT -resources.apps.*.git_source.resolved_commit string INPUT -resources.apps.*.git_source.source_code_path string INPUT -resources.apps.*.git_source.tag string INPUT +resources.apps.*.git_source *apps.GitSource INPUT STATE +resources.apps.*.git_source.branch string INPUT STATE +resources.apps.*.git_source.commit string INPUT STATE +resources.apps.*.git_source.git_repository *apps.GitRepository INPUT STATE +resources.apps.*.git_source.git_repository.provider string INPUT STATE +resources.apps.*.git_source.git_repository.url string INPUT STATE +resources.apps.*.git_source.resolved_commit string INPUT STATE +resources.apps.*.git_source.source_code_path string INPUT STATE +resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT +resources.apps.*.lifecycle.started *bool INPUT resources.apps.*.modified_status string INPUT resources.apps.*.name string ALL resources.apps.*.oauth2_app_client_id string ALL @@ -206,8 +208,9 @@ resources.apps.*.resources[*].uc_securable.securable_type apps.AppResourceUcSecu resources.apps.*.service_principal_client_id string ALL resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL -resources.apps.*.source_code_path string INPUT +resources.apps.*.source_code_path string INPUT STATE resources.apps.*.space string ALL +resources.apps.*.started bool STATE resources.apps.*.update_time string ALL resources.apps.*.updater string ALL resources.apps.*.url string ALL @@ -242,6 +245,7 @@ resources.catalogs.*.id string INPUT resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.catalogs.*.lifecycle.started *bool INPUT resources.catalogs.*.metastore_id string REMOTE resources.catalogs.*.modified_status string INPUT resources.catalogs.*.name string ALL @@ -387,6 +391,7 @@ resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE resources.clusters.*.lifecycle resources.Lifecycle INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT +resources.clusters.*.lifecycle.started *bool INPUT resources.clusters.*.modified_status string INPUT resources.clusters.*.node_type_id string ALL resources.clusters.*.num_workers int ALL @@ -517,6 +522,7 @@ resources.clusters.*.spec.workload_type.clients.notebooks bool REMOTE resources.clusters.*.ssh_public_keys []string ALL resources.clusters.*.ssh_public_keys[*] string ALL resources.clusters.*.start_time int64 REMOTE +resources.clusters.*.started bool STATE resources.clusters.*.state compute.State REMOTE resources.clusters.*.state_message string REMOTE resources.clusters.*.terminated_time int64 REMOTE @@ -553,6 +559,7 @@ resources.dashboards.*.file_path string INPUT resources.dashboards.*.id string INPUT resources.dashboards.*.lifecycle resources.Lifecycle INPUT resources.dashboards.*.lifecycle.prevent_destroy bool INPUT +resources.dashboards.*.lifecycle.started *bool INPUT resources.dashboards.*.lifecycle_state dashboards.LifecycleState ALL resources.dashboards.*.modified_status string INPUT resources.dashboards.*.parent_path string ALL @@ -581,6 +588,7 @@ resources.database_catalogs.*.database_name string ALL resources.database_catalogs.*.id string INPUT resources.database_catalogs.*.lifecycle resources.Lifecycle INPUT resources.database_catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.database_catalogs.*.lifecycle.started *bool INPUT resources.database_catalogs.*.modified_status string INPUT resources.database_catalogs.*.name string ALL resources.database_catalogs.*.uid string ALL @@ -615,6 +623,7 @@ resources.database_instances.*.enable_readable_secondaries bool ALL resources.database_instances.*.id string INPUT resources.database_instances.*.lifecycle resources.Lifecycle INPUT resources.database_instances.*.lifecycle.prevent_destroy bool INPUT +resources.database_instances.*.lifecycle.started *bool INPUT resources.database_instances.*.modified_status string INPUT resources.database_instances.*.name string ALL resources.database_instances.*.node_count int ALL @@ -653,6 +662,7 @@ resources.experiments.*.id string INPUT resources.experiments.*.last_update_time int64 REMOTE resources.experiments.*.lifecycle resources.Lifecycle INPUT resources.experiments.*.lifecycle.prevent_destroy bool INPUT +resources.experiments.*.lifecycle.started *bool INPUT resources.experiments.*.lifecycle_stage string REMOTE resources.experiments.*.modified_status string INPUT resources.experiments.*.name string ALL @@ -719,6 +729,7 @@ resources.external_locations.*.id string INPUT resources.external_locations.*.isolation_mode catalog.IsolationMode REMOTE resources.external_locations.*.lifecycle resources.Lifecycle INPUT resources.external_locations.*.lifecycle.prevent_destroy bool INPUT +resources.external_locations.*.lifecycle.started *bool INPUT resources.external_locations.*.metastore_id string REMOTE resources.external_locations.*.modified_status string INPUT resources.external_locations.*.name string ALL @@ -908,6 +919,7 @@ resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks boo resources.jobs.*.job_id int64 REMOTE resources.jobs.*.lifecycle resources.Lifecycle INPUT resources.jobs.*.lifecycle.prevent_destroy bool INPUT +resources.jobs.*.lifecycle.started *bool INPUT resources.jobs.*.max_concurrent_runs int ALL resources.jobs.*.modified_status string INPUT resources.jobs.*.name string ALL @@ -2094,6 +2106,7 @@ resources.model_serving_endpoints.*.endpoint_id string REMOTE resources.model_serving_endpoints.*.id string INPUT resources.model_serving_endpoints.*.lifecycle resources.Lifecycle INPUT resources.model_serving_endpoints.*.lifecycle.prevent_destroy bool INPUT +resources.model_serving_endpoints.*.lifecycle.started *bool INPUT resources.model_serving_endpoints.*.modified_status string INPUT resources.model_serving_endpoints.*.name string INPUT STATE resources.model_serving_endpoints.*.permissions []resources.ModelServingEndpointPermission INPUT @@ -2144,6 +2157,7 @@ resources.models.*.latest_versions[*].user_id string REMOTE resources.models.*.latest_versions[*].version string REMOTE resources.models.*.lifecycle resources.Lifecycle INPUT resources.models.*.lifecycle.prevent_destroy bool INPUT +resources.models.*.lifecycle.started *bool INPUT resources.models.*.modified_status string INPUT resources.models.*.name string ALL resources.models.*.permission_level ml.PermissionLevel REMOTE @@ -2462,6 +2476,7 @@ resources.pipelines.*.libraries[*].notebook.path string ALL resources.pipelines.*.libraries[*].whl string ALL resources.pipelines.*.lifecycle resources.Lifecycle INPUT resources.pipelines.*.lifecycle.prevent_destroy bool INPUT +resources.pipelines.*.lifecycle.started *bool INPUT resources.pipelines.*.modified_status string INPUT resources.pipelines.*.name string ALL resources.pipelines.*.notifications []pipelines.Notifications ALL @@ -2516,6 +2531,7 @@ resources.postgres_branches.*.id string INPUT resources.postgres_branches.*.is_protected bool INPUT STATE resources.postgres_branches.*.lifecycle resources.Lifecycle INPUT resources.postgres_branches.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_branches.*.lifecycle.started *bool INPUT resources.postgres_branches.*.modified_status string INPUT resources.postgres_branches.*.name string REMOTE resources.postgres_branches.*.no_expiry bool INPUT STATE @@ -2559,6 +2575,7 @@ resources.postgres_endpoints.*.group.min int INPUT STATE resources.postgres_endpoints.*.id string INPUT resources.postgres_endpoints.*.lifecycle resources.Lifecycle INPUT resources.postgres_endpoints.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_endpoints.*.lifecycle.started *bool INPUT resources.postgres_endpoints.*.modified_status string INPUT resources.postgres_endpoints.*.name string REMOTE resources.postgres_endpoints.*.no_suspension bool INPUT STATE @@ -2625,6 +2642,7 @@ resources.postgres_projects.*.initial_endpoint_spec.group.max int REMOTE resources.postgres_projects.*.initial_endpoint_spec.group.min int REMOTE resources.postgres_projects.*.lifecycle resources.Lifecycle INPUT resources.postgres_projects.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_projects.*.lifecycle.started *bool INPUT resources.postgres_projects.*.modified_status string INPUT resources.postgres_projects.*.name string REMOTE resources.postgres_projects.*.permissions []resources.DatabaseProjectPermission INPUT @@ -2707,6 +2725,7 @@ resources.quality_monitors.*.inference_log.timestamp_col string ALL resources.quality_monitors.*.latest_monitor_failure_msg string ALL resources.quality_monitors.*.lifecycle resources.Lifecycle INPUT resources.quality_monitors.*.lifecycle.prevent_destroy bool INPUT +resources.quality_monitors.*.lifecycle.started *bool INPUT resources.quality_monitors.*.modified_status string INPUT resources.quality_monitors.*.monitor_version int64 REMOTE resources.quality_monitors.*.notifications *catalog.MonitorNotifications ALL @@ -2756,6 +2775,7 @@ resources.registered_models.*.grants[*].privileges[*] string INPUT resources.registered_models.*.id string INPUT resources.registered_models.*.lifecycle resources.Lifecycle INPUT resources.registered_models.*.lifecycle.prevent_destroy bool INPUT +resources.registered_models.*.lifecycle.started *bool INPUT resources.registered_models.*.metastore_id string ALL resources.registered_models.*.modified_status string INPUT resources.registered_models.*.name string ALL @@ -2792,6 +2812,7 @@ resources.schemas.*.grants[*].privileges[*] resources.SchemaGrantPrivilege INPUT resources.schemas.*.id string INPUT resources.schemas.*.lifecycle resources.Lifecycle INPUT resources.schemas.*.lifecycle.prevent_destroy bool INPUT +resources.schemas.*.lifecycle.started *bool INPUT resources.schemas.*.metastore_id string REMOTE resources.schemas.*.modified_status string INPUT resources.schemas.*.name string ALL @@ -2822,6 +2843,7 @@ resources.secret_scopes.*.keyvault_metadata.dns_name string INPUT REMOTE resources.secret_scopes.*.keyvault_metadata.resource_id string INPUT REMOTE resources.secret_scopes.*.lifecycle resources.Lifecycle INPUT resources.secret_scopes.*.lifecycle.prevent_destroy bool INPUT +resources.secret_scopes.*.lifecycle.started *bool INPUT resources.secret_scopes.*.modified_status string INPUT resources.secret_scopes.*.name string INPUT REMOTE resources.secret_scopes.*.permissions []resources.SecretScopePermission INPUT @@ -2861,6 +2883,7 @@ resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT +resources.sql_warehouses.*.lifecycle.started *bool INPUT resources.sql_warehouses.*.max_num_clusters int ALL resources.sql_warehouses.*.min_num_clusters int ALL resources.sql_warehouses.*.modified_status string INPUT @@ -2879,6 +2902,7 @@ resources.sql_warehouses.*.permissions[*].level resources.SqlWarehousePermission resources.sql_warehouses.*.permissions[*].service_principal_name string INPUT resources.sql_warehouses.*.permissions[*].user_name string INPUT resources.sql_warehouses.*.spot_instance_policy sql.SpotInstancePolicy ALL +resources.sql_warehouses.*.started bool STATE resources.sql_warehouses.*.state sql.State REMOTE resources.sql_warehouses.*.tags *sql.EndpointTags ALL resources.sql_warehouses.*.tags.custom_tags []sql.EndpointTagPair ALL @@ -2942,6 +2966,7 @@ resources.synced_database_tables.*.effective_logical_database_name string ALL resources.synced_database_tables.*.id string INPUT resources.synced_database_tables.*.lifecycle resources.Lifecycle INPUT resources.synced_database_tables.*.lifecycle.prevent_destroy bool INPUT +resources.synced_database_tables.*.lifecycle.started *bool INPUT resources.synced_database_tables.*.logical_database_name string ALL resources.synced_database_tables.*.modified_status string INPUT resources.synced_database_tables.*.name string ALL @@ -2978,6 +3003,7 @@ resources.volumes.*.grants[*].privileges[*] resources.VolumeGrantPrivilege INPUT resources.volumes.*.id string INPUT resources.volumes.*.lifecycle resources.Lifecycle INPUT resources.volumes.*.lifecycle.prevent_destroy bool INPUT +resources.volumes.*.lifecycle.started *bool INPUT resources.volumes.*.metastore_id string REMOTE resources.volumes.*.modified_status string INPUT resources.volumes.*.name string ALL diff --git a/acceptance/bundle/resources/apps/lifecycle-started/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl new file mode 100644 index 0000000000..ee78dfc150 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt new file mode 100644 index 0000000000..4ad8a24b60 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -0,0 +1,106 @@ + +=== Deploy app with lifecycle.started=true +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify create request: no_compute must not be set +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps", + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + } +} + +=== Check app compute state: must be ACTIVE +>>> [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +=== Update description and re-deploy +>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify POST /deployments was called after update +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +=== Stop the app +>>> [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + +=== Change lifecycle.started to false and update +>>> update_file.py databricks.yml started: true started: false + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify POST /deployments is NOT called (lifecycle.started=false) +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +=== Check app compute state: must still be STOPPED +>>> [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +=== Switch lifecycle.started back to true and update +>>> update_file.py databricks.yml started: false started: true + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify POST /start and POST /deployments were called +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +=== Check app compute state: must be ACTIVE again +>>> [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script new file mode 100644 index 0000000000..2b9f234a4d --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -0,0 +1,52 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy app with lifecycle.started=true" +trace $CLI bundle deploy + +title "Verify create request: no_compute must not be set" +trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt +rm -f out.requests.txt + +title "Check app compute state: must be ACTIVE" +trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' + +title "Update description and re-deploy" +trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +trace $CLI bundle deploy + +title "Verify POST /deployments was called after update" +trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt +rm -f out.requests.txt + +title "Stop the app" +trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' + +title "Change lifecycle.started to false and update" +trace update_file.py databricks.yml "started: true" "started: false" +trace update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 +trace $CLI bundle deploy + +title "Verify POST /deployments is NOT called (lifecycle.started=false)" +trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt +rm -f out.requests.txt + +title "Check app compute state: must still be STOPPED" +trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' + +title "Switch lifecycle.started back to true and update" +trace update_file.py databricks.yml "started: false" "started: true" +trace update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 +trace $CLI bundle deploy + +title "Verify POST /start and POST /deployments were called" +trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt +rm -f out.requests.txt + +title "Check app compute state: must be ACTIVE again" +trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' diff --git a/acceptance/bundle/resources/apps/lifecycle-started/test.toml b/acceptance/bundle/resources/apps/lifecycle-started/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/appdeploy/app.go b/bundle/appdeploy/app.go new file mode 100644 index 0000000000..e73922d490 --- /dev/null +++ b/bundle/appdeploy/app.go @@ -0,0 +1,110 @@ +package appdeploy + +import ( + "context" + "fmt" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" + sdkapps "github.com/databricks/databricks-sdk-go/service/apps" +) + +func logProgress(ctx context.Context, msg string) { + if msg == "" { + return + } + cmdio.LogString(ctx, "✓ "+msg) +} + +// BuildDeployment constructs an AppDeployment from the app's source code path, inline config and git source. +func BuildDeployment(sourcePath string, config *resources.AppConfig, gitSource *sdkapps.GitSource) sdkapps.AppDeployment { + deployment := sdkapps.AppDeployment{ + Mode: sdkapps.AppDeploymentModeSnapshot, + SourceCodePath: sourcePath, + } + + if gitSource != nil { + deployment.GitSource = gitSource + } + + if config != nil { + if len(config.Command) > 0 { + deployment.Command = config.Command + } + + if len(config.Env) > 0 { + deployment.EnvVars = make([]sdkapps.EnvVar, len(config.Env)) + for i, env := range config.Env { + deployment.EnvVars[i] = sdkapps.EnvVar{ + Name: env.Name, + Value: env.Value, + ValueFrom: env.ValueFrom, + } + } + } + } + + return deployment +} + +// WaitForDeploymentToComplete waits for active and pending deployments on an app to finish. +func WaitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *sdkapps.App) error { + if app.ActiveDeployment != nil && + app.ActiveDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the active deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Active deployment is completed!") + } + + if app.PendingDeployment != nil && + app.PendingDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the pending deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Pending deployment is completed!") + } + + return nil +} + +// Deploy deploys the app using the provided deployment request. +// If another deployment is in progress, it waits for it to complete and retries. +func Deploy(ctx context.Context, w *databricks.WorkspaceClient, appName string, deployment sdkapps.AppDeployment) error { + wait, err := w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + existingApp, getErr := w.Apps.Get(ctx, sdkapps.GetAppRequest{Name: appName}) + if getErr != nil { + return fmt.Errorf("failed to get app %s: %w", appName, getErr) + } + + if waitErr := WaitForDeploymentToComplete(ctx, w, existingApp); waitErr != nil { + return waitErr + } + + wait, err = w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + return err + } + } + + _, err = wait.OnProgress(func(ad *sdkapps.AppDeployment) { + if ad.Status == nil { + return + } + logProgress(ctx, ad.Status.Message) + }).Get() + return err +} diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go new file mode 100644 index 0000000000..4a02b965d2 --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -0,0 +1,77 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// supportedForLifecycleStarted lists resource types that support lifecycle.started. +var supportedForLifecycleStarted = map[string]bool{ + "apps": true, + "clusters": true, + "sql_warehouses": true, +} + +type validateLifecycleStarted struct { + engine engine.EngineType +} + +// ValidateLifecycleStarted returns a mutator that validates lifecycle.started +// is only used on supported resource types (apps, clusters, sql_warehouses). +// lifecycle.started is only supported in direct deployment mode. +func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { + return &validateLifecycleStarted{engine: e} +} + +func (m *validateLifecycleStarted) Name() string { + return "ValidateLifecycleStarted" +} + +func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + // lifecycle.started is a direct-mode-only feature; ignore it in other modes. + if !m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + + _, err := dyn.MapByPattern( + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(path dyn.Path, v dyn.Value) (dyn.Value, error) { + resourceType := path[1].Key() + if supportedForLifecycleStarted[resourceType] { + return v, nil + } + + startedV, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("lifecycle"), dyn.Key("started"))) + if err != nil { + return v, nil + } + + started, ok := startedV.AsBool() + if !ok || !started { + return v, nil + } + + resourceKey := path.String() + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.started is not supported for %s; it is only supported for apps, clusters, and sql_warehouses", resourceKey), + Locations: []dyn.Location{startedV.Location()}, + }) + + return v, nil + }, + ) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} diff --git a/bundle/config/mutator/validate_lifecycle_started_test.go b/bundle/config/mutator/validate_lifecycle_started_test.go new file mode 100644 index 0000000000..311915892d --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started_test.go @@ -0,0 +1,108 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func boolPtr(b bool) *bool { return &b } + +func TestValidateLifecycleStarted_UnsupportedResource(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(true), + }, + }, + JobSettings: jobs.JobSettings{Name: "my_job"}, + }, + }, + }, + }, + } + + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) + require.Error(t, m.Error()) + assert.Contains(t, m.Error().Error(), "lifecycle.started is not supported for resources.jobs.my_job") +} + +func TestValidateLifecycleStarted_SupportedResources(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + SqlWarehouses: map[string]*resources.SqlWarehouse{ + "my_warehouse": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(true), + }, + }, + CreateWarehouseRequest: sql.CreateWarehouseRequest{ + Name: "my_warehouse", + }, + }, + }, + }, + }, + } + + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) + assert.NoError(t, m.Error()) +} + +func TestValidateLifecycleStarted_StartedFalse(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(false), + }, + }, + JobSettings: jobs.JobSettings{Name: "my_job"}, + }, + }, + }, + }, + } + + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) + assert.NoError(t, m.Error()) +} + +func TestValidateLifecycleStarted_TerraformModeIgnored(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(true), + }, + }, + JobSettings: jobs.JobSettings{Name: "my_job"}, + }, + }, + }, + }, + } + + // In TF mode, lifecycle.started is ignored — no error even for unsupported resource types. + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineTerraform)) + assert.NoError(t, m.Error()) +} diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index c3de7ce8ea..2ab74b4e30 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -5,4 +5,8 @@ package resources type Lifecycle struct { // Lifecycle setting to prevent the resource from being destroyed. PreventDestroy bool `json:"prevent_destroy,omitempty"` + + // If set to true, the resource will be deployed in started mode. + // Supported only for apps, clusters, and sql_warehouses. + Started *bool `json:"started,omitempty"` } diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go index b25d403766..87095eb0aa 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app.go +++ b/bundle/deploy/terraform/tfdyn/convert_app.go @@ -38,7 +38,7 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out return err } - // We always set no_compute to true as it allows DABs not to wait for app compute to be started when app is created. + // Always skip compute during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_compute", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go index e53b22a38d..3a2439014a 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -29,7 +29,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, return err } - // We always set no_wait as it allows DABs not to wait for cluster to be started. + // Always skip wait during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_wait", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/lifecycle.go b/bundle/deploy/terraform/tfdyn/lifecycle.go index 1600cdef2d..669aa83040 100644 --- a/bundle/deploy/terraform/tfdyn/lifecycle.go +++ b/bundle/deploy/terraform/tfdyn/lifecycle.go @@ -11,7 +11,19 @@ func convertLifecycle(ctx context.Context, vout, vLifecycle dyn.Value) (dyn.Valu return vout, nil } - vout, err := dyn.Set(vout, "lifecycle", vLifecycle) + // Strip lifecycle.started: it is a DABs-only field not understood by Terraform. + var err error + vLifecycle, err = dyn.DropKeys(vLifecycle, []string{"started"}) + if err != nil { + return dyn.InvalidValue, err + } + + // If only lifecycle.started was set (now empty), skip setting the lifecycle block. + if m, ok := vLifecycle.AsMap(); ok && m.Len() == 0 { + return vout, nil + } + + vout, err = dyn.Set(vout, "lifecycle", vLifecycle) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index e753079dac..ff6843cb50 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -7,13 +7,26 @@ import ( "strings" "time" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/apps" ) +// AppState is the state type for App resources. It extends apps.App with fields +// needed for app deployments (Apps.Deploy) that are not part of the remote state. +type AppState struct { + apps.App + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Started bool `json:"started,omitempty"` +} + type ResourceApp struct { client *databricks.WorkspaceClient } @@ -22,18 +35,41 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { return &ResourceApp{client: client} } -func (*ResourceApp) PrepareState(input *resources.App) *apps.App { - return &input.App +func (*ResourceApp) PrepareState(input *resources.App) *AppState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &AppState{ + App: input.App, + SourceCodePath: input.SourceCodePath, + Config: input.Config, + GitSource: input.GitSource, + Started: started, + } +} + +// RemapState maps the remote apps.App to AppState for diff comparison. +// Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, +// so they default to zero values, which prevents false drift detection. +func (*ResourceApp) RemapState(remote *apps.App) *AppState { + return &AppState{ + App: *remote, + SourceCodePath: "", + Config: nil, + GitSource: nil, + Started: false, + } } func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) { return r.client.Apps.GetByName(ctx, id) } -func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, *apps.App, error) { +func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { + // With lifecycle.started=true, start the app compute (no_compute=false). + // Otherwise, skip compute startup during creation. + noCompute := !config.Started request := apps.CreateAppRequest{ - App: *config, - NoCompute: true, + App: config.App, + NoCompute: noCompute, ForceSendFields: nil, } @@ -68,11 +104,11 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, * return app.Name, nil, nil } -func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, changes Changes) (*apps.App, error) { +func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, changes Changes) (*apps.App, error) { updateMask := strings.Join(collectUpdatePathsWithPrefix(changes, ""), ",") request := apps.AsyncUpdateAppRequest{ - App: config, + App: &config.App, AppName: id, UpdateMask: updateMask, } @@ -89,15 +125,64 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) } + + // With lifecycle.started=true, ensure the app compute is running and deploy the latest code. + if config.Started { + // Start compute if it is stopped (mirrors bundle run behavior). + app, err := r.client.Apps.GetByName(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get app %s: %w", id, err) + } + if isComputeStopped(app) { + startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) + if err != nil { + return nil, fmt.Errorf("failed to start app %s: %w", id, err) + } + startedApp, err := startWaiter.Get() + if err != nil { + return nil, fmt.Errorf("failed to wait for app %s to start: %w", id, err) + } + if err := appdeploy.WaitForDeploymentToComplete(ctx, r.client, startedApp); err != nil { + return nil, err + } + } + deployment := appdeploy.BuildDeployment(config.SourceCodePath, config.Config, config.GitSource) + if err := appdeploy.Deploy(ctx, r.client, id, deployment); err != nil { + return nil, err + } + } + return nil, nil } +// localOnlyFields are AppState fields that have no counterpart in the remote state. +// They must not appear in the App update_mask. +var localOnlyFields = map[string]bool{ + "source_code_path": true, + "config": true, + "git_source": true, + "started": true, +} + +func (*ResourceApp) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *apps.App) error { + if change.Action == deployplan.Update && localOnlyFields[p.Prefix(1).String()] { + change.Action = deployplan.Skip + } + return nil +} + +func isComputeStopped(app *apps.App) bool { + return app.ComputeStatus == nil || + app.ComputeStatus.State == apps.ComputeStateStopped || + app.ComputeStatus.State == apps.ComputeStateError +} + func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { _, err := r.client.Apps.DeleteByName(ctx, id) return err } -func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *apps.App) (*apps.App, error) { +func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*apps.App, error) { return r.waitForApp(ctx, r.client, config.Name) } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index e0eeed5b77..ad9ca01e8a 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -57,7 +57,7 @@ func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) @@ -113,7 +113,7 @@ func TestAppDoCreate_RetriesWhenGetReturnsNotFound(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index a8f78d12f9..2eea7d517d 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -16,6 +16,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) +type ClusterState struct { + compute.ClusterSpec + Started bool `json:"started,omitempty"` +} + type ResourceCluster struct { client *databricks.WorkspaceClient } @@ -26,11 +31,15 @@ func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { } } -func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.ClusterSpec { - return &input.ClusterSpec +func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &ClusterState{ + ClusterSpec: input.ClusterSpec, + Started: started, + } } -func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { +func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterState { spec := &compute.ClusterSpec{ ApplyPolicyDefaultValues: false, Autoscale: input.Autoscale, @@ -71,27 +80,35 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.Clu if input.Spec != nil { spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues } - return spec + return &ClusterState{ClusterSpec: *spec, Started: false} } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*compute.ClusterDetails, error) { return r.client.Clusters.GetByClusterId(ctx, id) } -func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterSpec) (string, *compute.ClusterDetails, error) { - wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(config)) +func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (string, *compute.ClusterDetails, error) { + wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(&config.ClusterSpec)) if err != nil { return "", nil, err } + // With lifecycle.started=true, wait for the cluster to reach the running state. + if config.Started { + details, err := wait.Get() + if err != nil { + return "", nil, err + } + return details.ClusterId, details, nil + } return wait.ClusterId, nil, nil } -func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec, _ Changes) (*compute.ClusterDetails, error) { +func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, _ Changes) (*compute.ClusterDetails, error) { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 timeout := 15 * time.Minute _, err := retries.Poll(ctx, timeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { - wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, config)) + wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, &config.ClusterSpec)) if err == nil { return wait, nil } @@ -107,7 +124,7 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compu return nil, err } -func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *compute.ClusterSpec) error { +func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *ClusterState) error { _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ ClusterId: id, NumWorkers: config.NumWorkers, @@ -129,6 +146,10 @@ func (r *ResourceCluster) OverrideChangeDesc(ctx context.Context, p *structpath. path := p.Prefix(1).String() switch path { + case "started": + // started is lifecycle metadata, not an actual cluster property. + change.Action = deployplan.Skip + case "data_security_mode": // We do change skip here in the same way TF provider does suppress diff if the alias is used. // https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L109-L117 diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 5d9d7793b7..066f63e184 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -4,12 +4,19 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/sql" ) +type SqlWarehouseState struct { + sql.CreateWarehouseRequest + Started bool `json:"started,omitempty"` +} + type ResourceSqlWarehouse struct { client *databricks.WorkspaceClient } @@ -20,12 +27,16 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq } // PrepareState converts bundle config to the SDK type. -func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *sql.CreateWarehouseRequest { - return &input.CreateWarehouseRequest +func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &SqlWarehouseState{ + CreateWarehouseRequest: input.CreateWarehouseRequest, + Started: started, + } } -func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sql.CreateWarehouseRequest { - return &sql.CreateWarehouseRequest{ +func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *SqlWarehouseState { + return &SqlWarehouseState{Started: false, CreateWarehouseRequest: sql.CreateWarehouseRequest{ AutoStopMins: warehouse.AutoStopMins, Channel: warehouse.Channel, ClusterSize: warehouse.ClusterSize, @@ -40,7 +51,7 @@ func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sq Tags: warehouse.Tags, WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), - } + }} } // DoRead reads the warehouse by id. @@ -49,16 +60,24 @@ func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*sql.GetW } // DoCreate creates the warehouse and returns its id. -func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateWarehouseRequest) (string, *sql.GetWarehouseResponse, error) { - waiter, err := r.client.Warehouses.Create(ctx, *config) +func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehouseState) (string, *sql.GetWarehouseResponse, error) { + waiter, err := r.client.Warehouses.Create(ctx, config.CreateWarehouseRequest) if err != nil { return "", nil, err } + // With lifecycle.started=true, wait for the warehouse to reach the running state. + if config.Started { + warehouse, err := waiter.Get() + if err != nil { + return "", nil, err + } + return warehouse.Id, warehouse, nil + } return waiter.Id, nil, nil } // DoUpdate updates the warehouse in place. -func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ Changes) (*sql.GetWarehouseResponse, error) { +func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, _ Changes) (*sql.GetWarehouseResponse, error) { request := sql.EditWarehouseRequest{ AutoStopMins: config.AutoStopMins, Channel: config.Channel, @@ -92,3 +111,11 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string) error { return r.client.Warehouses.DeleteById(ctx, oldID) } + +func (*ResourceSqlWarehouse) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *sql.GetWarehouseResponse) error { + if change.Action == deployplan.Update && p.Prefix(1).String() == "started" { + // started is lifecycle metadata, not an actual warehouse property. + change.Action = deployplan.Skip + } + return nil +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ade20e0ccd..1a2805ac4d 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,8 +16,16 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ + // source_code_path, config, git_source, and started are bundle-specific deployment fields not present in the remote App state. + "apps": { + "config", + "git_source", + "source_code_path", + "started", + }, "clusters": { "apply_policy_default_values", + "started", }, "external_locations": { "skip_validation", @@ -33,6 +41,9 @@ var knownMissingInRemoteType = map[string][]string{ "route_optimized", "tags", }, + "sql_warehouses": { + "started", + }, "quality_monitors": { "skip_builtin_dashboard", "warehouse_id", @@ -90,11 +101,6 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, - "apps": { - "config", - "source_code_path", - "git_source", - }, "dashboards": { "file_path", }, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8b4817bd4f..efdc3243de 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -774,6 +774,9 @@ github.com/databricks/cli/bundle/config/resources.Lifecycle: "prevent_destroy": "description": |- Lifecycle setting to prevent the resource from being destroyed. + "started": + "description": |- + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermission: "group_name": "description": |- diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 6e43f42291..ad987ceff3 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,6 +25,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), + mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/run/app.go b/bundle/run/app.go index 1dd4484093..c3a6497f1d 100644 --- a/bundle/run/app.go +++ b/bundle/run/app.go @@ -7,10 +7,10 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -124,7 +124,7 @@ func (a *appRunner) start(ctx context.Context) error { // active and pending deployments fields (if any). If there are active or pending deployments, // we need to wait for them to complete before we can do the new deployment. // Otherwise, the new deployment will fail. - err = waitForDeploymentToComplete(ctx, w, startedApp) + err = appdeploy.WaitForDeploymentToComplete(ctx, w, startedApp) if err != nil { return err } @@ -133,108 +133,10 @@ func (a *appRunner) start(ctx context.Context) error { return nil } -func waitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *apps.App) error { - // We first wait for the active deployment to complete. - if app.ActiveDeployment != nil && - app.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the active deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Active deployment is completed!") - } - - // Then, we wait for the pending deployment to complete. - if app.PendingDeployment != nil && - app.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the pending deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Pending deployment is completed!") - } - - return nil -} - -// buildAppDeployment creates an AppDeployment struct with inline config if provided -func (a *appRunner) buildAppDeployment() apps.AppDeployment { - deployment := apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: a.app.SourceCodePath, - } - - // Add git source if provided - if a.app.GitSource != nil { - deployment.GitSource = a.app.GitSource - } - - // Add inline config if provided - if a.app.Config != nil { - if len(a.app.Config.Command) > 0 { - deployment.Command = a.app.Config.Command - } - - if len(a.app.Config.Env) > 0 { - deployment.EnvVars = make([]apps.EnvVar, len(a.app.Config.Env)) - for i, env := range a.app.Config.Env { - deployment.EnvVars[i] = apps.EnvVar{ - Name: env.Name, - Value: env.Value, - ValueFrom: env.ValueFrom, - } - } - } - } - - return deployment -} - func (a *appRunner) deploy(ctx context.Context) error { - app := a.app - b := a.bundle - w := b.WorkspaceClient() - - wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - // If deploy returns an error, then there's an active deployment in progress, wait for it to complete. - // For this we first need to get an app and its acrive and pending deployments and then wait for them. - if err != nil { - app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name}) - if err != nil { - return fmt.Errorf("failed to get app %s: %w", app.Name, err) - } - - err = waitForDeploymentToComplete(ctx, w, app) - if err != nil { - return err - } - - // Now we can try to deploy the app again - wait, err = w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - if err != nil { - return err - } - } - - _, err = wait.OnProgress(func(ad *apps.AppDeployment) { - if ad.Status == nil { - return - } - logProgress(ctx, ad.Status.Message) - }).Get() - if err != nil { - return err - } - - return nil + w := a.bundle.WorkspaceClient() + deployment := appdeploy.BuildDeployment(a.app.SourceCodePath, a.app.Config, a.app.GitSource) + return appdeploy.Deploy(ctx, w, a.app.Name, deployment) } func (a *appRunner) Cancel(ctx context.Context) error { diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 112fcea28d..1c05fa63eb 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" @@ -294,11 +295,7 @@ func TestBuildAppDeploymentWithValueFrom(t *testing.T) { }, } - runner := &appRunner{ - app: app, - } - - deployment := runner.buildAppDeployment() + deployment := appdeploy.BuildDeployment(app.SourceCodePath, app.Config, app.GitSource) require.Equal(t, apps.AppDeploymentModeSnapshot, deployment.Mode) require.Equal(t, "/path/to/app", deployment.SourceCodePath) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2f48fb85f9..57d388291e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1241,6 +1241,10 @@ "prevent_destroy": { "description": "Lifecycle setting to prevent the resource from being destroyed.", "$ref": "#/$defs/bool" + }, + "started": { + "description": "Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode.", + "$ref": "#/$defs/bool" } }, "additionalProperties": false diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 8abd4cd416..8d7ac8bfc7 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -89,6 +89,79 @@ func (s *FakeWorkspace) AppsGetUpdate(_ Request, name string) Response { } } +func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + var deployment apps.AppDeployment + if err := json.Unmarshal(req.Body, &deployment); err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + + deployment.DeploymentId = "deploy-1" + deployment.Status = &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + } + + return Response{Body: deployment} +} + +func (s *FakeWorkspace) AppsGetDeployment(_ Request, name, deploymentID string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: apps.AppDeployment{ + DeploymentId: deploymentID, + Status: &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + }, + }} +} + +func (s *FakeWorkspace) AppsStart(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateActive, + Message: "App compute is active.", + } + s.Apps[name] = app + + return Response{Body: app} +} + +func (s *FakeWorkspace) AppsStop(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateStopped, + Message: "App compute is stopped.", + } + s.Apps[name] = app + + return Response{Body: app} +} + func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { var app apps.App diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9e30cb5f0c..b2a95b1902 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -379,6 +379,22 @@ func AddDefaultHandlers(server *Server) { // Apps: + server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req Request) any { + return req.Workspace.AppsCreateDeployment(req, req.Vars["name"]) + }) + + server.Handle("GET", "/api/2.0/apps/{name}/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.AppsGetDeployment(req, req.Vars["name"], req.Vars["deployment_id"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req Request) any { + return req.Workspace.AppsStart(req, req.Vars["name"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/stop", func(req Request) any { + return req.Workspace.AppsStop(req, req.Vars["name"]) + }) + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req Request) any { return req.Workspace.AppsCreateUpdate(req, req.Vars["name"]) }) diff --git a/python/databricks/bundles/catalogs/_models/lifecycle.py b/python/databricks/bundles/catalogs/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/catalogs/_models/lifecycle.py +++ b/python/databricks/bundles/catalogs/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/jobs/_models/lifecycle.py b/python/databricks/bundles/jobs/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/jobs/_models/lifecycle.py +++ b/python/databricks/bundles/jobs/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/pipelines/_models/lifecycle.py b/python/databricks/bundles/pipelines/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/pipelines/_models/lifecycle.py +++ b/python/databricks/bundles/pipelines/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/schemas/_models/lifecycle.py b/python/databricks/bundles/schemas/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/schemas/_models/lifecycle.py +++ b/python/databricks/bundles/schemas/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/volumes/_models/lifecycle.py b/python/databricks/bundles/volumes/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/volumes/_models/lifecycle.py +++ b/python/databricks/bundles/volumes/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle