From 29407cf391326f674a5427f275d8002465f2db5a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 07:18:51 +0000 Subject: [PATCH 1/8] direct: Add preliminary job_runs resource Introduce the job_runs bundle resource backed by the Jobs RunNow API. This is a preliminary skeleton: it wires up the config type, the direct engine resource, registration, and drift handling. Generated schema, validation output, and tests are intentionally left for follow-ups. --- .../config/mutator/resourcemutator/run_as.go | 11 +++ bundle/config/resources.go | 3 + bundle/config/resources/job_run.go | 71 ++++++++++++++ bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/job_run.go | 93 +++++++++++++++++++ bundle/direct/dresources/resources.yml | 66 +++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 bundle/config/resources/job_run.go create mode 100644 bundle/direct/dresources/job_run.go diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 4f5e3ce9036..132f0d1dfa5 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -126,6 +126,17 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { )) } + // Job runs execute under the triggered job's own identity; the RunNow API + // has no run_as field, so a differing bundle run_as cannot be honored. + if len(b.Config.Resources.JobRuns) > 0 { + diags = diags.Extend(reportRunAsNotSupported( + "job_runs", + b.Config.GetLocation("resources.job_runs"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) + } + return diags } diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 3dc7dc295d3..6c5a7a38a26 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -12,6 +12,7 @@ import ( // Resources defines Databricks resources associated with the bundle. type Resources struct { Jobs map[string]*resources.Job `json:"jobs,omitempty"` + JobRuns map[string]*resources.JobRun `json:"job_runs,omitempty"` Pipelines map[string]*resources.Pipeline `json:"pipelines,omitempty"` Models map[string]*resources.MlflowModel `json:"models,omitempty"` @@ -94,6 +95,7 @@ func (r *Resources) AllResources() []ResourceGroup { descriptions := SupportedResources() return []ResourceGroup{ collectResourceMap(descriptions["jobs"], r.Jobs), + collectResourceMap(descriptions["job_runs"], r.JobRuns), collectResourceMap(descriptions["pipelines"], r.Pipelines), collectResourceMap(descriptions["models"], r.Models), collectResourceMap(descriptions["experiments"], r.Experiments), @@ -153,6 +155,7 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) func SupportedResources() map[string]resources.ResourceDescription { return map[string]resources.ResourceDescription{ "jobs": (&resources.Job{}).ResourceDescription(), + "job_runs": (&resources.JobRun{}).ResourceDescription(), "pipelines": (&resources.Pipeline{}).ResourceDescription(), "models": (&resources.MlflowModel{}).ResourceDescription(), "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go new file mode 100644 index 00000000000..765a2938df1 --- /dev/null +++ b/bundle/config/resources/job_run.go @@ -0,0 +1,71 @@ +package resources + +import ( + "context" + "net/url" + "strconv" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// JobRun is the bundle config for a triggered job run. The run is described by +// the same fields as the Jobs RunNow API request, so we embed jobs.RunNow +// directly instead of re-declaring them. +type JobRun struct { + BaseResource + jobs.RunNow +} + +func (r *JobRun) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, r) +} + +func (r JobRun) MarshalJSON() ([]byte, error) { + return marshal.Marshal(r) +} + +// Exists reports whether the run identified by id (a numeric run id) still +// exists in the workspace. A run is the unit of existence here: once RunNow has +// been called, the run is retrievable via GetRun for as long as the workspace +// retains its history. +func (r *JobRun) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + runID, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return false, err + } + _, err = w.Jobs.GetRun(ctx, jobs.GetRunRequest{ + RunId: runID, + }) + if err != nil { + log.Debugf(ctx, "job run %s does not exist", id) + return false, err + } + return true, nil +} + +func (r *JobRun) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "job_run", + PluralName: "job_runs", + SingularTitle: "Job Run", + PluralTitle: "Job Runs", + } +} + +// GetName returns the in-product name. A run has no user-assigned name, so this +// is empty. +func (r *JobRun) GetName() string { + return "" +} + +func (r *JobRun) GetURL() string { + return r.URL +} + +// InitializeURL is a no-op for now: surfacing a stable run URL is deferred to a +// later milestone. +func (r *JobRun) InitializeURL(_ url.URL) { +} diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 6cc1eb55437..414ebcbe68a 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -8,6 +8,7 @@ import ( var SupportedResources = map[string]any{ "jobs": (*ResourceJob)(nil), + "job_runs": (*ResourceJobRun)(nil), "pipelines": (*ResourcePipeline)(nil), "experiments": (*ResourceExperiment)(nil), "catalogs": (*ResourceCatalog)(nil), diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go new file mode 100644 index 00000000000..4e9dfe47a49 --- /dev/null +++ b/bundle/direct/dresources/job_run.go @@ -0,0 +1,93 @@ +package dresources + +import ( + "context" + "fmt" + "strconv" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// JobRunRemote is the return type for DoRead. It embeds jobs.RunNow so that +// every field of the state type (jobs.RunNow) is also a valid path in the +// remote type, as required by the framework. GetRun does not echo back the +// RunNow request, so the embedded RunNow is left zero on read; the actual +// remote identity lives in RunId. Drift on the embedded request fields is +// suppressed via ignore_remote_changes in resources.yml. +type JobRunRemote struct { + jobs.RunNow + + RunId int64 `json:"run_id,omitempty"` +} + +func (s *JobRunRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s JobRunRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +type ResourceJobRun struct { + client *databricks.WorkspaceClient +} + +func (*ResourceJobRun) New(client *databricks.WorkspaceClient) *ResourceJobRun { + return &ResourceJobRun{ + client: client, + } +} + +func (*ResourceJobRun) PrepareState(input *resources.JobRun) *jobs.RunNow { + return &input.RunNow +} + +func (*ResourceJobRun) RemapState(remote *JobRunRemote) *jobs.RunNow { + return &remote.RunNow +} + +func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, error) { + runID, err := parseRunID(id) + if err != nil { + return nil, err + } + run, err := r.client.Jobs.GetRun(ctx, jobs.GetRunRequest{ + RunId: runID, + }) + if err != nil { + return nil, err + } + return &JobRunRemote{RunId: run.RunId}, nil +} + +func (r *ResourceJobRun) DoCreate(ctx context.Context, config *jobs.RunNow) (string, *JobRunRemote, error) { + // RunNow returns immediately with the new run id; waiting for completion is + // a later milestone. + wait, err := r.client.Jobs.RunNow(ctx, *config) + if err != nil { + return "", nil, err + } + return strconv.FormatInt(wait.RunId, 10), nil, nil +} + +// DoUpdate is intentionally not implemented: there is no API to modify a run in +// place. Every request field is marked recreate_on_changes in resources.yml, so +// any config change goes through delete + create (a fresh RunNow). + +// DoDelete is a no-op: a triggered run cannot be "undeployed". On recreate the +// framework calls this before DoCreate, so a no-op delete followed by RunNow +// re-triggers the run, which is the intended behavior. +func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *jobs.RunNow) error { + return nil +} + +func parseRunID(id string) (int64, error) { + result, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return 0, fmt.Errorf("internal error: run id is not integer: %q: %w", id, err) + } + return result, nil +} diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 9c12da9f39b..0a55df2651d 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -97,6 +97,72 @@ resources: - field: tasks[*].new_cluster.data_security_mode - field: job_clusters[*].new_cluster.data_security_mode + job_runs: + # GetRun returns the run's execution metadata, never the RunNow request that + # triggered it. So on every plan the request fields look like remote drift. + # Ignore that drift (input_only): the request fields are write-only inputs. + ignore_remote_changes: + - field: dbt_commands + reason: input_only + - field: idempotency_token + reason: input_only + - field: jar_params + reason: input_only + - field: job_id + reason: input_only + - field: job_parameters + reason: input_only + - field: notebook_params + reason: input_only + - field: only + reason: input_only + - field: performance_target + reason: input_only + - field: pipeline_params + reason: input_only + - field: python_named_params + reason: input_only + - field: python_params + reason: input_only + - field: queue + reason: input_only + - field: spark_submit_params + reason: input_only + - field: sql_params + reason: input_only + # There is no API to modify a run in place, so any change to the request must + # re-trigger a fresh run. With no DoUpdate, recreate (no-op delete + RunNow) + # is the only path; every request field is marked immutable here. + recreate_on_changes: + - field: dbt_commands + reason: immutable + - field: idempotency_token + reason: immutable + - field: jar_params + reason: immutable + - field: job_id + reason: immutable + - field: job_parameters + reason: immutable + - field: notebook_params + reason: immutable + - field: only + reason: immutable + - field: performance_target + reason: immutable + - field: pipeline_params + reason: immutable + - field: python_named_params + reason: immutable + - field: python_params + reason: immutable + - field: queue + reason: immutable + - field: spark_submit_params + reason: immutable + - field: sql_params + reason: immutable + pipelines: recreate_on_changes: - field: storage From 56938ddba5e4d5cec1dc771ef7401192e3946896 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 10:53:39 +0000 Subject: [PATCH 2/8] direct: Generate job_runs bundle schema Run the schema generator for the new job_runs resource and add real descriptions for the job_runs map, JobRun.lifecycle, and JobRun.python_named_params so the required-annotations guard test passes. --- bundle/internal/schema/annotations.yml | 3 + .../internal/schema/annotations_openapi.yml | 120 +++++++++++++++++ .../schema/annotations_openapi_overrides.yml | 7 + bundle/schema/jsonschema.json | 123 ++++++++++++++++++ 4 files changed, 253 insertions(+) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 69d7c9d025d..4b6c2593b69 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -198,6 +198,9 @@ github.com/databricks/cli/bundle/config.Resources: "genie_spaces": "description": |- PLACEHOLDER + "job_runs": + "description": |- + The job run definitions for the bundle, where each key is the name of the job run. "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 1c301cdc92f..4d1a16e75db 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -841,6 +841,126 @@ github.com/databricks/cli/bundle/config/resources.Job: "webhook_notifications": "description": |- A collection of system notification IDs to notify when runs of this job begin or complete. +github.com/databricks/cli/bundle/config/resources.JobRun: + "dbt_commands": + "description": |- + An array of commands to execute for jobs with the dbt task, for example `"dbt_commands": ["dbt deps", "dbt seed", "dbt deps", "dbt seed", "dbt run"]` + + ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE + "idempotency_token": + "description": |- + An optional token to guarantee the idempotency of job run requests. If a run with the provided token already exists, + the request does not create a new run but returns the ID of the existing run instead. If a run with the provided token is deleted, + an error is returned. + + If you specify the idempotency token, upon failure you can retry until the request succeeds. Databricks guarantees that exactly one run + is launched with that idempotency token. + + This token must have at most 64 characters. + + For more information, see [How to ensure idempotency for jobs](https://kb.databricks.com/jobs/jobs-idempotency.html). + "jar_params": + "description": |- + A list of parameters for jobs with Spark JAR tasks, for example `"jar_params": ["john doe", "35"]`. + The parameters are used to invoke the main function of the main class specified in the Spark JAR task. + If not specified upon `run-now`, it defaults to an empty list. + jar_params cannot be specified in conjunction with notebook_params. + The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. + + ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE + "job_id": + "description": |- + The ID of the job to be executed + "job_parameters": + "description": |- + Job-level parameters used in the run. for example `"param": "overriding_val"` + "notebook_params": + "description": |- + A map from keys to values for jobs with notebook task, for example `"notebook_params": {"name": "john doe", "age": "35"}`. + The map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function. + + If not specified upon `run-now`, the triggered run uses the job’s base parameters. + + notebook_params cannot be specified in conjunction with jar_params. + + ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. + + The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE + "only": + "description": |- + A list of task keys to run inside of the job. If this field is not provided, all tasks in the job will be run. + "performance_target": + "description": |- + The performance mode on a serverless job. The performance target determines the level of compute performance or cost-efficiency for the run. This field overrides the performance target defined on the job level. + + * `STANDARD`: Enables cost-efficient execution of serverless workloads. + * `PERFORMANCE_OPTIMIZED`: Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance. + "pipeline_params": + "description": |- + Controls whether the pipeline should perform a full refresh + "python_named_params": + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE + "python_params": + "description": |- + A list of parameters for jobs with Python tasks, for example `"python_params": ["john doe", "35"]`. + The parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite + the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) + cannot exceed 10,000 bytes. + + ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. + + Important + + These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. + Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE + "queue": + "description": |- + The queue settings of the run. + "spark_submit_params": + "description": |- + A list of parameters for jobs with spark submit task, for example `"spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"]`. + The parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the + parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) + cannot exceed 10,000 bytes. + + ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. + + Important + + These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. + Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE + "sql_params": + "description": |- + A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. + + ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. + "deprecation_message": |- + This field is deprecated + "x-databricks-preview": |- + PRIVATE github.com/databricks/cli/bundle/config/resources.MlflowExperiment: "artifact_location": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 25fc5869500..2bcdbfdd344 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -257,6 +257,13 @@ github.com/databricks/cli/bundle/config/resources.Job: "run_as": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.JobRun: + "lifecycle": + "description": |- + Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed. + "python_named_params": + "description": |- + A map from keys to values for jobs with Python wheel tasks, for example `"python_named_params": {"name": "task", "data": "dbfs:/path/to/data.json"}`. github.com/databricks/cli/bundle/config/resources.MlflowExperiment: "_": "markdown_description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index f88a9389348..0d18155f20b 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -963,6 +963,111 @@ } ] }, + "resources.JobRun": { + "oneOf": [ + { + "type": "object", + "properties": { + "dbt_commands": { + "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "idempotency_token": { + "description": "An optional token to guarantee the idempotency of job run requests. If a run with the provided token already exists,\nthe request does not create a new run but returns the ID of the existing run instead. If a run with the provided token is deleted,\nan error is returned.\n\nIf you specify the idempotency token, upon failure you can retry until the request succeeds. Databricks guarantees that exactly one run\nis launched with that idempotency token.\n\nThis token must have at most 64 characters.\n\nFor more information, see [How to ensure idempotency for jobs](https://kb.databricks.com/jobs/jobs-idempotency.html).", + "$ref": "#/$defs/string" + }, + "jar_params": { + "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "job_id": { + "description": "The ID of the job to be executed", + "$ref": "#/$defs/int64" + }, + "job_parameters": { + "description": "Job-level parameters used in the run. for example `\"param\": \"overriding_val\"`", + "$ref": "#/$defs/map/string" + }, + "lifecycle": { + "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "notebook_params": { + "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "only": { + "description": "A list of task keys to run inside of the job. If this field is not provided, all tasks in the job will be run.", + "$ref": "#/$defs/slice/string" + }, + "performance_target": { + "description": "The performance mode on a serverless job. The performance target determines the level of compute performance or cost-efficiency for the run. This field overrides the performance target defined on the job level.\n\n* `STANDARD`: Enables cost-efficient execution of serverless workloads.\n* `PERFORMANCE_OPTIMIZED`: Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PerformanceTarget" + }, + "pipeline_params": { + "description": "Controls whether the pipeline should perform a full refresh", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" + }, + "python_named_params": { + "description": "A map from keys to values for jobs with Python wheel tasks, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "python_params": { + "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "queue": { + "description": "The queue settings of the run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings" + }, + "spark_submit_params": { + "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "sql_params": { + "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + } + }, + "additionalProperties": false, + "required": [ + "job_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Lifecycle": { "oneOf": [ { @@ -2731,6 +2836,10 @@ "genie_spaces": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" }, + "job_runs": { + "description": "The job run definitions for the bundle, where each key is the name of the job run.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -12815,6 +12924,20 @@ } ] }, + "resources.JobRun": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.JobRun" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.MlflowExperiment": { "oneOf": [ { From 6fc4fce58910f6af26cc3eeba0ea3c449c7070ac Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 12:00:21 +0000 Subject: [PATCH 3/8] direct: Generate job_runs required-field validation Regenerate the validation rules so job_runs picks up its required field (job_id) and the performance_target enum. Missing required fields surface as warnings via the existing validate:required mutator. --- bundle/internal/validation/generated/enum_fields.go | 2 ++ bundle/internal/validation/generated/required_fields.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 8beb3e25a4a..ed1a594fee1 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -64,6 +64,8 @@ var EnumFields = map[string][]string{ "resources.genie_spaces.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.job_runs.*.performance_target": {"PERFORMANCE_OPTIMIZED", "STANDARD"}, + "resources.jobs.*.continuous.pause_status": {"PAUSED", "UNPAUSED"}, "resources.jobs.*.continuous.task_retry_mode": {"NEVER", "ON_FAILURE"}, "resources.jobs.*.deployment.kind": {"BUNDLE", "SYSTEM_MANAGED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 3d47858587e..2e7b6924d81 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -66,6 +66,9 @@ var RequiredFields = map[string][]string{ "resources.genie_spaces.*.permissions[*]": {"level"}, + "resources.job_runs.*": {"job_id"}, + "resources.job_runs.*.queue": {"enabled"}, + "resources.jobs.*.deployment": {"kind"}, "resources.jobs.*.environments[*]": {"environment_key"}, "resources.jobs.*.git_source": {"git_provider", "git_url"}, From 361c9c811eedfaf8c991d21936b6243976b473ab Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 12:39:15 +0000 Subject: [PATCH 4/8] direct: Add job_runs happy-path acceptance test Deploy a bundle with a job and a job_run referencing it, and assert that exactly one RunNow request is made and the returned run id is stored in state. Restricted to the direct engine since job_runs has no Terraform equivalent. --- .../resources/job_runs/basic/databricks.yml | 15 ++++++++ .../resources/job_runs/basic/out.test.toml | 3 ++ .../resources/job_runs/basic/output.txt | 34 +++++++++++++++++++ .../bundle/resources/job_runs/basic/script | 15 ++++++++ .../bundle/resources/job_runs/basic/test.toml | 4 +++ 5 files changed, 71 insertions(+) create mode 100644 acceptance/bundle/resources/job_runs/basic/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/basic/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/basic/output.txt create mode 100644 acceptance/bundle/resources/job_runs/basic/script create mode 100644 acceptance/bundle/resources/job_runs/basic/test.toml diff --git a/acceptance/bundle/resources/job_runs/basic/databricks.yml b/acceptance/bundle/resources/job_runs/basic/databricks.yml new file mode 100644 index 00000000000..5ea02d60dc2 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/basic/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: job-runs-basic + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/basic/out.test.toml b/acceptance/bundle/resources/job_runs/basic/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/basic/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt new file mode 100644 index 00000000000..5100598f5d9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -0,0 +1,34 @@ + +=== deploy triggers exactly one run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== exactly one run-now request was made +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [MY_JOB_ID] + } +} + +=== the triggered run id is stored in state +>>> read_id.py my_run +[MY_RUN_ID] + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/basic/script b/acceptance/bundle/resources/job_runs/basic/script new file mode 100644 index 00000000000..ce2d8c414e5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/basic/script @@ -0,0 +1,15 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT + +title "deploy triggers exactly one run" +trace $CLI bundle deploy + +title "exactly one run-now request was made" +trace print_requests.py //jobs/run-now + +title "the triggered run id is stored in state" +trace read_id.py my_run +trace read_id.py my_job diff --git a/acceptance/bundle/resources/job_runs/basic/test.toml b/acceptance/bundle/resources/job_runs/basic/test.toml new file mode 100644 index 00000000000..4b94d8b58e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/basic/test.toml @@ -0,0 +1,4 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true From 3d39a877ce95d26aacbe0a696edb663458177d6c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 12:50:09 +0000 Subject: [PATCH 5/8] direct: Add job_runs redeploy acceptance test Verify that changing a job_run's configuration re-triggers the run: after the initial deploy, a config change (job_parameters) causes a second, distinct RunNow request on redeploy via the recreate-on-change path. --- .../job_runs/redeploy/databricks.yml | 17 ++++++ .../resources/job_runs/redeploy/out.test.toml | 3 ++ .../resources/job_runs/redeploy/output.txt | 54 +++++++++++++++++++ .../bundle/resources/job_runs/redeploy/script | 17 ++++++ .../resources/job_runs/redeploy/test.toml | 4 ++ 5 files changed, 95 insertions(+) create mode 100644 acceptance/bundle/resources/job_runs/redeploy/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/redeploy/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/redeploy/output.txt create mode 100644 acceptance/bundle/resources/job_runs/redeploy/script create mode 100644 acceptance/bundle/resources/job_runs/redeploy/test.toml diff --git a/acceptance/bundle/resources/job_runs/redeploy/databricks.yml b/acceptance/bundle/resources/job_runs/redeploy/databricks.yml new file mode 100644 index 00000000000..370829ce20c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/redeploy/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: job-runs-redeploy + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + job_parameters: + env: dev diff --git a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt new file mode 100644 index 00000000000..fb73c865fb9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -0,0 +1,54 @@ + +=== initial deploy triggers the first run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [MY_JOB_ID], + "job_parameters": { + "env": "dev" + } + } +} + +=== change the run configuration and redeploy +>>> update_file.py databricks.yml env: dev env: prod + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== the config change triggered a second, different run +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [MY_JOB_ID], + "job_parameters": { + "env": "prod" + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script new file mode 100644 index 00000000000..d712dd7b079 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -0,0 +1,17 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT + +title "initial deploy triggers the first run" +trace $CLI bundle deploy +trace read_id.py my_job +trace print_requests.py //jobs/run-now + +title "change the run configuration and redeploy" +trace update_file.py databricks.yml "env: dev" "env: prod" +trace $CLI bundle deploy + +title "the config change triggered a second, different run" +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/redeploy/test.toml b/acceptance/bundle/resources/job_runs/redeploy/test.toml new file mode 100644 index 00000000000..4b94d8b58e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/redeploy/test.toml @@ -0,0 +1,4 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true From 3fb23a7e13e0aa7f11c5c8667d3019803ab41749 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 13:23:32 +0000 Subject: [PATCH 6/8] direct: Update resource enumeration tests for job_runs Several reflection/enumeration tests iterate or hardcode the full resource set and were not updated when the job_runs resource was added, leaving CI red. Add job_runs to each: the run_as allResourceTypes list, the permissions unsupportedResources list, the target-mode mock bundle, the Terraform lifecycle skip list (direct-only), a bind fixture + GetRun mock, and a direct CRUD fixture that triggers a run against a newly created job. The CRUD test also treats job_runs as a no-op delete, since a triggered run cannot be undeployed and stays readable afterward. --- .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 3 ++ .../mutator/resourcemutator/run_as_test.go | 1 + bundle/config/resources_test.go | 6 ++++ bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/direct/dresources/all_test.go | 28 ++++++++++++++++++- 6 files changed, 39 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 3fa97c41a7f..2e55f0d2a1d 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -31,6 +31,7 @@ var unsupportedResources = []string{ "postgres_catalogs", "postgres_synced_tables", "vector_search_indexes", + "job_runs", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 440221001a7..13d58ec1b25 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -92,6 +92,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + JobRuns: map[string]*resources.JobRun{ + "job_run1": {RunNow: jobs.RunNow{JobId: 1234}}, + }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": {CreatePipeline: pipelines.CreatePipeline{Name: "pipeline1", Continuous: true}}, }, diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index af1470848d7..7562604c075 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -42,6 +42,7 @@ func allResourceTypes(t *testing.T) []string { "experiments", "external_locations", "genie_spaces", + "job_runs", "jobs", "model_serving_endpoints", "models", diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d56a24ced46..8cd9e4425f2 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -157,6 +157,11 @@ func TestResourcesBindSupport(t *testing.T) { JobSettings: jobs.JobSettings{}, }, }, + JobRuns: map[string]*resources.JobRun{ + "my_job_run": { + RunNow: jobs.RunNow{}, + }, + }, Pipelines: map[string]*resources.Pipeline{ "my_pipeline": { CreatePipeline: pipelines.CreatePipeline{}, @@ -315,6 +320,7 @@ func TestResourcesBindSupport(t *testing.T) { ctx := t.Context() m := mocks.NewMockWorkspaceClient(t) m.GetMockJobsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPipelinesAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockRegisteredModelsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index b60bff612c7..248a66c21a9 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -18,6 +18,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "catalogs", "external_locations", "genie_spaces", + "job_runs", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 30adb4640cc..727b5c2cbe4 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -319,6 +319,30 @@ var testDeps = map[string]prepareWorkspace{ return testConfig["vector_search_indexes"], nil }, + "job_runs": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + // A run can only be triggered against an existing job, so create one first. + resp, err := client.Jobs.Create(ctx, jobs.CreateJob{ + Name: "job-for-run", + Tasks: []jobs.Task{ + { + TaskKey: "t", + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "/Workspace/Users/user@example.com/notebook", + }, + }, + }, + }) + if err != nil { + return nil, err + } + + return &resources.JobRun{ + RunNow: jobs.RunNow{ + JobId: resp.JobId, + }, + }, nil + }, + "jobs.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { resp, err := client.Jobs.Create(ctx, jobs.CreateJob{ Name: "job-permissions", @@ -942,7 +966,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + // job_runs has a no-op DoDelete (a triggered run cannot be "undeployed"), so + // the run remains readable after delete, like permissions and grants. + deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || group == "job_runs" remoteAfterDelete, err := adapter.DoRead(ctx, createdID) if deleteIsNoop { From 158f4dd0f33fcf5a78376e5c70910b1dc3bb1547 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 13:44:12 +0000 Subject: [PATCH 7/8] direct: Fix job_runs lint and regenerate stale artifacts Fully populate the GetRunRequest and JobRunRemote literals in job_run.go to satisfy exhaustruct, and regenerate the schema/refschema/apitypes artifacts so the committed job_runs generated files match their generators. --- acceptance/bundle/refschema/out.fields.txt | 39 +++++++++++++++++++ .../direct/dresources/apitypes.generated.yml | 2 + bundle/direct/dresources/job_run.go | 11 +++++- .../direct/dresources/resources.generated.yml | 2 + .../internal/schema/annotations_openapi.yml | 14 +++++++ bundle/schema/jsonschema.json | 14 +++---- 6 files changed, 73 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 80b553561a5..9f11c72b93a 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -752,6 +752,45 @@ resources.genie_spaces.*.permissions[*].group_name string ALL resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL resources.genie_spaces.*.permissions[*].service_principal_name string ALL resources.genie_spaces.*.permissions[*].user_name string ALL +resources.job_runs.*.dbt_commands []string ALL +resources.job_runs.*.dbt_commands[*] string ALL +resources.job_runs.*.id string INPUT +resources.job_runs.*.idempotency_token string ALL +resources.job_runs.*.jar_params []string ALL +resources.job_runs.*.jar_params[*] string ALL +resources.job_runs.*.job_id int64 ALL +resources.job_runs.*.job_parameters map[string]string ALL +resources.job_runs.*.job_parameters.* string ALL +resources.job_runs.*.lifecycle resources.Lifecycle INPUT +resources.job_runs.*.lifecycle.prevent_destroy bool INPUT +resources.job_runs.*.modified_status string INPUT +resources.job_runs.*.notebook_params map[string]string ALL +resources.job_runs.*.notebook_params.* string ALL +resources.job_runs.*.only []string ALL +resources.job_runs.*.only[*] string ALL +resources.job_runs.*.performance_target jobs.PerformanceTarget ALL +resources.job_runs.*.pipeline_params *jobs.PipelineParams ALL +resources.job_runs.*.pipeline_params.full_refresh bool ALL +resources.job_runs.*.pipeline_params.full_refresh_selection []string ALL +resources.job_runs.*.pipeline_params.full_refresh_selection[*] string ALL +resources.job_runs.*.pipeline_params.refresh_flow_selection []string ALL +resources.job_runs.*.pipeline_params.refresh_flow_selection[*] string ALL +resources.job_runs.*.pipeline_params.refresh_selection []string ALL +resources.job_runs.*.pipeline_params.refresh_selection[*] string ALL +resources.job_runs.*.pipeline_params.reset_checkpoint_selection []string ALL +resources.job_runs.*.pipeline_params.reset_checkpoint_selection[*] string ALL +resources.job_runs.*.python_named_params map[string]string ALL +resources.job_runs.*.python_named_params.* string ALL +resources.job_runs.*.python_params []string ALL +resources.job_runs.*.python_params[*] string ALL +resources.job_runs.*.queue *jobs.QueueSettings ALL +resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.run_id int64 REMOTE +resources.job_runs.*.spark_submit_params []string ALL +resources.job_runs.*.spark_submit_params[*] string ALL +resources.job_runs.*.sql_params map[string]string ALL +resources.job_runs.*.sql_params.* string ALL +resources.job_runs.*.url string INPUT resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 93faba8ad5a..be59b1d156e 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -20,6 +20,8 @@ external_locations: catalog.CreateExternalLocation genie_spaces: dashboards.GenieUpdateSpaceRequest +job_runs: jobs.RunNow + jobs: jobs.JobSettings model_serving_endpoints: serving.CreateServingEndpoint diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 4e9dfe47a49..4e5f2b37071 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -55,12 +55,19 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, return nil, err } run, err := r.client.Jobs.GetRun(ctx, jobs.GetRunRequest{ - RunId: runID, + RunId: runID, + IncludeHistory: false, + IncludeResolvedValues: false, + PageToken: "", + ForceSendFields: nil, }) if err != nil { return nil, err } - return &JobRunRemote{RunId: run.RunId}, nil + // GetRun does not echo back the RunNow request, so the embedded RunNow is + // left zero here; the remote identity lives in RunId. + var emptyRunNow jobs.RunNow + return &JobRunRemote{RunNow: emptyRunNow, RunId: run.RunId}, nil } func (r *ResourceJobRun) DoCreate(ctx context.Context, config *jobs.RunNow) (string, *JobRunRemote, error) { diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index e2509c2618e..176d9ea3734 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -176,6 +176,8 @@ resources: - field: etag reason: spec:output_only + # job_runs: no api field behaviors + # jobs: no api field behaviors # model_serving_endpoints: no api field behaviors diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 4d1a16e75db..3fce2fdb82b 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -849,6 +849,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE "idempotency_token": @@ -874,6 +876,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE "job_id": @@ -896,6 +900,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE "only": @@ -913,6 +919,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: "python_named_params": "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE "python_params": @@ -930,6 +938,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE "queue": @@ -950,6 +960,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE "sql_params": @@ -959,6 +971,8 @@ github.com/databricks/cli/bundle/config/resources.JobRun: ⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks. "deprecation_message": |- This field is deprecated + "x-databricks-launch-stage": |- + PRIVATE_PREVIEW "x-databricks-preview": |- PRIVATE github.com/databricks/cli/bundle/config/resources.MlflowExperiment: diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 0d18155f20b..9bd14a6d350 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -969,7 +969,7 @@ "type": "object", "properties": { "dbt_commands": { - "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", + "description": "[Private Preview] An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", "$ref": "#/$defs/slice/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", @@ -981,7 +981,7 @@ "$ref": "#/$defs/string" }, "jar_params": { - "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", + "description": "[Private Preview] A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", "$ref": "#/$defs/slice/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", @@ -1001,7 +1001,7 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, "notebook_params": { - "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", + "description": "[Private Preview] A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", "$ref": "#/$defs/map/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", @@ -1021,7 +1021,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" }, "python_named_params": { - "description": "A map from keys to values for jobs with Python wheel tasks, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.", + "description": "[Private Preview] A map from keys to values for jobs with Python wheel tasks, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.", "$ref": "#/$defs/map/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", @@ -1029,7 +1029,7 @@ "deprecated": true }, "python_params": { - "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "description": "[Private Preview] A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", "$ref": "#/$defs/slice/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", @@ -1041,7 +1041,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings" }, "spark_submit_params": { - "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "description": "[Private Preview] A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", "$ref": "#/$defs/slice/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", @@ -1049,7 +1049,7 @@ "deprecated": true }, "sql_params": { - "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", + "description": "[Private Preview] A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.", "$ref": "#/$defs/map/string", "x-databricks-preview": "PRIVATE", "deprecationMessage": "This field is deprecated", From 336177f37fff4a134ea2c7e41c52bbc5f5ecfbe2 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 15 Jun 2026 14:28:45 +0000 Subject: [PATCH 8/8] direct: Update completeness tests for job_runs Add job_runs coverage to the two reflection-driven completeness tests: add "job_runs" to the noURL set in TestBundleResourcePluralNamesResolveInWorkspaceURLs (no stable workspace URL yet) and exercise Resources.JobRuns in the StateToBundle test suite. --- bundle/config/resources_test.go | 3 +++ bundle/statemgmt/state_load_test.go | 35 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 8cd9e4425f2..9c20b1cae0f 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -125,6 +125,9 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // Resources that intentionally have no workspace URL. noURL := map[string]bool{ "external_locations": true, + // A job run has no stable workspace URL yet; surfacing one is deferred + // to a later milestone (see JobRun.InitializeURL). + "job_runs": true, "postgres_branches": true, "postgres_endpoints": true, "postgres_projects": true, diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 672cd9855b2..74d2aa412c5 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -27,6 +27,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { state := ExportedResourcesMap{ "resources.jobs.test_job": {ID: "1"}, + "resources.job_runs.test_job_run": {ID: "1"}, "resources.pipelines.test_pipeline": {ID: "1"}, "resources.models.test_mlflow_model": {ID: "1"}, "resources.experiments.test_mlflow_experiment": {ID: "1"}, @@ -61,6 +62,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Jobs["test_job"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Jobs["test_job"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.JobRuns["test_job_run"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.JobRuns["test_job_run"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Pipelines["test_pipeline"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Pipelines["test_pipeline"].ModifiedStatus) @@ -147,6 +151,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + JobRuns: map[string]*resources.JobRun{ + "test_job_run": { + RunNow: jobs.RunNow{ + JobId: 1234, + }, + }, + }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { CreatePipeline: pipelines.CreatePipeline{ @@ -349,6 +360,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.Jobs["test_job"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Jobs["test_job"].ModifiedStatus) + assert.Empty(t, config.Resources.JobRuns["test_job_run"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.JobRuns["test_job_run"].ModifiedStatus) + assert.Empty(t, config.Resources.Pipelines["test_pipeline"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Pipelines["test_pipeline"].ModifiedStatus) @@ -445,6 +459,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + JobRuns: map[string]*resources.JobRun{ + "test_job_run": { + RunNow: jobs.RunNow{ + JobId: 1234, + }, + }, + "test_job_run_new": { + RunNow: jobs.RunNow{ + JobId: 5678, + }, + }, + }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { CreatePipeline: pipelines.CreatePipeline{ @@ -771,6 +797,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { state := ExportedResourcesMap{ "resources.jobs.test_job": {ID: "1"}, "resources.jobs.test_job_old": {ID: "2"}, + "resources.job_runs.test_job_run": {ID: "1"}, + "resources.job_runs.test_job_run_old": {ID: "2"}, "resources.pipelines.test_pipeline": {ID: "1"}, "resources.pipelines.test_pipeline_old": {ID: "2"}, "resources.models.test_mlflow_model": {ID: "1"}, @@ -828,6 +856,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.Jobs["test_job_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Jobs["test_job_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.JobRuns["test_job_run"].ID) + assert.Empty(t, config.Resources.JobRuns["test_job_run"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.JobRuns["test_job_run_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.JobRuns["test_job_run_old"].ModifiedStatus) + assert.Empty(t, config.Resources.JobRuns["test_job_run_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.JobRuns["test_job_run_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Pipelines["test_pipeline"].ID) assert.Empty(t, config.Resources.Pipelines["test_pipeline"].ModifiedStatus) assert.Equal(t, "2", config.Resources.Pipelines["test_pipeline_old"].ID)