Reproduce stale-published-content dashboard bug as acceptance test

acceptance/bundle/resources/dashboards/change-embed-credentials/out.patch.requests.terraform.txt

@@ -5,7 +5,7 @@
     "create_time": "[TIMESTAMP]",
     "dashboard_id": "[DASHBOARD_ID]",
     "display_name": "test dashboard",
-    "etag": [ETAG],
+    "etag": "[ETAG]",
     "lifecycle_state": "ACTIVE",
     "parent_path": "/Users/[USERNAME]/.bundle/change-embed-credentials-[UNIQUE_NAME]/default/resources",
     "path": "/Users/[USERNAME]/.bundle/change-embed-credentials-[UNIQUE_NAME]/default/resources/test dashboard.lvdash.json",

acceptance/bundle/resources/dashboards/change-embed-credentials/script

@@ -25,6 +25,7 @@ deploy_dashboard
 # Capture the dashboard ID as a replacement.
 dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
 echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS
+echo "$($CLI lakeview get $dashboard_id | jq -r '.etag'):ETAG" >> ACC_REPLS
 
 # Change embed_credentials to true - this should trigger an update
 export EMBED_CREDENTIALS="true"

acceptance/bundle/resources/dashboards/change-name/out.patch.requests.terraform.txt

@@ -5,7 +5,7 @@
     "create_time": "[TIMESTAMP]",
     "dashboard_id": "[DASHBOARD_ID]",
     "display_name": "dashboard2",
-    "etag": [ETAG],
+    "etag": "[ETAG]",
     "lifecycle_state": "ACTIVE",
     "parent_path": "/Users/[USERNAME]/.bundle/change-name-[UNIQUE_NAME]/default/resources",
     "path": "/Users/[USERNAME]/.bundle/change-name-[UNIQUE_NAME]/default/resources/dashboard1.lvdash.json",

acceptance/bundle/resources/dashboards/change-name/script

@@ -21,6 +21,7 @@ deploy_dashboard
 # Capture the dashboard ID as a replacement.
 dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id')
 echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS
+echo "$($CLI lakeview get $dashboard_id | jq -r '.etag'):ETAG" >> ACC_REPLS
 
 # Change the name
 export NAME="dashboard2"

acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json

@@ -11,7 +11,7 @@
         "dashboard_id": "[DASHBOARD_ID]",
         "display_name": "test bundle-deploy-dashboard-dataset [UUID]",
         "embed_credentials": true,
-        "etag": [ETAG],
+        "etag": "[ETAG]",
         "lifecycle_state": "ACTIVE",
         "parent_path": "/Workspace/Users/[USERNAME]",
         "path": "/Users/[USERNAME]/test bundle-deploy-dashboard-dataset [UUID].lvdash.json",
@@ -36,8 +36,8 @@
         "etag": {
           "action": "skip",
           "reason": "custom",
-          "old": [ETAG],
-          "remote": [ETAG]
+          "old": "[ETAG]",
+          "remote": "[ETAG]"
         },
         "serialized_dashboard": {
           "action": "skip",

acceptance/bundle/resources/dashboards/dataset-catalog-schema/script

@@ -19,6 +19,7 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
+echo "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag'):ETAG" >> ACC_REPLS
 
 trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path}'
 

acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json

@@ -20,7 +20,7 @@
         "etag": {
           "action": "update",
           "reason": "custom",
-          "old": [ETAG]
+          "old": "[ETAG]"
         }
       }
     }

acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script

@@ -13,6 +13,7 @@ DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
+echo "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag'):ETAG" >> ACC_REPLS
 
 # Verify dashboard was deployed
 trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path}'

acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json

@@ -21,7 +21,7 @@
         "dashboard_id": "[DASHBOARD_ID]",
         "display_name": "test-dashboard-[UNIQUE_NAME]",
         "embed_credentials": false,
-        "etag": [ETAG],
+        "etag": "[ETAG_2]",
         "lifecycle_state": "ACTIVE",
         "parent_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources",
         "path": "/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources/test-dashboard-[UNIQUE_NAME].lvdash.json",
@@ -33,8 +33,8 @@
       "changes": {
         "etag": {
           "action": "update",
-          "old": [ETAG],
-          "remote": [ETAG]
+          "old": "[ETAG_1]",
+          "remote": "[ETAG_2]"
         },
         "serialized_dashboard": {
           "action": "skip",

acceptance/bundle/resources/dashboards/detect-change/output.txt

@@ -30,7 +30,7 @@ Deployment complete!
 {
   "display_name": "test-dashboard-[UNIQUE_NAME]",
   "embed_credentials": false,
-  "etag": [ETAG],
+  "etag": "[ETAG_1]",
   "file_path": "dashboard.lvdash.json",
   "id": "[DASHBOARD_ID]",
   "parent_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources",

acceptance/bundle/resources/dashboards/detect-change/script

@@ -22,6 +22,7 @@ DASHBOARD_ID=$(jq -r '.id' out.summary.json)
 
 # Capture the dashboard ID as a replacement.
 echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
+echo "$($CLI lakeview get "$DASHBOARD_ID" | jq -r '.etag'):ETAG_1" >> ACC_REPLS
 
 rm out.summary.json
 title "Load the dashboard by its ID and confirm its display name: "
@@ -31,6 +32,7 @@ title "Make an out of band modification to the dashboard and confirm that it is
 RESOURCE_ID=$($CLI workspace get-status "${DASHBOARD_PATH}" | jq -r '.resource_id')
 DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\"}"
 $CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}" | jq '{lifecycle_state}'
+echo "$($CLI lakeview get "$DASHBOARD_ID" | jq -r '.etag'):ETAG_2" >> ACC_REPLS
 
 title "Try to redeploy the bundle and confirm that the out of band modification is detected:"
 trace $CLI bundle plan

acceptance/bundle/resources/dashboards/publish-failure-stale-content/dashboard.lvdash.json

@@ -0,0 +1 @@
+{"pages":[{"name":"test-page","displayName":"Test Dashboard"}]}

acceptance/bundle/resources/dashboards/publish-failure-stale-content/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: update-publish-failure-stale-content
+
+resources:
+  dashboards:
+    dashboard1:
+      display_name: my dashboard
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      file_path: ./dashboard.lvdash.json

acceptance/bundle/resources/dashboards/publish-failure-stale-content/output.txt

@@ -0,0 +1,172 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-publish-failure-stale-content/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] lakeview get [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard",
+  "etag": "[ETAG_1]"
+}
+
+>>> [CLI] lakeview get-published [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard"
+}
+
+>>> [CLI] bundle plan -o json
+json.plan.resources.dashboards.dashboard1.remote_state.etag = "[ETAG_1]";
+json.plan.resources.dashboards.dashboard1.remote_state.published = true;
+json.plan.resources.dashboards.dashboard1.changes.etag.action = "skip";
+json.plan.resources.dashboards.dashboard1.changes.etag.reason = "custom";
+json.plan.resources.dashboards.dashboard1.changes.etag.old = "[ETAG_1]";
+json.plan.resources.dashboards.dashboard1.changes.etag.remote = "[ETAG_1]";
+json.plan.resources.dashboards.dashboard1.changes.serialized_dashboard.reason = "etag_based";
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-publish-failure-stale-content/default/files...
+Deploying resources...
+Error: cannot update resources.dashboards.dashboard1: updating id=[DASHBOARD1_ID]: Fault injected by test. (400 INJECTED)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]/published
+HTTP Status: 400 Bad Request
+API error_code: INJECTED
+API message: Fault injected by test.
+
+Updating deployment state...
+
+Exit code: 1
+
+>>> print_requests.py //lakeview/dashboards
+{
+  "method": "PATCH",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]",
+  "body": {
+    "display_name": "my dashboard renamed",
+    "parent_path": "/Workspace/Users/[USERNAME]/.bundle/update-publish-failure-stale-content/default/resources",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n",
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]/published",
+  "body": {
+    "embed_credentials": false,
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+  }
+}
+
+>>> [CLI] lakeview get [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard renamed",
+  "etag": "[ETAG_2]"
+}
+
+>>> [CLI] lakeview get-published [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard"
+}
+
+>>> [CLI] bundle plan -o json
+Warning: dashboard "dashboard1" has been modified remotely
+  at resources.dashboards.dashboard1
+  in databricks.yml:7:7
+
+This dashboard has been modified remotely since the last bundle deployment.
+These modifications are untracked and will be overwritten on deploy.
+
+Make sure that the local dashboard definition matches what you intend to deploy
+before proceeding with the deployment.
+
+To overwrite the remote changes with your local version, use --force.
+The remote modifications will be lost.
+
+json.plan.resources.dashboards.dashboard1.new_state.value.published = true;
+json.plan.resources.dashboards.dashboard1.remote_state.etag = "[ETAG_2]";
+json.plan.resources.dashboards.dashboard1.remote_state.published = true;
+json.plan.resources.dashboards.dashboard1.changes.etag.action = "update";
+json.plan.resources.dashboards.dashboard1.changes.etag.old = "[ETAG_1]";
+json.plan.resources.dashboards.dashboard1.changes.etag.remote = "[ETAG_2]";
+json.plan.resources.dashboards.dashboard1.changes.serialized_dashboard.reason = "etag_based";
+
+>>> [CLI] bundle deploy
+Error: dashboard "dashboard1" has been modified remotely
+  at resources.dashboards.dashboard1
+  in databricks.yml:7:7
+
+This dashboard has been modified remotely since the last bundle deployment.
+These modifications are untracked and will be overwritten on deploy.
+
+Make sure that the local dashboard definition matches what you intend to deploy
+before proceeding with the deployment.
+
+To overwrite the remote changes with your local version, use --force.
+The remote modifications will be lost.
+
+
+Exit code: 1
+
+>>> print_requests.py //lakeview/dashboards
+
+>>> [CLI] lakeview get [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard renamed",
+  "etag": "[ETAG_2]"
+}
+
+>>> [CLI] lakeview get-published [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard"
+}
+
+>>> [CLI] bundle deploy --force
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-publish-failure-stale-content/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //lakeview/dashboards
+{
+  "method": "PATCH",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]",
+  "body": {
+    "display_name": "my dashboard renamed",
+    "parent_path": "/Workspace/Users/[USERNAME]/.bundle/update-publish-failure-stale-content/default/resources",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n",
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD1_ID]/published",
+  "body": {
+    "embed_credentials": false,
+    "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+  }
+}
+
+>>> [CLI] lakeview get [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard renamed",
+  "etag": "[ETAG_3]"
+}
+
+>>> [CLI] lakeview get-published [DASHBOARD1_ID]
+{
+  "display_name": "my dashboard renamed"
+}
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.dashboards.dashboard1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-publish-failure-stale-content/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/dashboards/publish-failure-stale-content/script

@@ -0,0 +1,51 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+unset MSYS_NO_PATHCONV
+
+# First deploy: dashboard is created and published successfully.
+trace $CLI bundle deploy
+replace_ids.py
+DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
+add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_1
+trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
+trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
+trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"
+rm out.requests.txt
+
+# Inject a single publish failure for the update below.
+fault.py "POST /api/2.0/lakeview/dashboards/*" 400 0 1
+
+# Rename the dashboard to trigger an Update.
+update_file.py databricks.yml "my dashboard" "my dashboard renamed"
+
+# Deploy: PATCH succeeds (bumping the remote etag) but publish fails.
+# SaveState is only called on success, so state retains the pre-PATCH etag.
+errcode trace $CLI bundle deploy
+trace print_requests.py //lakeview/dashboards
+add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_2
+trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
+trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
+trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"
+
+# Bug: re-running deploy fails with "modified remotely" because the stored etag
+# (pre-PATCH) no longer matches the remote etag (bumped by the PATCH above).
+# No API writes are attempted — the stale published content is stuck.
+errcode trace $CLI bundle deploy
+trace print_requests.py //lakeview/dashboards
+trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
+trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
+
+# --force bypasses the etag check and forces a full re-deploy (PATCH + POST /published),
+# which fixes the stale published content.
+trace $CLI bundle deploy --force
+trace print_requests.py //lakeview/dashboards
+add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_3
+trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}'
+trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}'
+trace $CLI bundle plan

acceptance/bundle/resources/dashboards/publish-failure-stale-content/test.toml

@@ -0,0 +1,16 @@
+Badness = "after publish failure, re-deploy fails with 'modified remotely' instead of republishing; --force is required as a workaround"
+Cloud = false
+Local = true
+RecordRequests = true
+
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+[[Repls]]
+Old = '\?[ow]=\d+'
+New = "?[WSPARAM]=[NUMID]"