direct: store a dashboard contents digest in state instead of the contents

.nextchanges/bundles/dashboard-state-hash.md

@@ -0,0 +1 @@
+direct: A dashboard's `serialized_dashboard` is no longer persisted in the deployment state. State now records only a `serialized_dashboard_hash` digest of the contents, which is all it was ever compared against, so state files no longer carry a copy of every dashboard's (often multi-MB) JSON. The full contents are still sent to the API on every create and update, and `bundle plan` still reports the real content diff. Existing state is migrated in place: the digest is derived from the contents already in state, so upgrading does not trigger a dashboard update ([#6105](https://github.com/databricks/cli/pull/6105)).

acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json

@@ -12,7 +12,7 @@
     "etag": [ETAG],
     "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources",
     "published": true,
-    "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}",
+    "serialized_dashboard_hash": "sha256_hashed_in_state:[HASH]",
     "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
    }
   }

acceptance/bundle/migrate/dashboards/out.new_state.json

@@ -12,7 +12,7 @@
     "etag": "[NUMID]",
     "parent_path": "/Workspace/Users/[USERNAME]",
     "published": true,
-    "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
+    "serialized_dashboard_hash": "sha256_hashed_in_state:[HASH]",
     "warehouse_id": "123456"
    }
   }

acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json

@@ -49,11 +49,16 @@
         },
         "serialized_dashboard": {
           "action": "skip",
-          "reason": "etag_based",
-          "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
+          "reason": "hashed_in_state",
           "new": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
           "remote": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
         },
+        "serialized_dashboard_hash": {
+          "action": "skip",
+          "reason": "input_only",
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]"
+        },
         "update_time": {
           "action": "skip",
           "reason": "spec:output_only",

acceptance/bundle/refschema/out.fields.txt

@@ -573,6 +573,7 @@ resources.dashboards.*.parent_path	string	ALL
 resources.dashboards.*.path	string	ALL
 resources.dashboards.*.published	bool	REMOTE	STATE
 resources.dashboards.*.serialized_dashboard	any	ALL
+resources.dashboards.*.serialized_dashboard_hash	string	REMOTE	STATE
 resources.dashboards.*.update_time	string	ALL
 resources.dashboards.*.url	string	INPUT
 resources.dashboards.*.warehouse_id	string	ALL

acceptance/bundle/resources/dashboard-state-hash/dashboard.tmpl

@@ -0,0 +1 @@
+{"pages":[{"name":"main","displayName":"$DASH_TITLE"}]}

acceptance/bundle/resources/dashboard-state-hash/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: dashboard-state-sha-$UNIQUE_NAME
+
+resources:
+  dashboards:
+    dashboard1:
+      display_name: $DASHBOARD_DISPLAY_NAME
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      embed_credentials: true
+      file_path: "dashboard.lvdash.json"
+      parent_path: /Users/$CURRENT_USER_NAME

acceptance/bundle/resources/dashboard-state-hash/out.create.serialized.json

@@ -0,0 +1,3 @@
+[
+  "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n"
+]

acceptance/bundle/resources/dashboard-state-hash/out.plan_create.direct.json

@@ -0,0 +1,20 @@
+{
+  "plan_version": 2,
+  "cli_version": "[CLI_VERSION]",
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "display_name": "dashboard-state-hash [UUID]",
+          "embed_credentials": true,
+          "parent_path": "/Workspace/Users/[USERNAME]",
+          "published": true,
+          "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n",
+          "serialized_dashboard_hash": "sha256_hashed_in_state:[HASH]",
+          "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-hash/out.plan_skip.direct.json

@@ -0,0 +1,70 @@
+{
+  "plan_version": 2,
+  "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "skip",
+      "remote_state": {
+        "create_time": "[TIMESTAMP]",
+        "dashboard_id": "[DASHBOARD_ID]",
+        "display_name": "dashboard-state-hash [UUID]",
+        "embed_credentials": true,
+        "etag": [ETAG],
+        "lifecycle_state": "ACTIVE",
+        "parent_path": "/Workspace/Users/[USERNAME]",
+        "path": "/Users/[USERNAME]/dashboard-state-hash [UUID].lvdash.json",
+        "published": true,
+        "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
+        "update_time": "[TIMESTAMP]",
+        "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+      },
+      "changes": {
+        "create_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        },
+        "dashboard_id": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[DASHBOARD_ID]"
+        },
+        "etag": {
+          "action": "skip",
+          "reason": "custom",
+          "old": [ETAG],
+          "remote": [ETAG]
+        },
+        "lifecycle_state": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "ACTIVE"
+        },
+        "path": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "/Users/[USERNAME]/dashboard-state-hash [UUID].lvdash.json"
+        },
+        "serialized_dashboard": {
+          "action": "skip",
+          "reason": "hashed_in_state",
+          "new": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n",
+          "remote": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
+        },
+        "serialized_dashboard_hash": {
+          "action": "skip",
+          "reason": "input_only",
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]"
+        },
+        "update_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-hash/out.plan_update.direct.json

@@ -0,0 +1,80 @@
+{
+  "plan_version": 2,
+  "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "plan": {
+    "resources.dashboards.dashboard1": {
+      "action": "update",
+      "new_state": {
+        "value": {
+          "display_name": "dashboard-state-hash [UUID]",
+          "embed_credentials": true,
+          "parent_path": "/Workspace/Users/[USERNAME]",
+          "published": true,
+          "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n",
+          "serialized_dashboard_hash": "sha256_hashed_in_state:[HASH][0]",
+          "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+        }
+      },
+      "remote_state": {
+        "create_time": "[TIMESTAMP]",
+        "dashboard_id": "[DASHBOARD_ID]",
+        "display_name": "dashboard-state-hash [UUID]",
+        "embed_credentials": true,
+        "etag": [ETAG],
+        "lifecycle_state": "ACTIVE",
+        "parent_path": "/Workspace/Users/[USERNAME]",
+        "path": "/Users/[USERNAME]/dashboard-state-hash [UUID].lvdash.json",
+        "published": true,
+        "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n",
+        "update_time": "[TIMESTAMP]",
+        "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
+      },
+      "changes": {
+        "create_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        },
+        "dashboard_id": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[DASHBOARD_ID]"
+        },
+        "etag": {
+          "action": "skip",
+          "reason": "custom",
+          "old": [ETAG],
+          "remote": [ETAG]
+        },
+        "lifecycle_state": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "ACTIVE"
+        },
+        "path": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "/Users/[USERNAME]/dashboard-state-hash [UUID].lvdash.json"
+        },
+        "serialized_dashboard": {
+          "action": "skip",
+          "reason": "hashed_in_state",
+          "new": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n",
+          "remote": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
+        },
+        "serialized_dashboard_hash": {
+          "action": "update",
+          "old": "sha256_hashed_in_state:[HASH][1]",
+          "new": "sha256_hashed_in_state:[HASH][0]"
+        },
+        "update_time": {
+          "action": "skip",
+          "reason": "spec:output_only",
+          "remote": "[TIMESTAMP]"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/dashboard-state-hash/out.state.direct.txt

@@ -0,0 +1 @@
+json.state.resources.dashboards.dashboard1.state.serialized_dashboard_hash = "sha256_hashed_in_state:[HASH]";

acceptance/bundle/resources/dashboard-state-hash/out.state_after_update.direct.txt

@@ -0,0 +1 @@
+json.state.resources.dashboards.dashboard1.state.serialized_dashboard_hash = "sha256_hashed_in_state:[HASH]";

acceptance/bundle/resources/dashboard-state-hash/out.update.serialized.json

@@ -0,0 +1,3 @@
+[
+  "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n"
+]

acceptance/bundle/resources/dashboard-state-hash/output.txt

@@ -0,0 +1,26 @@
+
+=== Create: plan and state keep the content digest; the API receives the full content
+>>> [CLI] bundle plan -o json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Re-plan with no local change is a no-op (server normalization is ignored, etag_based)
+>>> [CLI] bundle plan -o json
+
+=== Edit serialized_dashboard: the digest changes, so the local diff detects an update
+>>> [CLI] bundle plan -o json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.dashboards.dashboard1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/dashboard-state-hash/script

@@ -0,0 +1,60 @@
+DASHBOARD_DISPLAY_NAME="dashboard-state-hash $(uuid)"
+if [ -z "$CLOUD_ENV" ]; then
+    export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234"
+    echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS
+fi
+export DASHBOARD_DISPLAY_NAME
+envsubst < databricks.yml.tmpl > databricks.yml
+
+render_dashboard() {
+    # Strip CR so the rendered content is LF on every platform. dashboard.tmpl is not pinned to
+    # eol=lf, so on Windows it renders with a CRLF trailing newline; since the content is digested
+    # into serialized_dashboard_hash, a CRLF here would digest differently than the LF content in a
+    # saved plan and make the no-op re-plan (READPLAN=1) spuriously report an update.
+    DASH_TITLE="$1" envsubst < dashboard.tmpl | tr -d '\r' > dashboard.lvdash.json
+}
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+rm -f out.requests.txt
+
+title "Create: plan and state keep the content digest; the API receives the full content"
+render_dashboard "Sales Overview"
+trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+
+# Deploy. With READPLAN=1 this applies the SAVED plan file instead of re-planning. The
+# persisted state keeps only the digest, but the create API call must still send the FULL
+# serialized_dashboard. out.create.serialized.json is identical for READPLAN="" and
+# READPLAN=1, which proves the saved plan carries and applies the real content.
+# Not traced: the deploy command line differs by READPLAN (--plan vs none).
+$CLI bundle deploy $(readplanarg out.plan_create.direct.json)
+
+DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
+echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS
+
+jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.create.serialized.json
+rm -f out.requests.txt
+
+# Persisted state holds ONLY the content digest, never the dashboard JSON.
+print_state.py | gron.py | grep serialized_dashboard > out.state.$DATABRICKS_BUNDLE_ENGINE.txt
+
+title "Re-plan with no local change is a no-op (server normalization is ignored, etag_based)"
+trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json
+
+title "Edit serialized_dashboard: the digest changes, so the local diff detects an update"
+render_dashboard "Sales Overview v2"
+trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
+
+# Apply the update (in-memory or from the saved plan). The update API call must carry the
+# FULL new content; out.update.serialized.json is identical across READPLAN, proving the
+# saved plan applies the real content for updates too.
+$CLI bundle deploy $(readplanarg out.plan_update.direct.json)
+
+jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.update.serialized.json
+rm -f out.requests.txt
+
+# State holds the NEW content digest after the update.
+print_state.py | gron.py | grep serialized_dashboard > out.state_after_update.$DATABRICKS_BUNDLE_ENGINE.txt

acceptance/bundle/resources/dashboard-state-hash/test.toml

@@ -0,0 +1,24 @@
+# Direct-engine state format for dashboards: state persists serialized_dashboard_hash and
+# never the contents. Kept outside dashboards/ so it can run direct-only: terraform stores the
+# contents in full, and the saved-plan path (deploy --plan, i.e. READPLAN=1) is direct-only.
+# RecordRequests is inherited from resources/test.toml.
+# Local-only: the in-memory testserver covers this state format fully and deterministically. On
+# real cloud the read-after-deploy plan/state reads hit dashboard eventual consistency, and unlike
+# the tests under dashboards/ this dir doesn't inherit their retry/INJECT_STALE machinery.
+Local = true
+Cloud = false
+
+# dashboard.lvdash.json is rendered from dashboard.tmpl in the script (re-rendered for the edit).
+Ignore = ["dashboard.lvdash.json"]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.READPLAN = ["", "1"]
+
+# Etag can be both negative and positive.
+[[Repls]]
+Old = "\"[-0-9]{8,}\""
+New = "[ETAG]"
+
+[[Repls]]
+Old = "\"[0-9]{8,}\""
+New = "[ETAG]"

acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json

@@ -64,6 +64,12 @@
           "old": "[SERIALIZED_FIXTURE_OLD]",
           "new": "[SERIALIZED_FIXTURE_NEW]"
         },
+        "serialized_dashboard_hash": {
+          "action": "skip",
+          "reason": "input_only",
+          "old": "sha256_hashed_in_state:[HASH][0]",
+          "new": "sha256_hashed_in_state:[HASH][0]"
+        },
         "update_time": {
           "action": "skip",
           "reason": "spec:output_only",

acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json

@@ -13,6 +13,7 @@
           "parent_path": "/Workspace/Users/[USERNAME]/.bundle/delete-trashed-out-of-band-[UNIQUE_NAME]/default/resources",
           "published": true,
           "serialized_dashboard": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n",
+          "serialized_dashboard_hash": "sha256_hashed_in_state:[HASH]",
           "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]"
         }
       },
@@ -21,6 +22,11 @@
           "action": "update",
           "reason": "custom",
           "old": "[ETAG]"
+        },
+        "serialized_dashboard": {
+          "action": "skip",
+          "reason": "hashed_in_state",
+          "new": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n"
         }
       }
     }