internal: Refactor immutable folder upload as an internal resource in direct engine

acceptance/bundle/deploy/immutable-no-artifacts/.gitignore

@@ -0,0 +1,2 @@
+*.txt
+*.json

acceptance/bundle/deploy/immutable-no-artifacts/output.txt

@@ -8,8 +8,47 @@ Workspace:
 
 Validation OK!
 
+>>> [CLI] bundle plan -o json
+
+>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json
+[
+  {
+    "notebook_task": {
+      "base_parameters": {
+        "path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path"
+      },
+      "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
+    },
+    "task_key": "notebook_task"
+  },
+  {
+    "environment_key": "env",
+    "spark_python_task": {
+      "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
+    },
+    "task_key": "spark_python_task"
+  }
+]
+
+>>> jq .plan["resources.snapshots.immutable"] plan.json
+{
+  "action": "create",
+  "new_state": {
+    "value": {
+      "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "bundle_id": "[UUID]",
+      "acl": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ]
+    }
+  }
+}
+
 >>> [CLI] bundle deploy
-Uploading immutable bundle snapshot...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -23,6 +62,52 @@ Deployment complete!
 >>> [CLI] jobs get [NUMID]
 "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path"
 
+>>> title Check snapshot plan after deploy
+
+=== Check snapshot plan after deploy
+>>> [CLI] bundle plan -o json
+
+>>> jq .plan["resources.snapshots.immutable"] plan.json
+{
+  "action": "skip",
+  "remote_state": {
+    "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+    "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]"
+  },
+  "changes": {
+    "acl": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "new": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "remote": null
+    },
+    "bundle_id": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "[UUID]",
+      "new": "[UUID]",
+      "remote": ""
+    }
+  }
+}
+
+>>> [CLI] bundle deploy
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+\nSnapshot did not change as expected
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.jobs.my_job

acceptance/bundle/deploy/immutable-no-artifacts/script

@@ -6,13 +6,31 @@ cleanup() {
 trap cleanup EXIT
 
 trace $CLI bundle validate
-trace $CLI bundle deploy
+trace $CLI bundle plan -o json > plan.json
+trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json
+trace jq '.plan["resources.snapshots.immutable"]' plan.json
 
+trace $CLI bundle deploy
 
 # Get a job and check that task paths point into the snapshot
 JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id')
 trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file'
 trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path'
 trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path'
+PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file')
+
+trace title "Check snapshot plan after deploy"
+trace $CLI bundle plan -o json > plan.json
+trace jq '.plan["resources.snapshots.immutable"]' plan.json
+rm plan.json
+
+trace $CLI bundle deploy
 
+PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file')
+if [ "$PYTHON_FILE_1" == "$PYTHON_FILE_2" ]; then
+    echo "\nSnapshot did not change as expected"
+else
+    echo "\nSnapshot changed"
+    exit 1
+fi
 trace $CLI bundle destroy --auto-approve

acceptance/bundle/deploy/immutable-permissions-change/output.txt

@@ -1,7 +1,6 @@
 
 === Deploy without permissions
 >>> [CLI] bundle deploy
-Uploading immutable bundle snapshot...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -21,7 +20,6 @@ Consider using a adding a top-level permissions section such as the following:
 See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration.
   in databricks.yml:22:3
 
-Uploading immutable bundle snapshot...
 Deploying resources...
 Updating deployment state...
 Deployment complete!

acceptance/bundle/deploy/immutable/output.txt

@@ -10,10 +10,12 @@ Validation OK!
 
 >>> [CLI] bundle plan -o json
 Building python_artifact...
+
+>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json
 [
   {
     "notebook_task": {
-      "notebook_path": "${workspace.snapshot_path}/files/src/notebook"
+      "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
     },
     "task_key": "notebook_task"
   },
@@ -28,15 +30,45 @@ Building python_artifact...
   {
     "environment_key": "env",
     "spark_python_task": {
-      "python_file": "${workspace.snapshot_path}/files/src/main.py"
+      "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
     },
     "task_key": "spark_python_task"
   }
 ]
 
+>>> jq .plan["resources.jobs.my_job"].new_state.value.environments plan.json
+[
+  {
+    "environment_key": "env",
+    "spec": {
+      "dependencies": [
+        "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
+      ],
+      "environment_version": "4"
+    }
+  }
+]
+
+>>> jq .plan["resources.snapshots.immutable"] plan.json
+{
+  "action": "create",
+  "new_state": {
+    "value": {
+      "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "bundle_id": "[UUID]",
+      "acl": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ]
+    }
+  }
+}
+
 >>> [CLI] bundle deploy
 Building python_artifact...
-Uploading immutable bundle snapshot...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -52,6 +84,82 @@ Deployment complete!
   "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
 ]
 
+>>> title Check snapshot plan after deploy
+
+=== Check snapshot plan after deploy
+>>> [CLI] bundle plan -o json
+Building python_artifact...
+
+>>> jq .plan["resources.snapshots.immutable"] plan.json
+{
+  "action": "recreate",
+  "new_state": {
+    "value": {
+      "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "bundle_id": "[UUID]",
+      "acl": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ]
+    }
+  },
+  "remote_state": {
+    "relative_path": "[UUID]/[SNAPSHOT_HASH]",
+    "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]"
+  },
+  "changes": {
+    "acl": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "new": [
+        {
+          "user_name": "[USERNAME]",
+          "permission_level": "CAN_READ"
+        }
+      ],
+      "remote": null
+    },
+    "bundle_id": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": "[UUID]",
+      "new": "[UUID]",
+      "remote": ""
+    },
+    "full_path": {
+      "action": "recreate",
+      "reason": "immutable",
+      "old": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "new": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]",
+      "remote": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]"
+    },
+    "relative_path": {
+      "action": "recreate",
+      "reason": "immutable",
+      "old": "[UUID]/[SNAPSHOT_HASH]",
+      "new": "[UUID]/[SNAPSHOT_HASH]",
+      "remote": "[UUID]/[SNAPSHOT_HASH]"
+    }
+  }
+}
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Check that job uses a new snapshot\nSnapshot changed as expected
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.jobs.my_job

acceptance/bundle/deploy/immutable/script

@@ -5,15 +5,35 @@ cleanup() {
 trap cleanup EXIT
 
 trace $CLI bundle validate
-trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks'
-trace $CLI bundle deploy
+trace $CLI bundle plan -o json > plan.json
+trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json
+trace jq '.plan["resources.jobs.my_job"].new_state.value.environments' plan.json
+trace jq '.plan["resources.snapshots.immutable"]' plan.json
 
+trace $CLI bundle deploy
 
 # Get a job and check that task paths are immutable
 JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id')
 trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file'
 trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path'
 trace $CLI jobs get $JOB_ID | jq '.settings.environments[0].spec.dependencies'
+PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file')
+
+trace title "Check snapshot plan after deploy"
+trace $CLI bundle plan -o json > plan.json
+trace jq '.plan["resources.snapshots.immutable"]' plan.json
+rm plan.json
+
+trace $CLI bundle deploy
+
+title "Check that job uses a new snapshot"
+PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file')
+if [ "$PYTHON_FILE_1" != "$PYTHON_FILE_2" ]; then
+    echo "\nSnapshot changed as expected"
+else
+    echo "\nSnapshot did not change"
+    exit 1
+fi
 
 # Redirect run output to a log file — the real workspace produces different output than the local test server.
 $CLI bundle run my_job &> LOG.run

acceptance/bundle/refschema/out.fields.txt

@@ -3473,6 +3473,17 @@ resources.secret_scopes.*.permissions.acls[*]	workspace.AclItem	ALL
 resources.secret_scopes.*.permissions.acls[*].permission	workspace.AclPermission	ALL
 resources.secret_scopes.*.permissions.acls[*].principal	string	ALL
 resources.secret_scopes.*.permissions.scope_name	string	ALL
+resources.snapshots.*.acl	[]snapshot.ACLEntry	INPUT	STATE
+resources.snapshots.*.acl[*]	snapshot.ACLEntry	INPUT	STATE
+resources.snapshots.*.acl[*].group_name	string	INPUT	STATE
+resources.snapshots.*.acl[*].permission_level	string	INPUT	STATE
+resources.snapshots.*.acl[*].service_principal_name	string	INPUT	STATE
+resources.snapshots.*.acl[*].user_name	string	INPUT	STATE
+resources.snapshots.*.bundle_id	string	INPUT	STATE
+resources.snapshots.*.full_path	string	REMOTE	STATE
+resources.snapshots.*.relative_path	string	REMOTE	STATE
+resources.snapshots.*.remote_root	string	INPUT
+resources.snapshots.*.zip_content	string	INPUT
 resources.sql_warehouses.*.auto_stop_mins	int	ALL
 resources.sql_warehouses.*.channel	*sql.Channel	ALL
 resources.sql_warehouses.*.channel.dbsql_version	string	ALL

acceptance/bundle/resources/apps/immutable/output.txt

@@ -1,6 +1,5 @@
 
 >>> [CLI] bundle deploy
-Uploading immutable bundle snapshot...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -37,6 +36,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com
   "path": "/api/2.0/apps/my-immutable-app/deployments",
   "body": {
     "mode": "SNAPSHOT",
-    "source_code_path": "${workspace.snapshot_path}/files/app"
+    "source_code_path": "${resources.snapshots.immutable.full_path}/files/app"
   }
 }