diff --git a/.wsignore b/.wsignore
index fb58cddade1..58d4349dd45 100644
--- a/.wsignore
+++ b/.wsignore
@@ -25,3 +25,6 @@ python/databricks/bundles/*/_models/*.py
# Developed elsewhere:
internal/genkit/tagging.py
+
+# Vendored from https://github.com/databricks/mlops-stacks:
+acceptance/bundle/deploy/mlops-stacks/template/**
diff --git a/Taskfile.yml b/Taskfile.yml
index 1eb41d6ac53..cefe9986faf 100644
--- a/Taskfile.yml
+++ b/Taskfile.yml
@@ -598,6 +598,15 @@ tasks:
- task: test-update
- task: test-update-aws
+ # Follows upstream HEAD, so its result changes over time: keep it out of
+ # `generate-check`, which requires byte-for-byte reproducible output.
+ bump-mlops-stacks:
+ desc: Bump the vendored mlops-stacks acceptance template to github.com/databricks/mlops-stacks HEAD
+ cmds:
+ - "./tools/bump_mlops_stacks.py"
+ # The bump rewrites a test input, so refresh that test's golden output.
+ - "go test ./acceptance -run '^TestAccept/bundle/deploy/mlops-stacks' -update -timeout=${LOCAL_TIMEOUT:-60m}"
+
slowest:
desc: Show 50 slowest tests from last run
cmds:
diff --git a/acceptance/bundle/deploy/mlops-stacks/out.test.toml b/acceptance/bundle/deploy/mlops-stacks/out.test.toml
index 2d812727e32..e849ec85ace 100644
--- a/acceptance/bundle/deploy/mlops-stacks/out.test.toml
+++ b/acceptance/bundle/deploy/mlops-stacks/out.test.toml
@@ -1,4 +1,4 @@
-Local = false
+Local = true
Cloud = true
RequiresUnityCatalog = true
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
diff --git a/acceptance/bundle/deploy/mlops-stacks/script b/acceptance/bundle/deploy/mlops-stacks/script
index 9ce02d5fd60..f69ef7c9fca 100644
--- a/acceptance/bundle/deploy/mlops-stacks/script
+++ b/acceptance/bundle/deploy/mlops-stacks/script
@@ -29,7 +29,13 @@ trap cleanup EXIT
trace $CLI schemas create "${SCHEMA_NAME}" main | jq '{full_name}'
-trace $CLI bundle init mlops-stacks --config-file config.json
+# Local runs must not reach the network; cloud runs clone to keep that path covered.
+# The vendored copy in ./template/ can be updated with `task bump-mlops-stacks`
+if [ -z "$CLOUD_ENV" ]; then
+ trace $CLI bundle init "$TESTDIR/template" --config-file config.json
+else
+ trace $CLI bundle init mlops-stacks --config-file config.json
+fi
trace cat test_repo_mlops_stacks/README.md | head -n 4
cd "test_repo_mlops_stacks/project_name_${UNIQUE_NAME}" || exit 1
diff --git a/acceptance/bundle/deploy/mlops-stacks/template.REVISION b/acceptance/bundle/deploy/mlops-stacks/template.REVISION
new file mode 100644
index 00000000000..c4469c21626
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template.REVISION
@@ -0,0 +1 @@
+9b387898ad10439c90581e6064f19407121dc410
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/databricks_template_schema.json b/acceptance/bundle/deploy/mlops-stacks/template/databricks_template_schema.json
new file mode 100644
index 00000000000..ec840054ecc
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/databricks_template_schema.json
@@ -0,0 +1,266 @@
+{
+ "welcome_message": "Welcome to MLOps Stacks. For detailed information on project generation, see the README at https://github.com/databricks/mlops-stacks/blob/main/README.md.",
+ "min_databricks_cli_version": "v0.236.0",
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "order": 1,
+ "type": "string",
+ "description": "{{if false}}\n\nERROR: This template is not supported by your current Databricks CLI version.\nPlease hit control-C and go to https://docs.databricks.com/en/dev-tools/cli/install.html for instructions on upgrading the CLI to the minimum version supported by MLOps Stacks.\n\n\n{{end}}\nSelect if both CI/CD and the Project should be set up, or only one of them.\nYou can always set up the other later by running initialization again",
+ "default": "CICD_and_Project",
+ "enum": ["CICD_and_Project", "Project_Only", "CICD_Only"]
+ },
+ "input_project_name": {
+ "order": 2,
+ "type": "string",
+ "default": "my_mlops_project",
+ "description": "\nProject Name. Default",
+ "pattern": "^[^ .\\\\/]{3,}$",
+ "pattern_match_failure_message": "Project name must be at least 3 characters long and cannot contain the following characters: \"\\\", \"/\", \" \" and \".\".",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_Only"
+ }
+ }
+ }
+ },
+ "input_root_dir": {
+ "order": 3,
+ "type": "string",
+ "default": "{{ .input_project_name }}",
+ "description": "\nRoot directory name.\nFor monorepos, name of the root directory that contains all the projects.\nDefault",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_and_Project"
+ }
+ }
+ }
+ },
+ "input_cloud": {
+ "order": 4,
+ "type": "string",
+ "description": "\nSelect cloud",
+ "default": "azure",
+ "enum": ["azure", "aws", "gcp"]
+ },
+ "input_cicd_platform": {
+ "order": 5,
+ "type": "string",
+ "description": "\nSelect CICD platform",
+ "default": "github_actions",
+ "enum": ["github_actions", "github_actions_for_github_enterprise_servers", "azure_devops", "gitlab"],
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_databricks_staging_workspace_host": {
+ "order": 6,
+ "type": "string",
+ "default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-staging-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-staging-workspace.gcp.databricks.com{{end}}",
+ "description": "\nURL of staging Databricks workspace,\nIt will run PR CI and preview changes before they're deployed to production.\nDefault",
+ "pattern": "^(https.*)?$",
+ "pattern_match_failure_message": "Databricks staging workspace host URLs must start with https. Got invalid workspace host.",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_databricks_prod_workspace_host": {
+ "order": 7,
+ "type": "string",
+ "default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-prod-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-prod-workspace.gcp.databricks.com{{end}}",
+ "description": "\nURL of production Databricks workspace.\nDefault",
+ "pattern": "^(https.*)?$",
+ "pattern_match_failure_message": "Databricks production workspace host URLs must start with https. Got invalid workspace host.",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_default_branch": {
+ "order": 8,
+ "type": "string",
+ "default": "main",
+ "description": "\nName of the default branch,\nStaging resources are deployed from this branch and stages the latest ML code.\nDefault",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_release_branch": {
+ "order": 9,
+ "type": "string",
+ "default": "release",
+ "description": "\nName of the release branch.\nThe training and other production jobs pull ML code from this branch.\nDefault",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_read_user_group": {
+ "order": 10,
+ "type": "string",
+ "default": "users",
+ "description": "\nUser group name to give READ permissions to for project resources\n(ML jobs, integration test job runs, and machine learning resources).\nA group with this name must exist in both the staging and prod workspaces.\nDefault",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_Only"
+ }
+ }
+ }
+ },
+ "input_staging_catalog_name": {
+ "order": 12,
+ "type": "string",
+ "description": "\nName of the catalog in Unity Catalog that will host the staging UC resources. \nThis catalog must already exist and service principals must have access to it.\nDefault",
+ "default": "staging",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_prod_catalog_name": {
+ "order": 13,
+ "type": "string",
+ "description": "\nName of the catalog in Unity Catalog that will host the production UC resources.\nThis catalog must already exist and service principals must have access to it.\nDefault",
+ "default": "prod",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_test_catalog_name": {
+ "order": 14,
+ "type": "string",
+ "description": "\nName of the catalog in Unity Catalog that will be used for integration tests.\nThis catalog must already exist and service principals must have access to it.\nDefault",
+ "default": "test",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ },
+ "input_schema_name": {
+ "order": 15,
+ "type": "string",
+ "description": "\nName of schema to use when registering a model in Unity Catalog.\nThis schema must already exist and service principals must have access.\nWe recommend using the project name.\nDefault",
+ "default": "{{ .input_project_name }}",
+ "pattern": "^[^ .\\-\\/]*$",
+ "pattern_match_failure_message": "Valid schema names cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_Only"
+ }
+ }
+ }
+ },
+ "input_unity_catalog_read_user_group": {
+ "order": 16,
+ "type": "string",
+ "default": "account users",
+ "description": "\nUser group name to give EXECUTE privileges to models in Unity Catalog (UC).\nIt must exist in UC with access granted to the staging and prod workspaces.\nDefault",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_Only"
+ }
+ }
+ }
+ },
+ "input_inference_table_name": {
+ "order": 17,
+ "type": "string",
+ "description": "\nFully qualified name of inference table to attach monitoring to.\nThis table must already exist and service principals must have access.",
+ "default": "dev.{{ .input_project_name }}.predictions",
+ "pattern": "^[^ .\\-\\/]+(\\.[^ .\\-\\/]+){2}$",
+ "pattern_match_failure_message": "Fully qualified Unity Catalog table names must have catalog, schema, and table separated by \".\" and each cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_Only"
+ }
+ }
+ }
+ },
+ "input_include_feature_store": {
+ "order": 18,
+ "type": "string",
+ "description": "\nWhether to include Feature Store",
+ "default": "no",
+ "enum": ["no", "yes"],
+ "skip_prompt_if": {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "CICD_Only"
+ }
+ }
+ }
+ },
+ "input_docker_image": {
+ "order": 20,
+ "type": "string",
+ "description": "\nDocker image for the execution of Gitlab pipelines",
+ "default": "databricksfieldeng/mlopsstacks:latest",
+ "skip_prompt_if": {
+ "anyOf":[
+ {
+ "properties": {
+ "input_cicd_platform": {
+ "const": "github_actions"
+ }
+ }
+ },
+ {
+ "properties": {
+ "input_cicd_platform": {
+ "const": "github_actions_for_github_enterprise_servers"
+ }
+ }
+ },
+ {
+ "properties": {
+ "input_cicd_platform": {
+ "const": "azure_devops"
+ }
+ }
+ },
+ {
+ "properties": {
+ "input_setup_cicd_and_project": {
+ "const": "Project_Only"
+ }
+ }
+ }
+ ]
+ }
+ }
+ },
+ "success_message" : "\n*** Your MLOps Stack has been created in the '{{.input_root_dir}}{{if not (eq .input_setup_cicd_and_project `CICD_Only`) }}/{{.input_project_name}}{{end}}' directory! ***\n\nPlease refer to the README.md for further instructions on getting started."
+}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/library/functions.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/library/functions.tmpl
new file mode 100644
index 00000000000..9d66c9e9ff4
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/library/functions.tmpl
@@ -0,0 +1,26 @@
+# define functions
+{{ define "get_host" -}}
+ {{- with url . -}}
+ {{- print .Scheme `://` .Host -}}
+ {{- end -}}
+{{- end }}
+
+{{ define "generate_doc_link" -}}
+ {{- if eq .cloud `aws` -}}
+ {{- if eq .path `repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later` -}}
+ https://docs.databricks.com/repos/git-operations-with-repos.html#add-a-repo-connected-to-a-remote-repo
+ {{- else -}}
+ https://docs.databricks.com/{{ print .path }}
+ {{- end -}}
+ {{- else if eq .cloud `gcp` -}}
+ {{- if eq .path `repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later` -}}
+ https://docs.gcp.databricks.com/en/repos/git-operations-with-repos.html#clone-a-repo-connected-to-a-remote-repo
+ {{- else -}}
+ https://docs.gcp.databricks.com/{{ print .path }}
+ {{- end -}}
+ {{- else if eq .cloud `azure` -}}
+ https://learn.microsoft.com/azure/databricks/{{ (regexp `\.html`).ReplaceAllString (print .path) `` }}
+ {{- else -}}
+ {{ fail `Invalid selection of cloud in function generate_doc_link. Please choose from [azure, aws, gcp]` }}
+ {{- end -}}
+{{- end }}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/library/input_validation.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/library/input_validation.tmpl
new file mode 100644
index 00000000000..2f54565f956
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/library/input_validation.tmpl
@@ -0,0 +1,2 @@
+{{ define `validation` }}
+{{- end -}}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/library/template_variables.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/library/template_variables.tmpl
new file mode 100644
index 00000000000..1cd95cd55b0
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/library/template_variables.tmpl
@@ -0,0 +1,64 @@
+# define template variables
+{{ define `databricks_prod_workspace_host` -}}
+ {{- if not (eq (print .input_databricks_prod_workspace_host) "") -}}
+ {{- with url (print .input_databricks_prod_workspace_host) -}}
+ {{ print .Scheme `://` .Host }}
+ {{- end -}}
+ {{- else -}}
+ {{- if eq .input_cloud `azure` -}}
+ https://adb-xxxx.xx.azuredatabricks.net
+ {{- else if eq .input_cloud `aws` -}}
+ https://your-prod-workspace.cloud.databricks.com
+ {{- else if eq .input_cloud `gcp` -}}
+ https://your-prod-workspace.gcp.databricks.com
+ {{- end -}}
+ {{- end -}}
+{{- end }}
+
+{{ define `databricks_staging_workspace_host` -}}
+ {{- if not (eq (print .input_databricks_staging_workspace_host) "") -}}
+ {{- with url (print .input_databricks_staging_workspace_host) -}}
+ {{- print .Scheme `://` .Host -}}
+ {{- end -}}
+ {{- else -}}
+ {{- if eq .input_cloud `azure` -}}
+ https://adb-xxxx.xx.azuredatabricks.net
+ {{- else if eq .input_cloud `aws` -}}
+ https://your-staging-workspace.cloud.databricks.com
+ {{- else if eq .input_cloud `gcp` -}}
+ https://your-staging-workspace.gcp.databricks.com
+ {{- end -}}
+ {{- end -}}
+{{- end }}
+
+{{ define `cloud_specific_node_type_id` -}}
+ {{- if (eq .input_cloud `aws`) -}}
+ i3.xlarge
+ {{- else if (eq .input_cloud `azure`) -}}
+ Standard_D3_v2
+ {{- else if (eq .input_cloud `gcp`) -}}
+ n2-highmem-4
+ {{- else -}}
+ {{ fail `Unknown cloud platform` }}
+ {{- end -}}
+{{- end -}}
+
+{{ define `model_name` -}}
+ {{ .input_project_name }}-model
+{{- end }}
+
+{{ define `experiment_base_name` -}}
+ {{ .input_project_name }}-experiment
+{{- end }}
+
+{{ define `project_name_alphanumeric_underscore` -}}
+ {{- (regexp `-`).ReplaceAllString ((regexp `[^A-Za-z0-9_-]`).ReplaceAllString (print .input_project_name) ``) `_` -}}
+{{- end }}
+
+{{ define `cli_version` -}}
+ v0.236.0
+{{- end }}
+
+{{ define `stacks_version` -}}
+ 0.5
+{{- end }}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/run_validations.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/run_validations.tmpl
new file mode 100644
index 00000000000..45a82b8eb1b
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/run_validations.tmpl
@@ -0,0 +1 @@
+run validation{{ template `validation` . }}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/update_layout.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/update_layout.tmpl
new file mode 100644
index 00000000000..e1aee22344b
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/update_layout.tmpl
@@ -0,0 +1,48 @@
+# Remove unrelated CICD platform files
+{{ $project_name_alphanumeric_underscore := (regexp `-`).ReplaceAllString ((regexp `[^A-Za-z0-9_-]`).ReplaceAllString .input_project_name ``) `_` -}}
+{{ $root_dir := .input_root_dir}}
+
+{{ if (eq .input_setup_cicd_and_project `Project_Only`) }}
+ {{ skip (printf `%s/%s` $root_dir `.azure`) }}
+ {{ skip (printf `%s/%s` $root_dir `.github`) }}
+ {{ skip (printf `%s/%s` $root_dir `docs`) }}
+ {{ skip (printf `%s/%s` $root_dir `.gitignore`) }}
+ {{ skip (printf `%s/%s` $root_dir `README.md`) }}
+ {{ skip (printf `%s/%s` $root_dir `test-requirements.txt`) }}
+ {{ skip (printf `%s/%s` $root_dir `cicd.tar.gz`) }}
+ {{ skip (printf `%s/%s` $root_dir `cicd_params.json`) }}
+{{ else if (eq .input_setup_cicd_and_project `CICD_Only`) }}
+ {{ skip (printf `%s/%s` $root_dir $project_name_alphanumeric_underscore) }}
+ {{ skip (printf `%s/%s/%s-*` $root_dir `.github/workflows` .input_project_name) }}
+ {{ skip (printf `%s/%s/%s-*` $root_dir `.azure/devops-pipelines` .input_project_name) }}
+{{ end }}
+
+{{ if (or (eq .input_cicd_platform `github_actions`) (eq .input_cicd_platform `github_actions_for_github_enterprise_servers`)) }}
+ {{ skip (printf `%s/%s` $root_dir `.azure`) }}
+ {{ skip (printf `%s/%s` $root_dir `.gitlab`) }}
+{{ else if (eq .input_cicd_platform `azure_devops`) }}
+ {{ skip (printf `%s/%s` $root_dir `.github`) }}
+ {{ skip (printf `%s/%s` $root_dir `.gitlab`) }}
+{{ else if (eq .input_cicd_platform `gitlab`) }}
+ {{ skip (printf `%s/%s` $root_dir `.github`) }}
+ {{ skip (printf `%s/%s` $root_dir `.azure`) }}
+{{ end }}
+
+{{ if (not (eq .input_project_name `27896cf3-bb3e-476e-8129-96df0406d5c7`)) }}
+ {{ skip (printf `%s/%s` $root_dir `_params_testing_only.txt`) }}
+{{ end }}
+
+# Remove Feature Store code if not selected; remove Delta Train notebook if Feature Store is selected
+{{ if (eq .input_include_feature_store `yes`) }}
+ {{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `training/Train.py`) }}
+{{ else }}
+ {{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `feature_engineering`) }}
+ {{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `tests/feature_engineering`) }}
+ {{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `training/TrainWithFeatureStore.py`) }}
+ {{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/feature-engineering-workflow-resource.yml`) }}
+{{ end }}
+
+# Remove template files
+{{ skip (printf `%s/%s` $root_dir `cicd`) }}
+{{ skip `update_layout` }}
+{{ skip `run_validations` }}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/README.md.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/README.md.tmpl
new file mode 100644
index 00000000000..8c991e8a714
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/README.md.tmpl
@@ -0,0 +1,8 @@
+# CI/CD Workflow Definitions
+This directory contains CI/CD workflow definitions using [GitHub Actions](https://docs.github.com/en/actions),
+under ``workflows``. These workflows cover testing and deployment of both ML code (for model training, batch inference, etc) and
+Databricks ML resource definitions.
+
+To set up CI/CD for a new project,
+please refer to [Setting up CI/CD](<../../README.md#Setting up CI/CD>) and following the [MLOps Setup Guide](../../docs/mlops-setup.md)
+to make sure your `WORKFLOW_TOKEN` secret has been properly set up with `Workflow` permissions.
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/deploy-cicd.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/deploy-cicd.yml.tmpl
new file mode 100644
index 00000000000..845131813c7
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/deploy-cicd.yml.tmpl
@@ -0,0 +1,78 @@
+# This GitHub workflow sets up CI/CD workflows for a previously instantiated MLOps Stacks Project.
+name: Deploy CICD for Project Workflow
+
+on:
+ workflow_dispatch:
+ inputs:
+ project_name:
+ description: 'Project Name'
+ required: true
+ default: {{ .input_project_name }}
+
+env:
+ DATABRICKS_HOST: {{template `databricks_staging_workspace_host` .}}
+ {{ if (eq .input_cloud `azure`) -}}
+ ARM_TENANT_ID: {{`${{ secrets.STAGING_AZURE_SP_TENANT_ID }}`}}
+ ARM_CLIENT_ID: {{`${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }}`}}
+ ARM_CLIENT_SECRET: {{`${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }}`}}
+ {{ else -}}
+ DATABRICKS_TOKEN: {{`${{ secrets.STAGING_WORKSPACE_TOKEN }}`}}
+ {{- end }}
+
+jobs:
+ cicd:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Get current timestamp
+ id: timestamp
+ run: |
+ echo "timestamp=$(date +'%s')" >> "$GITHUB_ENV"
+ - uses: actions/checkout@v4
+ with:
+ ref: {{`${{ github.event.pull_request.head.sha || github.sha }}`}}
+ token: {{`${{ secrets.WORKFLOW_TOKEN }}`}}
+ - uses: databricks/setup-cli@{{template `cli_version` .}}
+ - name: Convert Project Name to Only Alphanumeric Characters
+ run: |
+ PROJECT_NAME_ALPHA=$(echo "{{`${{ github.event.inputs.project_name }}`}}" | tr ' -' '_')
+ echo "PROJECT_NAME_ALPHA=$PROJECT_NAME_ALPHA" >> "$GITHUB_ENV"
+ - name: Install jq
+ run: sudo apt-get install jq
+ - name: Unzip Bundle and Append Parameters to Input JSON
+ id: unzip
+ run: |
+ tar -xzvf cicd.tar.gz
+ INPUT_CLOUD_1=$(jq -r '.input_cloud' cicd_params.json)
+ INPUT_CLOUD_2=$(jq -r '.input_cloud' "$PROJECT_NAME_ALPHA/project_params.json")
+ if [ "$INPUT_CLOUD_1" != "$INPUT_CLOUD_2" ]; then
+ printf "Error: CICD cloud '%s' does not match project cloud '%s'\n" "$INPUT_CLOUD_1" "$INPUT_CLOUD_2"
+ exit 1
+ fi
+ printf '%s' "$(jq '. += {"input_project_name":"{{`${{ github.event.inputs.project_name }}`}}"}' cicd_params.json)" > cicd_params.json
+ printf '%s' "$(jq -s '.[0] + .[1]' cicd_params.json "$PROJECT_NAME_ALPHA/project_params.json")" > cicd_params.json
+ - name: Update databricks.yml
+ id: update
+ run: |
+ echo -e " staging:\n variables:\n catalog_name: {{ .input_staging_catalog_name }}\n workspace:\n host: {{template `databricks_staging_workspace_host` .}}\n\n prod:\n variables:\n catalog_name: {{ .input_prod_catalog_name }}\n workspace:\n host: {{template `databricks_prod_workspace_host` .}}\n\n test:\n variables:\n catalog_name: {{ .input_test_catalog_name }}\n workspace:\n host: {{template `databricks_staging_workspace_host` .}}" >> "$(PROJECT_NAME_ALPHA)\databricks.yml"
+ - name: Initialize Bundle
+ id: initialize
+ run: |
+ databricks bundle init ./cicd --config-file "cicd_params.json"
+ - name: Commit Changes
+ id: commit
+ env:
+ GITHUB_TOKEN: {{`${{ github.token }}`}}
+ run: |
+ git config --global user.name "Deploy CICD Bot"
+ git config --global user.email "noreply-cicd-bot@databricks.com"
+ git checkout -b add-cicd-for-{{`${{ github.event.inputs.project_name }}`}}-{{`${{ env.timestamp }}`}}
+ git add .github "$PROJECT_NAME_ALPHA/databricks.yml"
+ git commit -m "Add CICD for {{`${{ github.event.inputs.project_name }}`}}"
+ git push origin add-cicd-for-{{`${{ github.event.inputs.project_name }}`}}-{{`${{ env.timestamp }}`}}
+
+ - name: Create Pull Request
+ id: pr
+ env:
+ GITHUB_TOKEN: {{`${{ github.token }}`}}
+ run: |
+ gh pr create --base {{ .input_default_branch }} --head add-cicd-for-{{`${{ github.event.inputs.project_name }}`}}-{{`${{ env.timestamp }}`}} --title "Deploy CICD for {{`${{ github.event.inputs.project_name }}`}}" --body "This PR was generated by the Deploy CICD workflow." --reviewer {{`${{ github.actor }}`}}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/lint-cicd-workflow-files.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/lint-cicd-workflow-files.yml.tmpl
new file mode 100644
index 00000000000..1e6ebd4bdc4
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/lint-cicd-workflow-files.yml.tmpl
@@ -0,0 +1,19 @@
+name: Lint CI/CD workflow files
+on:
+ pull_request:
+ paths:
+ - '.github/workflows/**'
+ workflow_dispatch:
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Download actionlint
+ id: get_actionlint
+ run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
+ shell: bash
+ - name: Check workflow files
+ {{`run: ${{ steps.get_actionlint.outputs.executable }} -color`}}
+ shell: bash
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-cd-prod.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-cd-prod.yml.tmpl
new file mode 100644
index 00000000000..92ea44e9563
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-cd-prod.yml.tmpl
@@ -0,0 +1,40 @@
+# This GitHub workflow deploys Bundle resources (ML resource config and more)
+# defined under {{template `project_name_alphanumeric_underscore` .}}/resources/*
+# and {{template `project_name_alphanumeric_underscore` .}}/databricks.yml with prod deployment target configs,
+# when PRs are merged into the release branch
+name: Bundle Deployment for {{ .input_project_name }} Prod
+
+on:
+ push:
+ branches:
+ - '{{ .input_release_branch }}'
+ workflow_dispatch:
+
+defaults:
+ run:
+ working-directory: ./{{template `project_name_alphanumeric_underscore` .}}
+
+env:
+ {{ if (eq .input_cloud `azure`) -}}
+ ARM_TENANT_ID: {{`${{ secrets.PROD_AZURE_SP_TENANT_ID }}`}}
+ ARM_CLIENT_ID: {{`${{ secrets.PROD_AZURE_SP_APPLICATION_ID }}`}}
+ ARM_CLIENT_SECRET: {{`${{ secrets.PROD_AZURE_SP_CLIENT_SECRET }}`}}
+ {{ else -}}
+ DATABRICKS_TOKEN: {{`${{ secrets.PROD_WORKSPACE_TOKEN }}`}}
+ {{- end }}
+
+jobs:
+ prod:
+ concurrency: {{ .input_project_name }}-prod-bundle-job
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: databricks/setup-cli@{{template `cli_version` .}}
+ - name: Validate Bundle For Prod
+ id: validate
+ run: |
+ databricks bundle validate -t prod
+ - name: Deploy Bundle to Prod
+ id: deploy
+ run: |
+ databricks bundle deploy -t prod
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-cd-staging.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-cd-staging.yml.tmpl
new file mode 100644
index 00000000000..87d93314d29
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-cd-staging.yml.tmpl
@@ -0,0 +1,40 @@
+# This GitHub workflow deploys Bundle resources (ML resource config and more)
+# defined under {{template `project_name_alphanumeric_underscore` .}}/resources/*
+# and {{template `project_name_alphanumeric_underscore` .}}/databricks.yml with staging deployment target configs,
+# when PRs are merged into the default branch
+name: Bundle Deployment for {{ .input_project_name }} Staging
+
+on:
+ push:
+ branches:
+ - '{{ .input_default_branch }}'
+ workflow_dispatch:
+
+defaults:
+ run:
+ working-directory: ./{{template `project_name_alphanumeric_underscore` .}}
+
+env:
+ {{ if (eq .input_cloud `azure`) -}}
+ ARM_TENANT_ID: {{`${{ secrets.STAGING_AZURE_SP_TENANT_ID }}`}}
+ ARM_CLIENT_ID: {{`${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }}`}}
+ ARM_CLIENT_SECRET: {{`${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }}`}}
+ {{ else -}}
+ DATABRICKS_TOKEN: {{`${{ secrets.STAGING_WORKSPACE_TOKEN }}`}}
+ {{- end }}
+
+jobs:
+ staging:
+ concurrency: {{ .input_project_name }}-staging-bundle-job
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: databricks/setup-cli@{{template `cli_version` .}}
+ - name: Validate Bundle For Staging
+ id: validate
+ run: |
+ databricks bundle validate -t staging
+ - name: Deploy Bundle to Staging
+ id: deploy
+ run: |
+ databricks bundle deploy -t staging
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-ci.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-ci.yml.tmpl
new file mode 100644
index 00000000000..cbdef1b00d8
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/.github/workflows/{{.input_project_name}}-bundle-ci.yml.tmpl
@@ -0,0 +1,116 @@
+# This GitHub workflow validates Bundle config (ML resource config and more)
+# defined under {{template `project_name_alphanumeric_underscore` .}}/resources/*
+# and {{template `project_name_alphanumeric_underscore` .}}/databricks.yml, when PRs are merged into the main branch
+name: Bundle validation for {{ .input_project_name }}
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - '{{template `project_name_alphanumeric_underscore` .}}/**'
+
+defaults:
+ run:
+ working-directory: ./{{template `project_name_alphanumeric_underscore` .}}/
+
+env:
+ {{ if (eq .input_cloud `azure`) -}}
+ STAGING_ARM_TENANT_ID: {{`${{ secrets.STAGING_AZURE_SP_TENANT_ID }}`}}
+ STAGING_ARM_CLIENT_ID: {{`${{ secrets.STAGING_AZURE_SP_APPLICATION_ID }}`}}
+ STAGING_ARM_CLIENT_SECRET: {{`${{ secrets.STAGING_AZURE_SP_CLIENT_SECRET }}`}}
+ PROD_ARM_TENANT_ID: {{`${{ secrets.PROD_AZURE_SP_TENANT_ID }}`}}
+ PROD_ARM_CLIENT_ID: {{`${{ secrets.PROD_AZURE_SP_APPLICATION_ID }}`}}
+ PROD_ARM_CLIENT_SECRET: {{`${{ secrets.PROD_AZURE_SP_CLIENT_SECRET }}`}}
+ {{ else -}}
+ STAGING_WORKSPACE_TOKEN: {{`${{ secrets.STAGING_WORKSPACE_TOKEN }}`}}
+ PROD_WORKSPACE_TOKEN: {{`${{ secrets.PROD_WORKSPACE_TOKEN }}`}}
+ {{- end }}
+
+jobs:
+ staging:
+ concurrency: {{ .input_project_name }}-staging-bundle-job
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: {{`${{ github.event.pull_request.head.sha || github.sha }}`}}
+ - uses: databricks/setup-cli@{{template `cli_version` .}}
+ - name: Validate Bundle For Staging
+ id: validate
+ env:
+ {{ if (eq .input_cloud `azure`) -}}
+ ARM_TENANT_ID: {{`${{ env.STAGING_ARM_TENANT_ID }}`}}
+ ARM_CLIENT_ID: {{`${{ env.STAGING_ARM_CLIENT_ID }}`}}
+ ARM_CLIENT_SECRET: {{`${{ env.STAGING_ARM_CLIENT_SECRET }}`}}
+ {{ else -}}
+ DATABRICKS_TOKEN: {{`${{ env.STAGING_WORKSPACE_TOKEN }}`}}
+ {{- end }}
+ run: |
+ databricks bundle validate -t staging > ../validate_output.txt
+ - name: Create Comment with Bundle Configuration
+ uses: actions/github-script@v7
+ id: comment
+ with:
+ github-token: {{`${{ secrets.GITHUB_TOKEN }}`}}
+ script: |
+ const fs = require('fs');
+ const fileContents = fs.readFileSync('validate_output.txt', 'utf8');
+ const output = `#### Bundle Staging Config Validated 🖌
+ Staging Validation Output
+
+ \`\`\`\n
+ ${fileContents}
+ \`\`\`
+
+ Prod Validation Output
+
+ \`\`\`\n
+ ${fileContents}
+ \`\`\`
+
+
+ - Generates the other CICD pipelines mentioned below for projects
+ - Manually triggered with `project_name` as parameter
+> Note that this workflow will need to be triggered for each project to set up its CI/CD. In order to run the `Push CICD Bundle to a New Branch` step in the workflow, the project needs to enable the Build Service to be able to contribute and create a branch for the project,
+i.e when the deploy CI/CD pipeline is triggered, the build service that runs this pipeline needs the necessary permissions to be able to push. To do this, go to Project Settings -> Repositories -> Security -> Select Build Service under users and
+set "Contribute", "Create Branch", and "Contribute to pull requests" to "Allow".
+
+Project-Specific pipelines:
+- **`{{ .input_project_name }}-tests-ci.yml`**:
+ - **[CI]** Performs unit and integration tests
+ - Triggered on PR to main
+- **`{{ .input_project_name }}-bundle-cicd.yml`**:
+ - **[CI]** Performs validation of Databricks resources defined under `{{template `project_name_alphanumeric_underscore` .}}/resources`
+ - Triggered on PR to main
+ - **[CD]** Deploys Databricks resources to the staging workspace
+ - Triggered on merging into main
+ - **[CD]** Deploys Databricks resources to the prod workspace
+ - Triggered on merging into release
+> Note that these workflows are provided as example CI/CD workflows, and can be easily modified to match your preferred CI/CD order of operations.
+
+Within the CI/CD pipelines defined under `.azure/devops-pipelines`, we will be deploying Databricks resources to the defined staging and prod workspaces using the `databricks` CLI. This requires setting up authentication between the `databricks` CLI and Databricks. By default we show how to authenticate with service principals by passing [secret variables from a variable group](https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/cli/pipeline-variable-group-secret-nonsecret-variables?view=azure-devops). In a production setting it is recommended to either use an [Azure Key Vault](https://learn.microsoft.com/en-us/azure/devops/pipelines/release/azure-key-vault?view=azure-devops&tabs=yaml) to store these secrets, or alternatively use [Azure service connections](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml). We describe below how you can adapt the project Pipelines to leverage service connections. Let's add these.
+
+```
+git add .azure
+git commit -m "Adding devops-pipeline files"
+git push upstream {{ .input_default_branch }}
+```
+
+### Service principal approach [Default]
+
+By default, we provide Azure Pipelines where authentication is done using service principals.
+
+#### Requirements:
+- You must be an account admin to add service principals to the account.
+- You must be a Databricks workspace admin in the staging and prod workspaces. Verify that you're an admin by viewing the
+ [staging workspace admin console]({{template `databricks_staging_workspace_host` .}}#setting/accounts) and
+ [prod workspace admin console]({{template `databricks_prod_workspace_host` .}}#setting/accounts). If
+ the admin console UI loads instead of the Databricks workspace homepage, you are an admin.
+- Permissions to create Azure DevOps Pipelines in your Azure DevOps project. See the following [Azure DevOps prerequisites](https://learn.microsoft.com/en-us/azure/devops/organizations/security/about-permissions).
+- Permissions to create Azure DevOps build policies. See the following [prerequisites](https://learn.microsoft.com/azure/devops/repos/git/branch-policies).
+
+#### Steps:
+{{ if (eq .input_cloud `azure`) }}
+1. Create two service principals - one to be used for deploying and running staging resources, and one to be used for deploying and running production resources. See [here]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals")) }}) for details on how to create a service principal.
+1. [Add the staging and production service principals to your Azure Databricks account]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals#add-service-principals-to-your-account-using-the-account-console")) }}), and following this add the staging service principal to the staging workspace, and production service principal to the production workspace. See [here]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals")) }}) for details.
+1. Follow ['Get Azure AD tokens for the service principals']({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/api/latest/aad/service-prin-aad-token")) }})
+to get your service principal credentials (tenant id, application id, and client secret) for both the staging and prod service principals. You will use these credentials as variables in the project Azure Pipelines.
+{{ else }}
+1. Create two service principals - one to be used for deploying and running staging resources, and one to be used for deploying and running production resources. See [here]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html")) }}) for details on how to create a service principal.
+1. [Add the staging and production service principals to your Databricks account]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html#add-service-principals-to-your-account-using-the-account-console")) }}), and following this add the staging service principal to the staging workspace, and production service principal to the production workspace. See [here]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html")) }}) for details.
+1. Follow ['Get tokens for the service principals']({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html#manage-personal-access-tokens-for-a-service-principal")) }})
+to get your service principal token for both the staging and prod service principals. You will use the token as variables in the project Azure Pipelines.
+{{ end }}
+1. Create separate Azure Pipelines under your Azure DevOps project using the ‘Existing Azure Pipelines YAML file’ option. Create one pipeline for each script. See [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/create-first-pipeline) for more details on creating Azure Pipelines.
+1. Create a new variable group called `{{ .input_root_dir }} variable group` defining the following secret variables, for more details [here](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=classic#create-a-variable-group):
+{{ if (eq .input_cloud `azure`) }}
+ - `PROD_AZURE_SP_TENANT_ID`: tenant ID for the prod service principal
+ - `PROD_AZURE_SP_APPLICATION_ID`: application (client) ID for the prod service principal
+ - `PROD_AZURE_SP_CLIENT_SECRET`: client secret for the prod service principal
+ - `STAGING_AZURE_SP_TENANT_ID`: tenant ID for the staging service principal
+ - `STAGING_AZURE_SP_APPLICATION_ID`: application (client) ID for the staging service principal
+ - `STAGING_AZURE_SP_CLIENT_SECRET`: client secret for the prod service principal
+{{ else }}
+ - `PROD_WORKSPACE_TOKEN` : service principal token for prod workspace
+ - `STAGING_WORKSPACE_TOKEN` : service principal token for staging workspace
+{{ end }}
+ - Ensure that the Azure Pipelines created in the prior step have access to these variables by selecting the name of the pipelines under the 'Pipeline permissions' tab of this variable group.
+ - Alternatively you could store these secrets in an [Azure Key Vault](https://learn.microsoft.com/en-us/azure/devops/pipelines/release/key-vault-in-own-project?view=azure-devops&tabs=portal) and link those secrets as variables to be used in the Pipelines.
+1. Define [build validation branch policies](https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser#build-validation) for the `{{ .input_default_branch }}` branch using the Azure build pipelines created in step 1. This is required so that any PR changes to the `{{ .input_default_branch }}` must build successfully before PRs can complete.
+In the case of a monorepo, where there are multiple projects under a single repository, set a [path filter](https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser#path-filters) on the build validation policies, such that devops pipelines are only triggered when there are changes to the respective projects (e.g. the path filter would be `/project1/*` to trigger a devops pipeline when changes are made to _only_ files under the `project1` folder).
+
+{{ if (eq .input_cloud `azure`) }}
+### Service connection approach [Recommended in production settings]
+
+#### Requirements:
+- You must be an Azure account admin to add service principals to the account.
+- You must be a Databricks workspace admin in the staging and prod workspaces. Verify that you're an admin by viewing the
+ [staging workspace admin console]({{template `databricks_staging_workspace_host` .}}#setting/accounts) and
+ [prod workspace admin console]({{template `databricks_prod_workspace_host` .}}#setting/accounts). If
+ the admin console UI loads instead of the Databricks workspace homepage, you are an admin.
+- Permissions to create service connections within an Azure subscription. See the following [prerequisites](https://docs.microsoft.com/azure/devops/pipelines/library/service-endpoints).
+- Permissions to create Azure DevOps Pipelines in your Azure DevOps project. See the following [Azure DevOps prerequisites](https://learn.microsoft.com/en-us/azure/devops/organizations/security/about-permissions).
+- Permissions to create Azure DevOps build policies. See the following [prerequisites](https://learn.microsoft.com/azure/devops/repos/git/branch-policies).
+
+The ultimate aim of the service connection approach is to use two separate service connections, authenticated with a staging service principal and a production service principal, to deploy and run resources in the respective Azure Databricks workspaces. Taking this approach then negates the need to read client secrets or client IDs from the CI/CD pipelines.
+
+#### Steps:
+1. Create two service principals - one to be used for deploying and running staging resources, and one to be used for deploying and running production resources. See [here]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals")) }}) for details on how to create a service principal.
+1. [Add the staging and production service principals to your Azure Databricks account]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals#add-service-principals-to-your-account-using-the-account-console")) }}), and following this add the staging service principal to the staging workspace, and production service principal to the production workspace. See [here]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html")) }}) for details.
+1. [Create two Azure Resource Manager service connections](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection) - one to be used to deploy to staging Databricks resources, the other for production resources. Each of these service connections should be authenticated with the respective staging and production service principals created in the prior step.
+1. Update pipeline YAML files to use service connections rather than pipeline variables:
+ - First, remove any lines where the environment variables are set in tasks in `{{ .input_project_name }}-tests-ci.yml` or `{{ .input_project_name }}-bundle-cicd.yml` files. Specifically, any lines where the following env vars are used: `PROD_AZURE_SP_TENANT_ID`, `PROD_AZURE_SP_APPLICATION_ID`, `PROD_AZURE_SP_CLIENT_SECRET`, `STAGING_AZURE_SP_TENANT_ID`, `STAGING_AZURE_SP_APPLICATION_ID`, `STAGING_AZURE_SP_CLIENT_SECRET`
+ - Then, add the following AzureCLI task prior to installing the `databricks` cli in any of the pipeline jobs:
+
+```yaml
+# Get Azure Resource Manager variables using service connection
+- task: AzureCLI@2
+ displayName: 'Extract information from Azure CLI'
+ inputs:
+ azureSubscription: # TODO: insert SERVICE_CONNECTION_NAME
+ addSpnToEnvironment: true
+ scriptType: bash
+ scriptLocation: inlineScript
+ inlineScript: |
+ subscription_id=$(az account list --query "[?isDefault].id"|jq -r '.[0]')
+ echo "##vso[task.setvariable variable=ARM_CLIENT_ID]${servicePrincipalId}"
+ echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET;issecret=true]${servicePrincipalKey}"
+ echo "##vso[task.setvariable variable=ARM_TENANT_ID]${tenantId}"
+ echo "##vso[task.setvariable variable=ARM_SUBSCRIPTION_ID]${subscription_id}"
+```
+ > Note that you will have to update this code snippet with the respective service connection names, depending on which Databricks workspace you are deploying resources to.
+
+1. Create separate Azure Pipelines under your Azure DevOps project using the ‘Existing Azure Pipelines YAML file’ option. Create one pipeline for each script. See [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/create-first-pipeline) for more details on creating Azure Pipelines.
+1. Define [build validation branch policies](https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser#build-validation) for the `{{ .input_default_branch }}` branch using the Azure build pipelines created in step 1. This is required so that any PR changes to the `{{ .input_default_branch }}` must build successfully before PRs can complete.
+In the case of a monorepo, where there are multiple projects under a single repository, set a [path filter](https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser#path-filters) on the build validation policies, such that devops pipelines are only triggered when there are changes to the respective projects (e.g. the path filter would be `/project1/*` to trigger a devops pipeline when changes are made to _only_ files under the `project1` folder).
+{{ end }}
+
+### Setting up CI/CD workflows
+After setting up authentication for CI/CD, you can now set up CI/CD workflows. We provide a [Deploy CICD workflow](../.azure/devops-pipelines/deploy-cicd.yml) that can be used to generate the other CICD workflows mentioned below for projects.
+This workflow is manually triggered with `project_name` as parameter. This workflow will need to be triggered for each project to set up its set of CI/CD workflows that can be used to deploy ML resources and run ML jobs in the staging and prod workspaces.
+These workflows will be defined under `.azure/devops-pipelines`. After generating these workflows, be sure to go through the above workflow-specific steps again to add the appropriate build branch policies and filters.
+
+{{ else if (eq .input_cicd_platform `gitlab`) }}
+## Configure CI/CD - Gitlab Pipelines
+
+### Prerequisites
+* You must be an account admin to add service principals to the account.
+* You must be a Databricks workspace admin in the staging and prod workspaces. Verify that you're an admin by viewing the
+ [staging workspace admin console]({{template `databricks_staging_workspace_host` .}}#setting/accounts) and
+ [prod workspace admin console]({{template `databricks_prod_workspace_host` .}}#setting/accounts). If
+ the admin console UI loads instead of the Databricks workspace homepage, you are an admin.
+
+### Set up authentication for CI/CD
+#### Set up Service Principal
+To authenticate and manage ML resources created by CI/CD,
+[service principals]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals")) }})
+should be created and added to test, staging and prod workspaces.
+
+Service principals can be created and managed in the your cloud provider identity solution or in Databricks directly. We normally recommend setting up a Databricks managed service principal.
+Follow [Add a service principal to your Databricks account]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html#add-a-service-principal-to-your-databricks-account")) }})
+and [Add a service principal to a workspace]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "administration-guide/users-groups/service-principals.html#add-a-service-principal-to-a-workspace")) }})
+for details.
+
+{{ if eq .input_cloud `aws` }}
+For your convenience, we also have a [Terraform module](https://registry.terraform.io/modules/databricks/mlops-aws-project/databricks/latest) that can set up your service principals.
+{{ end }}
+
+#### Configure Service Principal (SP) permissions
+If the created project uses **Unity Catalog**, we expect a catalog to exist with the name of the deployment target by default.
+For example, if the deployment target is dev, we expect a catalog named dev to exist in the workspace.
+If you want to use different catalog names, please update the target names declared in the
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`)}}[{{ .input_project_name }}/databricks.yml](../{{template `project_name_alphanumeric_underscore` .}}/databricks.yml)
+{{- else }} `databricks.yml` {{ end }} file.
+If changing the staging, prod, or test deployment targets, you'll also need to update the workflows located in the .gitlab/pipelines directory.
+
+The SP must have proper permission in each respective environment and the catalog for the environments.
+
+For the integration test and the ML training job, the SP must have permissions to read the input Delta table and create experiment and models.
+i.e. for each environment:
+- USE_CATALOG
+- USE_SCHEMA
+- MODIFY
+- CREATE_MODEL
+- CREATE_TABLE
+
+For the batch inference job, the SP must have permissions to read input Delta table and modify the output Delta table.
+i.e. for each environment
+- USAGE permissions for the catalog and schema of the input and output table.
+- SELECT permission for the input table.
+- MODIFY permission for the output table if it pre-dates your job.
+
+#### Gitlab Environment and secrets for CI/CD
+After creating the service principals and adding them to the respective staging and prod workspaces,
+you need to ad the client id and secret to Gitlab so that it can authenticate into Databricks for the execution of the integration tests and deployments.
+First you need to setup the [Gitlab environments](https://docs.gitlab.com/ee/ci/environments/). Typically you setup the following environemnts:
+- integration
+- stage
+- production
+
+Add the following [Gitlab CI/CD variables](https://docs.gitlab.com/ee/ci/variables/) In each [environemt](https://docs.gitlab.com/ee/ci/environments/index.html#limit-the-environment-scope-of-a-cicd-variable), with the corresponding service principal information:
+- SP_CLIENT_ID
+- SP_CLIENT_SECRET
+
+Ensure that the variable visibility is set to masked and hidden.
+
+### Building and Pushing the Docker image
+Gitlab exectues the pipeline on a VM initialized from a Docker image.
+The default Docker image is: [databricksfieldeng/mlopsstack:latest](https://hub.docker.com/repository/docker/databricksfieldeng/mlopsstack/general).
+The Docker image should include all requirements to run the CI/CD pipelines (e.g. Databricks CLI, Python and its libraries used in the unit tests, Java for executing spark locally to the VM).
+The folder `.gitlab/docker/` includes the files to build the Docker image.
+
+You can execute the following script to locally build and push the Docker image into a registry on your local machine:
+`{{.input_root_dir}}/.gitlab/docker/push_image_to_gitlab.sh`
+Make sure that each pipeline yml file
+(in the folder `.gitlab/pipelines/`) points to that image.
+NOTE: you can use the same image for multiple projects.
+If a project requires additional Python libraries, they can also be installed as part of the pipelines scripts.
+
+### Setting up the CI/CD Pipeline
+Gitlab, by default, expects the pipeline file to be placed in the project root folder and to be named `.gitlab-ci.yml`.
+Change the pipeline default [pipeline configuration file](https://docs.gitlab.com/ee/ci/pipelines/settings.html#specify-a-custom-cicd-configuration-file):
+in the section `Settings > CI/CD > General Pipelines > CI/CD configuration file`,
+add the value `.gitlab/pipelines/{{.input_project_name}}-triggers-cicd.yml` which is our project main pipeline file.
+
+{{ end }}
+
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`)}}
+## Merge a PR with your initial ML code
+Create and push a PR branch adding the ML code to the repository.
+
+```
+git checkout -b add-ml-code
+git add .
+git commit -m "Add ML Code"
+git push upstream add-ml-code
+```
+
+Open a PR from the newly pushed branch. CI will run to ensure that tests pass
+on your initial ML code. Fix tests if needed, then get your PR reviewed and merged.
+After the pull request merges, pull the changes back into your local `{{ .input_default_branch }}`
+branch:
+
+```
+git checkout {{ .input_default_branch }}
+git pull upstream {{ .input_default_branch }}
+```
+{{- end }}
+
+{{ if not (eq .input_release_branch .input_default_branch) -}}
+## Create release branch
+Create and push a release branch called `{{ .input_release_branch }}` off of the `{{ .input_default_branch }}` branch of the repository:
+```
+git checkout -b {{ .input_release_branch }} {{ .input_default_branch }}
+git push upstream {{ .input_release_branch }}
+git checkout {{ .input_default_branch }}
+```
+
+Your production jobs (model training, batch inference) will pull ML code against this branch, while your staging jobs will pull ML code against the `{{ .input_default_branch }}` branch. Note that the `{{ .input_default_branch }}` branch will be the source of truth for ML resource configs and CI/CD workflows.
+
+For future ML code changes, iterate against the `{{ .input_default_branch }}` branch and regularly deploy your ML code from staging to production by merging code changes from the `{{ .input_default_branch }}` branch into the `{{ .input_release_branch }}` branch.
+{{ end -}}
+
+{{ if (eq .input_setup_cicd_and_project `CICD_and_Project`)}}
+## Deploy ML resources and enable production jobs
+Follow the instructions in [{{ .input_project_name }}/resources/README.md](../{{template `project_name_alphanumeric_underscore` .}}/resources/README.md) to deploy ML resources
+and production jobs.
+{{- end }}
+
+## Next steps
+After you configure CI/CD and deploy training & inference pipelines, notify data scientists working
+on the current project. They should now be able to follow the
+[ML pull request guide](ml-pull-request.md) and
+{{ if (eq .input_setup_cicd_and_project `CICD_and_Project`)}}[ML resource config guide](../{{template `project_name_alphanumeric_underscore` .}}/resources/README.md){{- end }} to propose, test, and deploy
+ML code and pipeline changes to production.
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/test-requirements.txt.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/test-requirements.txt.tmpl
new file mode 100644
index 00000000000..e160152b522
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/test-requirements.txt.tmpl
@@ -0,0 +1 @@
+pytest>=7.1.2
\ No newline at end of file
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/README.md.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/README.md.tmpl
new file mode 100644
index 00000000000..de0baae8a90
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/README.md.tmpl
@@ -0,0 +1,213 @@
+# {{ .input_project_name }}
+This project comes with example ML code to train, validate and deploy a regression model to predict NYC taxi fares.
+If you're a data scientist just getting started with this repo for a brand new ML project, we recommend
+adapting the provided example code to your ML problem. Then making and
+testing ML code changes on Databricks or your local machine.
+
+The "Getting Started" docs can be found at {{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/bundles/mlops-stacks.html")) }}.
+
+## Table of contents
+* [Code structure](#code-structure): structure of this project.
+{{ if (eq .input_include_feature_store `yes`) }}
+* [Configure your ML pipeline](#configure-your-ml-pipeline): making and testing ML code changes on Databricks or your local machine.
+{{ end }}
+* [Iterating on ML code](#iterating-on-ml-code): making and testing ML code changes on Databricks or your local machine.
+* [Next steps](#next-steps)
+
+This directory contains an ML project based on the default
+[Databricks MLOps Stacks](https://github.com/databricks/mlops-stacks),
+defining a production-grade ML pipeline for automated retraining and batch inference of an ML model on tabular data.
+
+## Code structure
+This project contains the following components:
+
+| Component | Description |
+|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ML Code | Example ML project code, with unit tested Python modules and notebooks |
+| ML Resources as Code | ML pipeline resources (training and batch inference jobs with schedules, etc) configured and deployed through [databricks CLI bundles]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/cli/bundle-cli.html")) }}) |
+
+contained in the following files:
+
+```
+{{ .input_root_dir }} <- Root directory. Both monorepo and polyrepo are supported.
+│
+├── {{template `project_name_alphanumeric_underscore` .}} <- Contains python code, notebooks and ML resources related to one ML project.
+│ │
+│ ├── requirements.txt <- Specifies Python dependencies for ML code (for example: model training, batch inference).
+│ │
+│ ├── databricks.yml <- databricks.yml is the root bundle file for the ML project that can be loaded by databricks CLI bundles. It defines the bundle name, workspace URL and resource config component to be included.
+│ │
+{{- if (eq .input_include_feature_store `no`) }}
+│ ├── training <- Training folder contains Notebook that trains and registers the model.
+│ │
+│ ├── validation <- Optional model validation step before deploying a model.
+│ │
+│ ├── monitoring <- Model monitoring, feature monitoring, etc.
+│ │
+│ ├── deployment <- Deployment and Batch inference workflows
+│ │ │
+│ │ ├── batch_inference <- Batch inference code that will run as part of scheduled workflow.
+│ │ │
+│ │ ├── model_deployment <- As part of CD workflow, deploy the registered model by assigning it the appropriate alias.
+│ │
+│ │
+│ ├── tests <- Unit tests for the ML project, including the modules under `features`.
+│ │
+│ ├── resources <- ML resource (ML jobs, MLflow models) config definitions expressed as code, across dev/staging/prod/test.
+│ │
+│ ├── model-workflow-resource.yml <- ML resource config definition for model training, validation, deployment workflow
+│ │
+│ ├── batch-inference-workflow-resource.yml <- ML resource config definition for batch inference workflow
+│ │
+│ ├── ml-artifacts-resource.yml <- ML resource config definition for model and experiment
+│ │
+│ ├── monitoring-resource.yml <- ML resource config definition for quality monitoring workflow
+{{- else if (eq .input_include_feature_store `yes`) }}
+│ ├── training <- Training folder contains Notebook that trains and registers the model with feature store support.
+│ │
+│ ├── feature_engineering <- Feature computation code (Python modules) that implements the feature transforms.
+│ │ The output of these transforms get persisted as Feature Store tables. Most development
+│ │ work happens here.
+│ │
+│ ├── validation <- Optional model validation step before deploying a model.
+│ │
+│ ├── monitoring <- Model monitoring, feature monitoring, etc.
+│ │
+│ ├── deployment <- Deployment and Batch inference workflows
+│ │ │
+│ │ ├── batch_inference <- Batch inference code that will run as part of scheduled workflow.
+│ │ │
+│ │ ├── model_deployment <- As part of CD workflow, deploy the registered model by assigning it the appropriate alias.
+│ │
+│ │
+│ ├── tests <- Unit tests for the ML project, including the modules under `features`.
+│ │
+│ ├── resources <- ML resource (ML jobs, MLflow models) config definitions expressed as code, across dev/staging/prod/test.
+│ │
+│ ├── model-workflow-resource.yml <- ML resource config definition for model training, validation, deployment workflow
+│ │
+│ ├── batch-inference-workflow-resource.yml <- ML resource config definition for batch inference workflow
+│ │
+│ ├── feature-engineering-workflow-resource.yml <- ML resource config definition for feature engineering workflow
+│ │
+│ ├── ml-artifacts-resource.yml <- ML resource config definition for model and experiment
+│ │
+│ ├── monitoring-resource.yml <- ML resource config definition for quality monitoring workflow
+{{- end }}
+```
+
+{{ if (eq .input_include_feature_store `yes`) }}
+## Configure your ML pipeline
+
+The sample ML code consists of the following:
+
+* Feature computation modules under `feature_engineering` folder.
+These sample module contains features logic that can be used to generate and populate tables in Feature Store.
+In each module, there is `compute_features_fn` method that you need to implement. This should compute a features dataframe
+(each column being a separate feature), given the input dataframe, timestamp column and time-ranges.
+The output dataframe will be persisted in a [time-series Feature Store table]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "machine-learning/feature-store/time-series.html")) }}).
+See the example modules' documentation for more information.
+* Python unit tests for feature computation modules in `tests/feature_engineering` folder.
+* Feature engineering notebook, `feature_engineering/GenerateAndWriteFeatures.py`, that reads input dataframes, dynamically loads feature computation modules, executes their `compute_features_fn` method and writes the outputs to a Feature Store table (creating it if missing).
+* Training notebook that [trains]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "machine-learning/feature-store/train-models-with-feature-store.html")) }} ) a regression model by creating a training dataset using the Feature Store client.
+* Model deployment and batch inference notebooks that deploy and use the trained model.
+* An automated integration test is provided (in `.github/workflows/{{ .input_project_name }}-run-tests.yml`) that executes a multi task run on Databricks involving the feature engineering and model training notebooks.
+
+To adapt this sample code for your use case, implement your own feature module, specifying configs such as input Delta tables/dataset path(s) to use when developing
+the feature engineering pipelines.
+1. Implement your feature module, address TODOs in `feature_engineering/features` and create unit test in `tests/feature_engineering`
+2. Update `resources/feature-engineering-workflow-resource.yml`. Fill in notebook parameters for `write_feature_table_job`.
+3. Update training data path in `resources/model-workflow-resource.yml`.
+
+We expect most of the development to take place in the `feature_engineering` folder.
+
+{{ end }}
+## Iterating on ML code
+
+### Deploy ML code and resources to dev workspace using Bundles
+
+Refer to [Local development and dev workspace](./resources/README.md#local-development-and-dev-workspace)
+to use databricks CLI bundles to deploy ML code together with ML resource configs to dev workspace.
+
+This will allow you to develop locally and use databricks CLI bundles to deploy to your dev workspace to test out code and config changes.
+
+### Develop on Databricks using Databricks Git folders
+
+#### Prerequisites
+You'll need:
+* Access to run commands on a cluster running Databricks Runtime ML version 11.0 or above in your dev Databricks workspace
+* To set up [Databricks Git folders]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "repos/index.html")) }}): see instructions below
+
+#### Configuring Databricks Git folders
+To use Git folders, [set up git integration]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "repos/repos-setup.html")) }}) in your dev workspace.
+
+If the current project has already been pushed to a hosted Git repo, follow the
+[UI workflow]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later")) }})
+to clone it into your dev workspace and iterate.
+
+Otherwise, e.g. if iterating on ML code for a new project, follow the steps below:
+* Follow the [UI workflow]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later")) }})
+ for creating a repo, but uncheck the "Create repo by cloning a Git repository" checkbox.
+* Install the `dbx` CLI via `pip install --upgrade dbx`
+* Run `databricks configure --profile {{ .input_project_name }}-dev --token --host `, passing the URL of your dev workspace.
+ This should prompt you to enter an API token
+* [Create a personal access token]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/auth/pat.html")) }})
+ in your dev workspace and paste it into the prompt from the previous step
+* From within the root directory of the current project, use the [dbx sync](https://dbx.readthedocs.io/en/latest/guides/python/devloop/mixed/#using-dbx-sync-repo-for-local-to-repo-synchronization) tool to copy code files from your local machine into the Repo by running
+ `dbx sync repo --profile {{ .input_project_name }}-dev --source . --dest-repo your-repo-name`, where `your-repo-name` should be the last segment of the full repo name (`/Repos/username/your-repo-name`)
+
+{{ if (eq .input_include_feature_store `yes`) }}
+### Develop locally
+
+You can iterate on the feature transform modules locally in your favorite IDE before running them on Databricks.
+
+#### Running code on Databricks
+You can iterate on ML code by running the provided `feature_engineering/GenerateAndWriteFeatures.py` notebook on Databricks using
+[Git folders]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "repos/index.html")) }}). This notebook drives execution of
+the feature transforms code defined under ``features``. You can use multiple browser tabs to edit
+logic in `features` and run the feature engineering pipeline in the `GenerateAndWriteFeatures.py` notebook.
+
+#### Prerequisites
+* Python 3.8+
+* Install feature engineering code and test dependencies via `pip install -I -r requirements.txt` from project root directory.
+* The features transform code uses PySpark and brings up a local Spark instance for testing, so [Java (version 8 and later) is required](https://spark.apache.org/docs/latest/#downloading).
+
+* Access to UC catalog and schema
+We expect a catalog to exist with the name of the deployment target by default.
+For example, if the deployment target is dev, we expect a catalog named dev to exist in the workspace.
+If you want to use different catalog names, please update the target names declared in the [databricks.yml](./databricks.yml) file.
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`) }}
+If changing the staging, prod, or test deployment targets, you'll also need to update the workflows located in the .github/workflows directory.
+{{- end }}
+
+For the ML training job, you must have permissions to read the input Delta table and create experiment and models.
+i.e. for each environment:
+- USE_CATALOG
+- USE_SCHEMA
+- MODIFY
+- CREATE_MODEL
+- CREATE_TABLE
+
+For the batch inference job, you must have permissions to read input Delta table and modify the output Delta table.
+i.e. for each environment
+- USAGE permissions for the catalog and schema of the input and output table.
+- SELECT permission for the input table.
+- MODIFY permission for the output table if it pre-dates your job.
+
+#### Run unit tests
+You can run unit tests for your ML code via `pytest tests`.
+
+{{ end }}
+
+## Next Steps
+
+When you're satisfied with initial ML experimentation (e.g. validated that a model with reasonable performance can be trained on your dataset) and ready to deploy production training/inference pipelines, ask your ops team to set up CI/CD for the current ML project if they haven't already. CI/CD can be set up as part of the
+
+MLOps Stacks initialization even if it was skipped in this case, or this project can be added to a repo setup with CI/CD already, following the directions under "Setting up CI/CD" in the repo root directory README.
+
+To add CI/CD to this repo:
+ 1. Run `databricks bundle init mlops-stacks` via the Databricks CLI
+ 2. Select the option to only initialize `CICD_Only`
+ 3. Provide the root directory of this project and answer the subsequent prompts
+
+More details can be found on the homepage [MLOps Stacks README](https://github.com/databricks/mlops-stacks/blob/main/README.md).
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/__init__.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/__init__.py.tmpl
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/databricks.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/databricks.yml.tmpl
new file mode 100644
index 00000000000..da265e6bcdd
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/databricks.yml.tmpl
@@ -0,0 +1,61 @@
+# The name of the bundle. run `databricks bundle schema` to see the full bundle settings schema.
+bundle:
+ # Do not modify the below line, this autogenerated field is used by the Databricks backend.
+ uuid: {{ bundle_uuid }}
+
+ name: {{ .input_project_name }}
+
+variables:
+ experiment_name:
+ description: Experiment name for the model training.
+ default: /Users/${workspace.current_user.userName}/${bundle.target}-{{template `experiment_base_name` .}}
+ model_name:
+ description: Model name for the model training.
+ default: {{template `model_name` .}}
+ catalog_name:
+ description: The catalog name to save the trained model
+
+include:
+ # Resources folder contains ML artifact resources for the ML project that defines model and experiment
+ # And workflows resources for the ML project including model training -> validation -> deployment,
+ # {{- if (eq .input_include_feature_store `yes`) }} feature engineering, {{ end }} batch inference, quality monitoring, metric refresh, alerts and triggering retraining
+ - ./resources/batch-inference-workflow-resource.yml
+ - ./resources/ml-artifacts-resource.yml
+ - ./resources/model-workflow-resource.yml
+ {{- if (eq .input_include_feature_store `yes`) }}
+ - ./resources/feature-engineering-workflow-resource.yml
+ {{- end }}
+ # TODO: uncomment once monitoring inference table has been created
+ # - ./resources/monitoring-resource.yml
+
+# Deployment Target specific values for workspace
+targets:
+ dev: # UC Catalog Name
+ mode: development
+ default: true
+ variables:
+ catalog_name: dev
+ workspace:
+ # TODO: add dev workspace URL
+ host:
+
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`)}}
+
+ staging:
+ variables:
+ catalog_name: {{ .input_staging_catalog_name }}
+ workspace:
+ host: {{template `databricks_staging_workspace_host` .}}
+
+ prod:
+ variables:
+ catalog_name: {{ .input_prod_catalog_name }}
+ workspace:
+ host: {{template `databricks_prod_workspace_host` .}}
+
+ test:
+ variables:
+ catalog_name: {{ .input_test_catalog_name }}
+ workspace:
+ host: {{template `databricks_staging_workspace_host` .}}
+{{ end }}
\ No newline at end of file
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/BatchInference.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/BatchInference.py.tmpl
new file mode 100644
index 00000000000..3a94480e422
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/BatchInference.py.tmpl
@@ -0,0 +1,69 @@
+# Databricks notebook source
+##################################################################################
+# Batch Inference Notebook
+#
+# This notebook is an example of applying a model for batch inference against an input delta table,
+# It is configured and can be executed as the batch_inference_job in the batch_inference_job workflow defined under
+# ``{{template `project_name_alphanumeric_underscore` .}}/resources/batch-inference-workflow-resource.yml``
+#
+# Parameters:
+#
+# * env (optional) - String name of the current environment (dev, staging, or prod). Defaults to "dev"
+# * input_table_name (required) - Delta table name containing your input data.
+# * output_table_name (required) - Delta table name where the predictions will be written to.
+# Note that this will create a new version of the Delta table if
+# the table already exists
+# * model_name (required) - The name of the model to be used in batch inference.
+##################################################################################
+
+
+# List of input args needed to run the notebook as a job.
+# Provide them via DB widgets or notebook arguments.
+#
+# Name of the current environment
+dbutils.widgets.dropdown("env", "dev", ["dev", "staging", "prod"], "Environment Name")
+# A Hive-registered Delta table containing the input features.
+dbutils.widgets.text("input_table_name", "", label="Input Table Name")
+# Delta table to store the output predictions.
+dbutils.widgets.text("output_table_name", "", label="Output Table Name")
+# Unity Catalog registered model name to use for the trained mode.
+dbutils.widgets.text(
+ "model_name", "dev.{{ .input_schema_name }}.{{template `model_name` .}}", label="Full (Three-Level) Model Name"
+)
+
+# COMMAND ----------
+
+# DBTITLE 1,Define input and output variables
+
+env = dbutils.widgets.get("env")
+input_table_name = dbutils.widgets.get("input_table_name")
+output_table_name = dbutils.widgets.get("output_table_name")
+model_name = dbutils.widgets.get("model_name")
+assert input_table_name != "", "input_table_name notebook parameter must be specified"
+assert output_table_name != "", "output_table_name notebook parameter must be specified"
+assert model_name != "", "model_name notebook parameter must be specified"
+alias = "champion"
+model_uri = f"models:/{model_name}@{alias}"
+
+# COMMAND ----------
+
+from mlflow import MlflowClient
+
+# Get model version from alias
+client = MlflowClient()
+model_version = client.get_model_version_by_alias(model_name, alias).version
+
+# COMMAND ----------
+
+# Get datetime
+from datetime import datetime
+
+ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+# COMMAND ----------
+# DBTITLE 1,Load model and run inference
+
+from predict import predict_batch
+
+predict_batch(spark, model_uri, input_table_name, output_table_name, model_version, ts)
+dbutils.notebook.exit(output_table_name)
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/README.md.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/README.md.tmpl
new file mode 100644
index 00000000000..2a86b6126b2
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/README.md.tmpl
@@ -0,0 +1,53 @@
+# Batch Inference
+To set up batch inference job via scheduled Databricks workflow, please refer to [{{template `project_name_alphanumeric_underscore` .}}/resources/README.md](../../resources/README.md)
+
+## Prepare the batch inference input table for the example Project
+Please run the following code in a notebook to generate the example batch inference input table.
+{{ if (eq .input_include_feature_store `yes`) }}
+```
+from pyspark.sql.functions import to_timestamp, lit
+from pyspark.sql.types import IntegerType
+import math
+from datetime import timedelta, timezone
+
+def rounded_unix_timestamp(dt, num_minutes=15):
+ """
+ Ceilings datetime dt to interval num_minutes, then returns the unix timestamp.
+ """
+ nsecs = dt.minute * 60 + dt.second + dt.microsecond * 1e-6
+ delta = math.ceil(nsecs / (60 * num_minutes)) * (60 * num_minutes) - nsecs
+ return int((dt + timedelta(seconds=delta)).replace(tzinfo=timezone.utc).timestamp())
+
+
+rounded_unix_timestamp_udf = udf(rounded_unix_timestamp, IntegerType())
+
+df = spark.table("delta.`dbfs:/databricks-datasets/nyctaxi-with-zipcodes/subsampled`")
+df.withColumn(
+ "rounded_pickup_datetime",
+ to_timestamp(rounded_unix_timestamp_udf(df["tpep_pickup_datetime"], lit(15))),
+).withColumn(
+ "rounded_dropoff_datetime",
+ to_timestamp(rounded_unix_timestamp_udf(df["tpep_dropoff_datetime"], lit(30))),
+).drop(
+ "tpep_pickup_datetime"
+).drop(
+ "tpep_dropoff_datetime"
+).drop(
+ "fare_amount"
+).write.mode(
+ "overwrite"
+).saveAsTable(
+ name="hive_metastore.default.taxi_scoring_sample_feature_store_inference_input"
+)
+```
+{{ else }}
+```
+df = spark.table(
+ "delta.`dbfs:/databricks-datasets/nyctaxi-with-zipcodes/subsampled`"
+).drop("fare_amount")
+
+df.write.mode("overwrite").saveAsTable(
+ name=".{{ .input_schema_name }}.feature_store_inference_input"
+)
+```
+{{ end }}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/predict.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/predict.py.tmpl
new file mode 100644
index 00000000000..641f8dfc8a1
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/batch_inference/predict.py.tmpl
@@ -0,0 +1,40 @@
+import mlflow
+from pyspark.sql.functions import struct, lit, to_timestamp
+
+
+def predict_batch(
+ spark_session, model_uri, input_table_name, output_table_name, model_version, ts
+):
+ """
+ Apply the model at the specified URI for batch inference on the table with name input_table_name,
+ writing results to the table with name output_table_name
+ """
+ table = spark_session.table(input_table_name)
+ {{ if (eq .input_include_feature_store `yes`) }}
+ from databricks.feature_engineering import FeatureEngineeringClient
+
+ fe_client = FeatureEngineeringClient()
+
+ prediction_df = fe_client.score_batch(model_uri=model_uri, df=table)
+ output_df = (
+ prediction_df.withColumn("prediction", prediction_df["prediction"])
+ .withColumn("model_id", lit(model_version))
+ .withColumn("timestamp", to_timestamp(lit(ts)))
+ )
+ {{ else }}
+ predict = mlflow.pyfunc.spark_udf(
+ spark_session, model_uri, result_type="double"
+ )
+
+ output_df = (
+ table.withColumn("prediction", predict(struct(*table.columns)))
+ .withColumn("model_id", lit(model_version))
+ .withColumn("timestamp", to_timestamp(lit(ts)))
+ )
+ {{ end -}}
+
+ output_df.display()
+
+ # Model predictions are written to the Delta table provided as input.
+ # Delta is the default format in Databricks Runtime 8.0 and above.
+ output_df.write.format("delta").mode("overwrite").saveAsTable(output_table_name)
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/model_deployment/ModelDeployment.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/model_deployment/ModelDeployment.py.tmpl
new file mode 100644
index 00000000000..951ba945f9f
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/model_deployment/ModelDeployment.py.tmpl
@@ -0,0 +1,43 @@
+# Databricks notebook source
+##################################################################################
+# Helper notebook to transition the model stage. This notebook is run
+# after the Train.py notebook as part of a multi-task job, in order to transition model
+# to target stage after training completes.
+#
+# Note that we deploy the model to the stage in MLflow Model Registry equivalent to the
+# environment in which the multi-task job is executed (e.g deploy the trained model to
+# stage=Production if triggered in the prod environment). In a practical setting, we would
+# recommend enabling the model validation step between model training and automatically
+# registering the model to the Production stage in prod.
+#
+# This notebook has the following parameters:
+#
+# * env (required) - String name of the current environment for model deployment, which decides the target stage.
+# * model_uri (required) - URI of the model to deploy. Must be in the format "models://", as described in
+# https://www.mlflow.org/docs/latest/model-registry.html#fetching-an-mlflow-model-from-the-model-registry
+# This parameter is read as a task value
+# ({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/databricks-utils.html")) }}),
+# rather than as a notebook widget. That is, we assume a preceding task (the Train.py
+# notebook) has set a task value with key "model_uri".
+##################################################################################
+
+# List of input args needed to run the notebook as a job.
+# Provide them via DB widgets or notebook arguments.
+#
+# Name of the current environment
+dbutils.widgets.dropdown("env", "None", ["None", "staging", "prod"], "Environment Name")
+
+# COMMAND ----------
+
+from deploy import deploy
+
+model_uri = dbutils.jobs.taskValues.get("Train", "model_uri", debugValue="")
+env = dbutils.widgets.get("env")
+assert env != "None", "env notebook parameter must be specified"
+assert model_uri != "", "model_uri notebook parameter must be specified"
+deploy(model_uri, env)
+
+# COMMAND ----------
+print(
+ f"Successfully completed model deployment for {model_uri}"
+)
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/model_deployment/deploy.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/model_deployment/deploy.py.tmpl
new file mode 100644
index 00000000000..56254638393
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/deployment/model_deployment/deploy.py.tmpl
@@ -0,0 +1,34 @@
+import sys
+from mlflow import MlflowClient
+
+def deploy(model_uri, env):
+ """Deploys an already-registered model in Unity catalog by assigning it the appropriate alias for model deployment.
+
+ :param model_uri: URI of the model to deploy. Must be in the format "models://", as described in
+ https://www.mlflow.org/docs/latest/model-registry.html#fetching-an-mlflow-model-from-the-model-registry
+ :param env: name of the environment in which we're performing deployment, i.e one of "dev", "staging", "prod".
+ Defaults to "dev"
+ :return:
+ """
+ print(f"Deployment running in env: {env}")
+ _, model_name, version = model_uri.split("/")
+ client = MlflowClient()
+ mv = client.get_model_version(model_name, version)
+ target_alias = "champion"
+ if target_alias not in mv.aliases:
+ client.set_registered_model_alias(
+ name=model_name,
+ alias=target_alias,
+ version=version)
+ print(f"Assigned alias '{target_alias}' to model version {model_uri}.")
+
+ # remove "challenger" alias if assigning "champion" alias
+ if target_alias == "champion" and "challenger" in mv.aliases:
+ print(f"Removing 'challenger' alias from model version {model_uri}.")
+ client.delete_registered_model_alias(
+ name=model_name,
+ alias="challenger")
+
+
+if __name__ == "__main__":
+ deploy(model_uri=sys.argv[1], env=sys.argv[2])
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/MonitoredMetricViolationCheck.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/MonitoredMetricViolationCheck.py.tmpl
new file mode 100644
index 00000000000..7db01bf1433
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/MonitoredMetricViolationCheck.py.tmpl
@@ -0,0 +1,61 @@
+# Databricks notebook source
+##################################################################################
+# This notebook runs a sql query and set the result as job task value
+#
+# This notebook has the following parameters:
+#
+# * table_name_under_monitor (required) - The name of a table that is currently being monitored
+# * metric_to_monitor (required) - Metric to be monitored for threshold violation
+# * metric_violation_threshold (required) - Threshold value for metric violation
+# * num_evaluation_windows (required) - Number of windows to check for violation
+# * num_violation_windows (required) - Number of windows that need to violate the threshold
+##################################################################################
+
+# List of input args needed to run the notebook as a job.
+# Provide them via DB widgets or notebook arguments.
+#
+# Name of the table that is currently being monitored
+dbutils.widgets.text(
+ "table_name_under_monitor", "{{ .input_inference_table_name }}", label="Full (three-Level) table name"
+)
+# Metric to be used for threshold violation check
+dbutils.widgets.text(
+ "metric_to_monitor", "root_mean_squared_error", label="Metric to be monitored for threshold violation"
+)
+
+# Threshold value to be checked
+dbutils.widgets.text(
+ "metric_violation_threshold", "100", label="Threshold value for metric violation"
+)
+
+# Threshold value to be checked
+dbutils.widgets.text(
+ "num_evaluation_windows", "5", label="Number of windows to check for violation"
+)
+
+# Threshold value to be checked
+dbutils.widgets.text(
+ "num_violation_windows", "2", label="Number of windows that need to violate the threshold"
+)
+
+# COMMAND ----------
+
+from metric_violation_check_query import sql_query
+
+table_name_under_monitor = dbutils.widgets.get("table_name_under_monitor")
+metric_to_monitor = dbutils.widgets.get("metric_to_monitor")
+metric_violation_threshold = dbutils.widgets.get("metric_violation_threshold")
+num_evaluation_windows = dbutils.widgets.get("num_evaluation_windows")
+num_violation_windows = dbutils.widgets.get("num_violation_windows")
+
+formatted_sql_query = sql_query.format(
+ table_name_under_monitor=table_name_under_monitor,
+ metric_to_monitor=metric_to_monitor,
+ metric_violation_threshold=metric_violation_threshold,
+ num_evaluation_windows=num_evaluation_windows,
+ num_violation_windows=num_violation_windows)
+is_metric_violated = bool(spark.sql(formatted_sql_query).toPandas()["query_result"][0])
+
+dbutils.jobs.taskValues.set("is_metric_violated", is_metric_violated)
+
+
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/README.md.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/README.md.tmpl
new file mode 100644
index 00000000000..55194d92051
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/README.md.tmpl
@@ -0,0 +1,10 @@
+# Monitoring
+
+To enable monitoring as part of a scheduled Databricks workflow, please:
+- Create the inference table that you want to monitor and was passed in as an initialization parameter.
+- Update all the TODOs in the [monitoring resource file](../resources/monitoring-resource.yml).
+- Uncomment the monitoring workflow from the main Databricks Asset Bundles file [databricks.yml](../databricks.yml).
+
+For more details, refer to [{{template `project_name_alphanumeric_underscore` .}}/resources/README.md](../resources/README.md).
+The implementation supports monitoring of batch inference tables directly.
+For real time inference tables, unpacking is required before monitoring can be attached.
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/metric_violation_check_query.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/metric_violation_check_query.py.tmpl
new file mode 100644
index 00000000000..7be5eb295d5
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/monitoring/metric_violation_check_query.py.tmpl
@@ -0,0 +1,79 @@
+# This file is used for the main SQL query that checks the last {num_evaluation_windows} metric violations and whether at least {num_violation_windows} of those runs violate the condition.
+
+"""The SQL query is divided into three main parts. The first part selects the top {num_evaluation_windows}
+values of the metric to be monitored, ordered by the time window, and saves as recent_metrics.
+```sql
+WITH recent_metrics AS (
+ SELECT
+ {metric_to_monitor},
+ window
+ FROM
+ {table_name_under_monitor}_profile_metrics
+ WHERE
+ column_name = ":table"
+ AND slice_key IS NULL
+ AND model_id != "*"
+ AND log_type = "INPUT"
+ ORDER BY
+ window DESC
+ LIMIT
+ {num_evaluation_windows}
+)
+```
+The `column_name = ":table"` and `slice_key IS NULL` conditions ensure that the metric
+is selected for the entire table within the given granularity. The `log_type = "INPUT"`
+condition ensures that the primary table metrics are considered, but not the baseline
+table metrics. The `model_id!= "*"` condition ensures that the metric aggregated across
+all model IDs is not selected.
+
+The second part of the query determines if the metric values have been violated with two cases.
+The first case checks if the metric value is greater than the threshold for at least {num_violation_windows} windows:
+```sql
+(SELECT COUNT(*) FROM recent_metrics WHERE {metric_to_monitor} > {metric_violation_threshold}) >= {num_violation_windows}
+```
+The second case checks if the most recent metric value is greater than the threshold. This is to make sure we only trigger retraining
+if the most recent window was violated, avoiding unnecessary retraining if the violation was in the past and the metric is now within the threshold:
+```sql
+(SELECT {metric_to_monitor} FROM recent_metrics ORDER BY window DESC LIMIT 1) > {metric_violation_threshold}
+```
+
+The final part of the query sets the `query_result` to 1 if both of the above conditions are met, and 0 otherwise:
+```sql
+SELECT
+ CASE
+ WHEN
+ # Check if the metric value is greater than the threshold for at least {num_violation_windows} windows
+ AND
+ # Check if the most recent metric value is greater than the threshold
+ THEN 1
+ ELSE 0
+ END AS query_result
+```
+"""
+
+sql_query = """WITH recent_metrics AS (
+ SELECT
+ {metric_to_monitor},
+ window
+ FROM
+ {table_name_under_monitor}_profile_metrics
+ WHERE
+ column_name = ":table"
+ AND slice_key IS NULL
+ AND model_id != "*"
+ AND log_type = "INPUT"
+ ORDER BY
+ window DESC
+ LIMIT
+ {num_evaluation_windows}
+)
+SELECT
+ CASE
+ WHEN
+ (SELECT COUNT(*) FROM recent_metrics WHERE {metric_to_monitor} > {metric_violation_threshold}) >= {num_violation_windows}
+ AND
+ (SELECT {metric_to_monitor} FROM recent_metrics ORDER BY window DESC LIMIT 1) > {metric_violation_threshold}
+ THEN 1
+ ELSE 0
+ END AS query_result
+"""
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/project_params.json.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/project_params.json.tmpl
new file mode 100644
index 00000000000..7273af903ba
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/project_params.json.tmpl
@@ -0,0 +1,4 @@
+{
+ "input_cloud": "{{ .input_cloud }}",
+ "input_include_feature_store": "{{ .input_include_feature_store }}"
+}
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/pytest.ini.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/pytest.ini.tmpl
new file mode 100644
index 00000000000..04680e74955
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/pytest.ini.tmpl
@@ -0,0 +1,4 @@
+# Configure pytest to detect local modules in the current directory
+# See https://docs.pytest.org/en/7.1.x/reference/reference.html#confval-pythonpath for details
+[pytest]
+pythonpath = .
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/requirements.txt.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/requirements.txt.tmpl
new file mode 100644
index 00000000000..c9329d48c5c
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/requirements.txt.tmpl
@@ -0,0 +1,7 @@
+mlflow>=3.1
+lightgbm
+numpy
+pandas
+scikit-learn>=1.1.1
+matplotlib>=3.5.2
+pillow>=10.0.1
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/README.md.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/README.md.tmpl
new file mode 100644
index 00000000000..0e9f4618554
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/README.md.tmpl
@@ -0,0 +1,315 @@
+# Databricks ML Resource Configurations
+[(back to project README)](../README.md)
+
+## Table of contents
+* [Intro](#intro)
+* [Local development and dev workspace](#local-development-and-dev-workspace)
+* [Develop and test config changes](#develop-and-test-config-changes)
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`) }}
+* [CI/CD](#set-up-cicd)
+* [Deploy initial ML resources](#deploy-initial-ml-resources)
+* [Deploy config changes](#deploy-config-changes)
+{{- end }}
+
+## Intro
+
+### databricks CLI bundles
+MLOps Stacks ML resources are configured and deployed through [databricks CLI bundles]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/cli/bundle-cli.html")) }}).
+The bundle setting file must be expressed in YAML format and must contain at minimum the top-level bundle mapping.
+
+The databricks CLI bundles top level is defined by file `{{template `project_name_alphanumeric_underscore` .}}/databricks.yml`.
+During databricks CLI bundles deployment, the root config file will be loaded, validated and deployed to workspace provided by the environment together with all the included resources.
+
+ML Resource Configurations in this directory:
+ - model workflow (`{{template `project_name_alphanumeric_underscore` .}}/resources/model-workflow-resource.yml`)
+ - batch inference workflow (`{{template `project_name_alphanumeric_underscore` .}}/resources/batch-inference-workflow-resource.yml`)
+ - monitoring resource and workflow (`{{template `project_name_alphanumeric_underscore` .}}/resources/monitoring-resource.yml`)
+ - feature engineering workflow (`{{template `project_name_alphanumeric_underscore` .}}/resources/feature-engineering-workflow-resource.yml`)
+ - model definition and experiment definition (`{{template `project_name_alphanumeric_underscore` .}}/resources/ml-artifacts-resource.yml`)
+
+
+### Deployment Config & CI/CD integration
+The ML resources can be deployed to databricks workspace based on the databricks CLI bundles deployment config.
+Deployment configs of different deployment targets share the general ML resource configurations with added ability to specify deployment target specific values (workspace URI, model name, jobs notebook parameters, etc).
+
+{{- if (eq .input_setup_cicd_and_project `Project_Only`) }}
+
+NOTE: This project was not setup with CI/CD workflows. You can setup CI/CD with a new initialization of MLOps Stacks. The rest of this section only applies if you are using a monorepo setup with CI/CD previously or have setup CI/CD otherwise.
+{{- else }}
+This project ships with CI/CD workflows for developing and deploying ML resource configurations based on deployment config.
+{{- end }}
+
+
+For Model Registry in Unity Catalog, we expect a catalog to exist with the name of the deployment target by default. For example, if the deployment target is `dev`, we expect a catalog named `dev` to exist in the workspace.
+If you want to use different catalog names, please update the `targets` declared in the `{{template `project_name_alphanumeric_underscore` .}}/databricks.yml` and `{{template `project_name_alphanumeric_underscore` .}}/resources/ml-artifacts-resource.yml` files.
+If changing the `staging`, `prod`, or `test` deployment targets, you'll need to update the
+{{- if (eq .input_setup_cicd_and_project `Project_Only`) }} workflows located in the root directory if in a monorepo setup with CI/CD. Otherwise you can setup CI/CD with a new initialization of MLOps Stacks.
+{{- else if or (eq .input_cicd_platform `github_actions`) (eq .input_cicd_platform `github_actions_for_github_enterprise_servers`) }} workflows located in the `.github/workflows` directory.
+{{- else if (eq .input_cicd_platform `azure_devops`) }} pipelines located in the `azure-pipelines` directory.
+{{- else if (eq .input_cicd_platform `gitlab`) }} pipelines located in the `gitlab` directory.{{- end }}
+
+
+| Deployment Target | Description | Databricks Workspace | Model Name | Experiment Name |
+|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------|-------------------------------------|------------------------------------------------|
+| dev | The `dev` deployment target is used by ML engineers to deploy ML resources to development workspace with `dev` configs. The config is for ML project development purposes. | dev workspace | dev-{{template `model_name` .}} | /dev-{{template `experiment_base_name` .}} |
+| staging | The `staging` deployment target is part of the CD pipeline. Latest {{ .input_default_branch }} content will be deployed to staging workspace with `staging` config. | staging workspace | staging-{{template `model_name` .}} | /staging-{{template `experiment_base_name` .}} |
+| prod | The `prod` deployment target is part of the CD pipeline. Latest {{ .input_release_branch }} content will be deployed to prod workspace with `prod` config. | prod workspace | prod-{{template `model_name` .}} | /prod-{{template `experiment_base_name` .}} |
+| test | The `test` deployment target is part of the CI pipeline. For changes targeting the {{ .input_default_branch }} branch, upon making a PR, an integration test will be triggered and ML resources deployed to the staging workspace defined under `test` deployment target. | staging workspace | test-{{template `model_name` .}} | /test-{{template `experiment_base_name` .}} |
+
+During ML code development, you can deploy local ML resource configurations together with ML code to the a Databricks workspace to run the training, model validation or batch inference pipelines. The deployment will use `dev` config by default.
+
+You can open a PR (pull request) to modify ML code or the resource config against {{ .input_default_branch }} branch.
+The PR will trigger Python unit tests, followed by an integration test executed on the staging workspace, as defined under the `test` environment resource.
+
+Upon merging a PR to the {{ .input_default_branch }} branch, the {{ .input_default_branch }} branch content will be deployed to the staging workspace with `staging` environment resource configurations.
+
+Upon merging code into the release branch, the release branch content will be deployed to prod workspace with `prod` environment resource configurations.
+
+
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`) }}
+
+{{- end }}
+
+## Local development and dev workspace
+
+### Set up authentication
+
+To set up the databricks CLI using a Databricks personal access token, take the following steps:
+
+1. Follow [databricks CLI]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/cli/databricks-cli.html")) }}) to download and set up the databricks CLI locally.
+2. Complete the `TODO` in `{{template `project_name_alphanumeric_underscore` .}}/databricks.yml` to add the dev workspace URI under `targets.dev.workspace.host`.
+3. [Create a personal access token]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/auth/pat.html")) }})
+ in your dev workspace and copy it.
+4. Set an env variable `DATABRICKS_TOKEN` with your Databricks personal access token in your terminal. For example, run `export DATABRICKS_TOKEN=dapi12345` if the access token is dapi12345.
+5. You can now use the databricks CLI to validate and deploy ML resource configurations to the dev workspace.
+
+Alternatively, you can use the other approaches described in the [databricks CLI]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "dev-tools/cli/databricks-cli.html")) }}) documentation to set up authentication. For example, using your Databricks username/password, or seting up a local profile.
+
+### Validate and provision ML resource configurations
+1. After installing the databricks CLI and creating the `DATABRICKS_TOKEN` env variable, change to the `{{template `project_name_alphanumeric_underscore` .}}` directory.
+2. Run `databricks bundle validate` to validate the Databricks resource configurations.
+3. Run `databricks bundle deploy` to provision the Databricks resource configurations to the dev workspace. The resource configurations and your ML code will be copied together to the dev workspace. The defined resources such as Lakeflow Jobs, MLflow Model and MLflow Experiment will be provisioned according to the config files under `{{template `project_name_alphanumeric_underscore` .}}/resources`.
+4. Go to the Databricks dev workspace, check the defined model, experiment and workflows status, and interact with the created workflows.
+
+### Destroy ML resource configurations
+After development is done, you can run `databricks bundle destroy` to destroy (remove) the defined Databricks resources in the dev workspace. Any model version with `Production` or `Staging` stage will prevent the model from being deleted. Please update the version stage to `None` or `Archived` before destroying the ML resources.
+
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`) }}
+## Set up CI/CD
+Please refer to [mlops-setup](../../docs/mlops-setup.md#configure-cicd) for instructions to set up CI/CD.
+
+## Deploy initial ML resources
+After completing the prerequisites, create and push a PR branch adding all files to the Git repo:
+```
+git checkout -b add-ml-resource-config-and-code
+git add .
+git commit -m "Add ML resource config and ML code"
+git push upstream add-ml-resource-config-and-code
+```
+Open a pull request to merge the pushed branch into the `{{ .input_default_branch }}` branch.
+Upon creating this PR, the CI workflows will be triggered.
+These CI workflow will run unit and integration tests of the ML code,
+in addition to validating the Databricks resources to be deployed to both staging and prod workspaces.
+Once CI passes, merge the PR into the `{{ .input_default_branch }}` branch. This will deploy an initial set of Databricks resources to the staging workspace.
+resources will be deployed to the prod workspace on pushing code to the `{{ .input_release_branch }}` branch.
+
+Follow the next section to configure the input and output data tables for the batch inference job.
+{{- end }}
+
+### Setting up the batch inference job
+The batch inference job expects an input Delta table with a schema that your registered model accepts (out-of-the-box we provide a working example using the `nyctaxi` dataset with the schema [trip_distance, pickup_zip and dropoff_zip]). To use the batch
+inference job, set up such a Delta table in both your staging and prod workspaces. For example, with the `nyctaxi` dataset, the following code can be run in a Databricks notebook to create the batch inference input table:
+```
+# To test the batch job the training data can be used as input
+
+input_table_path = "/databricks-datasets/nyctaxi-with-zipcodes/subsampled"
+training_df = spark.read.format("delta").load(input_table_path)
+
+# drop unused columns
+df = df.drop("tpep_pickup_datetime", "tpep_dropoff_datetime")
+
+# target column used for monitoring, not for predictions
+df = df.withColumnRenamed('fare_amount', 'price')
+
+spark.sql("CREATE DATABASE IF NOT EXISTS my_catalog.my_schema")
+
+def write_to_table(df, database, table):
+ (df.write
+ .format("delta")
+ .mode("overwrite")
+ .option("overwriteSchema", "true")
+ .saveAsTable(f"{database}.{table}"))
+
+# Write the DataFrame to a Delta table
+write_to_table(training_df, database="my_catalog.my_schema", table="batch_input_table")
+```
+Following this, update the batch_inference_job base parameters in `{{template `project_name_alphanumeric_underscore` .}}/resources/batch-inference-workflow-resource.yml` to pass
+the name of the input Delta table and the name of the output Delta table to which to write batch predictions.
+
+As the batch job will be run with the credentials of the service principal that provisioned it, make sure that the service
+principal corresponding to a particular environment has permissions to read the input Delta table and modify the output Delta table in that environment's workspace. If the Delta table is in the [Unity Catalog](https://www.databricks.com/product/unity-catalog), these permissions are
+
+* `USAGE` permissions for the catalog and schema of the input and output table.
+* `SELECT` permission for the input table.
+* `MODIFY` permission for the output table if it pre-dates your job.
+
+### Setting up model validation
+The model validation workflow focuses on building a plug-and-play stack component for continuous deployment (CD) of models
+in staging and prod.
+Its central purpose is to evaluate a registered model and validate its quality before deploying the model to Production/Staging.
+
+Model validation contains three components:
+* [model-workflow-resource.yml](./model-workflow-resource.yml) contains the resource config and input parameters for model validation.
+* [validation.py](../validation/validation.py) defines custom metrics and validation thresholds that are referenced by the above resource config files.
+* [ModelValidation](../validation/ModelValidation.py) contains the validation job implementation. In most cases you don't need to modify this file.
+
+To set up and enable model validation, update [validation.py](../validation/validation.py) to return desired custom metrics and validation thresholds, then
+resolve the `TODOs` in the ModelValidation task of [model-workflow-resource.yml](./model-workflow-resource.yml).
+
+
+### Setting up monitoring
+The monitoring workflow focuses on building a plug-and-play stack component for monitoring the feature drifts and model drifts and retrain based on the
+violation threshold defined given the ground truth labels.
+
+Its central purpose is to track production model performances, feature distributions and comparing different versions.
+
+Monitoring contains four components:
+* [metric_violation_check_query.py](../monitoring/metric_violation_check_query.py) defines a query that checks for violation of the monitored metric.
+* [MonitoredMetricViolationCheck](../monitoring/MonitoredMetricViolationCheck.py) acts as an entry point, executing the violation check query against the monitored inference table.
+It emits a boolean value based on the query result.
+* [monitoring-resource.yml](./monitoring-resource.yml) contains the resource config, inputs parameters for monitoring, and orchestrates model retraining based on monitoring. It first runs the [MonitoredMetricViolationCheck](../monitoring/MonitoredMetricViolationCheck.py)
+entry point then decides whether to execute the model retraining workflow.
+
+To set up and enable monitoring:
+* If it is not done already, generate inference table, join it with ground truth labels, and update the table name in [monitoring-resource.yml](./monitoring-resource.yml).
+* Resolve the `TODOs` in [monitoring-resource.yml](./monitoring-resource.yml)
+* Uncomment the monitoring workflow in [databricks.yml](../databricks.yml)
+* OPTIONAL: Update the query in [metric_violation_check_query.py](../monitoring/metric_violation_check_query.py) to customize when the metric is considered to be in violation.
+
+NOTE: If ground truth labels are not available, you can still set up monitoring but should disable the retraining workflow.
+
+Retraining Constraints:
+The retraining job has constraints for optimal functioning:
+* Labels must be provided by the user, joined correctly for retraining history, and available on time with the retraining frequency.
+* Retraining Frequency is tightly coupled with the granularity of the monitor. Users should take into account and ensure that their retraining frequency is equal to or close to the granularity of the monitor.
+ * If the granularity of the monitor is 1 day and retraining frequency is 1 hour, the job will preemptively stop as there is no new data to evaluate retraining criteria
+ * If the granularity of the monitor is 1 day and retraining frequency is 1 week, retraining would be stale and not be efficient
+
+Permissions:
+Permissions for monitoring are inherited from the original table's permissions.
+* Users who own the monitored table or its parent catalog/schema can create, update, and view monitors.
+* Users with read permissions on the monitored table can view its monitor.
+
+Therefore, ensure that service principals are the owners or have the necessary permissions to manage the monitored table.
+
+## Develop and test config changes
+
+### databricks CLI bundles schema overview
+To get started, open `{{template `project_name_alphanumeric_underscore` .}}/resources/batch-inference-workflow-resource.yml`. The file contains the ML resource definition of a batch inference job, like:
+
+```$xslt
+resources:
+ jobs:
+ batch_inference_job:
+ name: ${bundle.target}-{{ .input_project_name }}-batch-inference-job
+ tasks:
+ - task_key: batch_inference_job
+ environment_key: default
+ notebook_task:
+ notebook_path: ../deployment/batch_inference/BatchInference.py
+ base_parameters:
+ env: ${bundle.target}
+ input_table_name: batch_inference_input_table_name
+ ...
+
+ environments:
+ - environment_key: default
+ spec:
+ environment_version: "5"
+ dependencies:
+ - -r ../requirements.txt
+```
+
+The example above defines a Databricks job with name `${bundle.target}-{{ .input_project_name }}-batch-inference-job`
+that runs the notebook under `{{template `project_name_alphanumeric_underscore` .}}/deployment/batch_inference/BatchInference.py` to regularly apply your ML model for batch inference.
+
+Jobs run on [serverless compute]({{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "jobs/run-serverless-jobs.html")) }}) by default: the task references a serverless environment by its `environment_key` (`default` here), and that key is defined once under `environments`, with its own dependencies installed from `requirements.txt`. Multiple tasks in the same job can share the same `environment_key` to reuse one environment definition.
+
+We specify a `batch_inference_job` under `resources/jobs` to define a databricks workflow with internal key `batch_inference_job` and job name `{bundle.target}-{{ .input_project_name }}-batch-inference-job`.
+The workflow contains a single task with task key `batch_inference_job`. The task runs notebook `../deployment/batch_inference/BatchInference.py` with provided parameters `env` and `input_table_name` passing to the notebook.
+After setting up databricks CLI, you can run command `databricks bundle schema` to learn more about databricks CLI bundles schema.
+
+The notebook_path is the relative path starting from the resource yaml file.
+
+### Environment config based variables
+The `${bundle.target}` will be replaced by the environment config name during the bundle deployment. For example, during the deployment of a `test` environment config, the job name will be
+`test-{{ .input_project_name }}-batch-inference-job`. During the deployment of the `staging` environment config, the job name will be
+`staging-{{ .input_project_name }}-batch-inference-job`.
+
+
+To use different values based on different environment, you can use bundle variables based on the given target, for example,
+```$xslt
+variables:
+ batch_inference_input_table:
+ description: The table name to be used for input to the batch inference workflow.
+ default: input_table
+
+targets:
+ dev:
+ variables:
+ batch_inference_input_table: dev_table
+ test:
+ variables:
+ batch_inference_input_table: test_table
+
+resources:
+ jobs:
+ batch_inference_job:
+ name: ${bundle.target}-{{ .input_project_name }}-batch-inference-job
+ tasks:
+ - task_key: batch_inference_job
+ environment_key: default
+ notebook_task:
+ notebook_path: ../deployment/batch_inference/BatchInference.py
+ base_parameters:
+ env: ${bundle.target}
+ input_table_name: ${var.batch_inference_input_table}
+ ...
+
+ environments:
+ - environment_key: default
+ spec:
+ environment_version: "5"
+ dependencies:
+ - -r ../requirements.txt
+```
+The `batch_inference_job` notebook parameter `input_table_name` is using a bundle variable `batch_inference_input_table` with default value "input_table".
+The variable value will be overwritten with "dev_table" for `dev` environment config and "test_table" for `test` environment config:
+- during deployment with the `dev` environment config, the `input_table_name` parameter will get the value "dev_table"
+- during deployment with the `staging` environment config, the `input_table_name` parameter will get the value "input_table"
+- during deployment with the `prod` environment config, the `input_table_name` parameter will get the value "input_table"
+- during deployment with the `test` environment config, the `input_table_name` parameter will get the value "test_table"
+
+### Test config changes
+To test out a config change, simply edit one of the fields above. For example, increase the cluster size by updating `num_workers` from 3 to 4.
+
+Then follow [Local development and dev workspace](#local-development-and-dev-workspace) to deploy the change to the dev workspace.
+Alternatively you can open a PR. Continuous integration will then validate the updated config and deploy tests to the to staging workspace.
+
+{{- if (eq .input_setup_cicd_and_project `CICD_and_Project`) }}
+## Deploy config changes
+
+### Dev workspace deployment
+Please refer to [Local development and dev workspace](#local-development-and-dev-workspace).
+
+### Test workspace deployment(CI)
+After setting up CI/CD, PRs against the {{ .input_default_branch }} branch will trigger CI workflows to run unit tests, integration test and resource validation.
+The integration test will deploy MLflow model, MLflow experiment and Databricks workflow resources defined under the `test` environment resource config to the staging workspace. The integration test then triggers a run of the model workflow to verify the ML code.
+
+### Staging and Prod workspace deployment(CD)
+After merging a PR to the {{ .input_default_branch }} branch, continuous deployment automation will deploy the `staging` resources to the staging workspace.
+
+When you about to cut a release, you can create and merge a PR to merge changes from {{ .input_default_branch }} to {{ .input_release_branch }}. Continuous deployment automation will deploy `prod` resources to the prod workspace.
+{{- end }}
+
+[Back to project README](../README.md)
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/batch-inference-workflow-resource.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/batch-inference-workflow-resource.yml.tmpl
new file mode 100644
index 00000000000..dd2020c6faa
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/batch-inference-workflow-resource.yml.tmpl
@@ -0,0 +1,40 @@
+common_permissions: &permissions
+ permissions:
+ - level: CAN_VIEW
+ group_name: users
+
+resources:
+ jobs:
+ batch_inference_job:
+ name: ${bundle.target}-{{ .input_project_name }}-batch-inference-job
+ tasks:
+ - task_key: batch_inference_job
+ environment_key: default
+ notebook_task:
+ notebook_path: ../deployment/batch_inference/BatchInference.py
+ base_parameters:
+ env: ${bundle.target}
+ {{ if (eq .input_include_feature_store `yes`) }}input_table_name: ${bundle.target}.{{ .input_schema_name }}.feature_store_inference_input # TODO: create input table for inference
+ {{- else -}}input_table_name: taxi_scoring_sample # TODO: create input table for inference{{ end }}
+ output_table_name: ${var.catalog_name}.{{ .input_schema_name }}.predictions
+ model_name: ${var.catalog_name}.{{ .input_schema_name }}.${var.model_name}
+ # git source information of current ML resource deployment. It will be persisted as part of the workflow run
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+
+ environments:
+ - environment_key: default
+ spec:
+ environment_version: "5"
+ dependencies:
+ - -r ../requirements.txt
+ schedule:
+ quartz_cron_expression: "0 0 11 * * ?" # daily at 11am
+ timezone_id: UTC
+ <<: *permissions
+ # If you want to turn on notifications for this job, please uncomment the below code,
+ # and provide a list of emails to the on_failure argument.
+ #
+ # email_notifications:
+ # on_failure:
+ # - first@company.com
+ # - second@company.com
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/ml-artifacts-resource.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/ml-artifacts-resource.yml.tmpl
new file mode 100644
index 00000000000..683bcd8fd2c
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/ml-artifacts-resource.yml.tmpl
@@ -0,0 +1,28 @@
+# Allow users to read the experiment
+common_permissions: &permissions
+ permissions:
+ - level: CAN_READ
+ group_name: users
+
+# Allow users to execute models in Unity Catalog
+grants: &grants
+ grants:
+ - privileges:
+ - EXECUTE
+ principal: {{ .input_unity_catalog_read_user_group }}
+
+# Defines model and experiments
+resources:
+ registered_models:
+ model:
+ name: ${var.model_name}
+ catalog_name: ${var.catalog_name}
+ schema_name: {{ .input_schema_name }}
+ comment: Registered model in Unity Catalog for the "{{ .input_project_name }}" ML Project for ${bundle.target} deployment target.
+ <<: *grants
+
+ experiments:
+ experiment:
+ name: ${var.experiment_name}
+ <<: *permissions
+ description: MLflow Experiment used to track runs for {{ .input_project_name }} project.
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/model-workflow-resource.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/model-workflow-resource.yml.tmpl
new file mode 100644
index 00000000000..35c4582bd6c
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/model-workflow-resource.yml.tmpl
@@ -0,0 +1,96 @@
+common_permissions: &permissions
+ permissions:
+ - level: CAN_VIEW
+ group_name: users
+
+resources:
+ jobs:
+ model_training_job:
+ name: ${bundle.target}-{{ .input_project_name }}-model-training-job
+ tasks:
+ - task_key: Train
+ environment_key: default
+ {{ if (eq .input_include_feature_store `no`) }}notebook_task:
+ notebook_path: ../training/Train.py
+ base_parameters:
+ env: ${bundle.target}
+ # TODO: Update training_data_path
+ training_data_path: /databricks-datasets/nyctaxi-with-zipcodes/subsampled
+ experiment_name: ${var.experiment_name}
+ model_name: ${var.catalog_name}.{{ .input_schema_name }}.${var.model_name}
+ # git source information of current ML resource deployment. It will be persisted as part of the workflow run
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+ {{ else if (eq .input_include_feature_store `yes`) }}notebook_task:
+ notebook_path: ../training/TrainWithFeatureStore.py
+ base_parameters:
+ env: ${bundle.target}
+ # TODO: Update training_data_path
+ training_data_path: /databricks-datasets/nyctaxi-with-zipcodes/subsampled
+ experiment_name: ${var.experiment_name}
+ model_name: ${var.catalog_name}.{{ .input_schema_name }}.${var.model_name}
+ pickup_features_table: ${var.catalog_name}.{{ .input_schema_name }}.trip_pickup_features
+ dropoff_features_table: ${var.catalog_name}.{{ .input_schema_name }}.trip_dropoff_features
+ # git source information of current ML resource deployment. It will be persisted as part of the workflow run
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+ {{ end }}
+ - task_key: ModelValidation
+ environment_key: default
+ depends_on:
+ - task_key: Train
+ notebook_task:
+ notebook_path: ../validation/ModelValidation.py
+ base_parameters:
+ experiment_name: ${var.experiment_name}
+ # The `run_mode` defines whether model validation is enabled or not.
+ # It can be one of the three values:
+ # `disabled` : Do not run the model validation notebook.
+ # `dry_run` : Run the model validation notebook. Ignore failed model validation rules and proceed to move
+ # model to Production stage.
+ # `enabled` : Run the model validation notebook. Move model to Production stage only if all model validation
+ # rules are passing.
+ # TODO: update run_mode
+ run_mode: dry_run
+ # Whether to load the current registered "Production" stage model as baseline.
+ # Baseline model is a requirement for relative change and absolute change validation thresholds.
+ # TODO: update enable_baseline_comparison
+ enable_baseline_comparison: "false"
+ # Please refer to data parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+ # TODO: update validation_input
+ validation_input: SELECT * FROM delta.`dbfs:/databricks-datasets/nyctaxi-with-zipcodes/subsampled`
+ # A string describing the model type. The model type can be either "regressor" and "classifier".
+ # Please refer to model_type parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+ # TODO: update model_type
+ model_type: regressor
+ # The string name of a column from data that contains evaluation labels.
+ # Please refer to targets parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+ # TODO: targets
+ targets: fare_amount
+ # git source information of current ML resource deployment. It will be persisted as part of the workflow run
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+ - task_key: ModelDeployment
+ environment_key: default
+ depends_on:
+ - task_key: ModelValidation
+ notebook_task:
+ notebook_path: ../deployment/model_deployment/ModelDeployment.py
+ base_parameters:
+ env: ${bundle.target}
+ # git source information of current ML resource deployment. It will be persisted as part of the workflow run
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+ environments:
+ - environment_key: default
+ spec:
+ environment_version: "5"
+ dependencies:
+ - -r ../requirements.txt
+ schedule:
+ quartz_cron_expression: "0 0 9 * * ?" # daily at 9am
+ timezone_id: UTC
+ <<: *permissions
+ # If you want to turn on notifications for this job, please uncomment the below code,
+ # and provide a list of emails to the on_failure argument.
+ #
+ # email_notifications:
+ # on_failure:
+ # - first@company.com
+ # - second@company.com
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/monitoring-resource.yml.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/monitoring-resource.yml.tmpl
new file mode 100644
index 00000000000..1e9129fa6d8
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/resources/monitoring-resource.yml.tmpl
@@ -0,0 +1,80 @@
+# Please complete all the TODOs in this file.
+# The regression monitor defined here works OOB with this example regression notebook: {{ template `generate_doc_link` (map (pair "cloud" .input_cloud) (pair "path" "_extras/notebooks/source/monitoring/regression-monitor.html")) }}
+# NOTE: Monitoring only works on Unity Catalog tables.
+
+common_permissions: &permissions
+ permissions:
+ - level: CAN_VIEW
+ group_name: users
+
+resources:
+ quality_monitors:
+ {{ .input_project_name }}_quality_monitor:
+ table_name: {{ .input_inference_table_name }}
+ # TODO: Update the output schema name as per your requirements
+ output_schema_name: ${bundle.target}.{{ .input_project_name }}
+ # TODO: Update the below parameters as per your requirements
+ assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring
+ inference_log:
+ granularities: [1 day]
+ model_id_col: model_id
+ prediction_col: prediction
+ label_col: price
+ problem_type: PROBLEM_TYPE_REGRESSION
+ timestamp_col: timestamp
+ schedule:
+ quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am
+ timezone_id: UTC
+ jobs:
+ retraining_job:
+ name: ${bundle.target}-{{ .input_project_name }}-monitoring-retraining-job
+ tasks:
+ - task_key: monitored_metric_violation_check
+ environment_key: default
+ notebook_task:
+ notebook_path: ../monitoring/MonitoredMetricViolationCheck.py
+ base_parameters:
+ env: ${bundle.target}
+ table_name_under_monitor: {{ .input_inference_table_name }}
+ # TODO: Update the metric to be monitored and violation threshold
+ metric_to_monitor: root_mean_squared_error
+ metric_violation_threshold: 100
+ num_evaluation_windows: 5
+ num_violation_windows: 2
+ # git source information of current ML resource deployment. It will be persisted as part of the workflow run
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+
+ - task_key: is_metric_violated
+ depends_on:
+ - task_key: monitored_metric_violation_check
+ condition_task:
+ op: EQUAL_TO
+ left: "{{"{{tasks.monitored_metric_violation_check.values.is_metric_violated}}"}}"
+ right: "true"
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+
+ - task_key: trigger_retraining
+ depends_on:
+ - task_key: is_metric_violated
+ outcome: "true"
+ run_job_task:
+ job_id: ${resources.jobs.model_training_job.id}
+ git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
+
+ environments:
+ - environment_key: default
+ spec:
+ environment_version: "5"
+ dependencies:
+ - -r ../requirements.txt
+ schedule:
+ quartz_cron_expression: "0 0 18 * * ?" # daily at 6pm
+ timezone_id: UTC
+ <<: *permissions
+ # If you want to turn on notifications for this job, please uncomment the below code,
+ # and provide a list of emails to the on_failure argument.
+ #
+ # email_notifications:
+ # on_failure:
+ # - first@company.com
+ # - second@company.com
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/__init__.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/__init__.py.tmpl
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/training/__init__.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/training/__init__.py.tmpl
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/training/test_notebooks.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/training/test_notebooks.py.tmpl
new file mode 100644
index 00000000000..7293b1b382f
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/tests/training/test_notebooks.py.tmpl
@@ -0,0 +1,9 @@
+import pathlib
+
+
+def test_notebook_format():
+ # Verify that all Databricks notebooks have the required header
+ paths = list(pathlib.Path("./notebooks").glob("**/*.py"))
+ for f in paths:
+ notebook_str = open(str(f)).read()
+ assert notebook_str.startswith("# Databricks notebook source")
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/training/Train.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/training/Train.py.tmpl
new file mode 100644
index 00000000000..77896142264
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/training/Train.py.tmpl
@@ -0,0 +1,139 @@
+# Databricks notebook source
+##################################################################################
+# Model Training Notebook
+#
+# This notebook shows an example of a Model Training pipeline using Delta tables.
+# It is configured and can be executed as the "Train" task in the model_training_job workflow defined under
+# ``{{template `project_name_alphanumeric_underscore` .}}/resources/model-workflow-resource.yml``
+#
+# Parameters:
+# * env (required): - Environment the notebook is run in (staging, or prod). Defaults to "staging".
+# * training_data_path (required) - Path to the training data.
+# * experiment_name (required) - MLflow experiment name for the training runs. Will be created if it doesn't exist.
+# * model_name (required) - Three-level name (..) to register the trained model in Unity Catalog.
+#
+##################################################################################
+
+# COMMAND ----------
+
+# DBTITLE 1, Notebook arguments
+# List of input args needed to run this notebook as a job.
+# Provide them via DB widgets or notebook arguments.
+
+# Notebook Environment
+dbutils.widgets.dropdown("env", "staging", ["staging", "prod"], "Environment Name")
+env = dbutils.widgets.get("env")
+
+# Path to the Hive-registered Delta table containing the training data.
+dbutils.widgets.text(
+ "training_data_path",
+ "/databricks-datasets/nyctaxi-with-zipcodes/subsampled",
+ label="Path to the training data",
+)
+
+# MLflow experiment name.
+dbutils.widgets.text(
+ "experiment_name",
+ f"/dev-{{template `experiment_base_name` .}}",
+ label="MLflow experiment name",
+)
+
+# Unity Catalog registered model name to use for the trained model.
+dbutils.widgets.text(
+ "model_name", "dev.{{ .input_schema_name }}.{{template `model_name` .}}", label="Full (Three-Level) Model Name"
+)
+
+
+# COMMAND ----------
+
+# DBTITLE 1,Define input and output variables
+input_table_path = dbutils.widgets.get("training_data_path")
+experiment_name = dbutils.widgets.get("experiment_name")
+model_name = dbutils.widgets.get("model_name")
+
+# COMMAND ----------
+
+# DBTITLE 1, Set experiment
+import mlflow
+
+mlflow.set_experiment(experiment_name)
+
+# COMMAND ----------
+
+# DBTITLE 1, Load raw data
+training_df = spark.read.format("delta").load(input_table_path)
+training_df.display()
+
+# COMMAND ----------
+
+# DBTITLE 1, Helper function
+from mlflow import MlflowClient
+import mlflow.pyfunc
+
+
+def get_latest_model_version(model_name):
+ latest_version = 1
+ mlflow_client = MlflowClient()
+ for mv in mlflow_client.search_model_versions(f"name='{model_name}'"):
+ version_int = int(mv.version)
+ if version_int > latest_version:
+ latest_version = version_int
+ return latest_version
+
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC Train a LightGBM model on the data, then log and register the model with MLflow.
+
+# COMMAND ----------
+
+# DBTITLE 1, Train model
+import mlflow
+from sklearn.model_selection import train_test_split
+import lightgbm as lgb
+import mlflow.lightgbm
+
+# Collect data into a Pandas array for training. Since the timestamp columns would likely
+# cause the model to overfit the data, exclude them to avoid training on them.
+columns = [col for col in training_df.columns if col not in ['tpep_pickup_datetime', 'tpep_dropoff_datetime']]
+data = training_df.toPandas()[columns]
+
+train, test = train_test_split(data, random_state=123)
+X_train = train.drop(["fare_amount"], axis=1)
+X_test = test.drop(["fare_amount"], axis=1)
+y_train = train.fare_amount
+y_test = test.fare_amount
+
+mlflow.lightgbm.autolog()
+train_lgb_dataset = lgb.Dataset(X_train, label=y_train.values)
+test_lgb_dataset = lgb.Dataset(X_test, label=y_test.values)
+
+param = {"num_leaves": 32, "objective": "regression", "metric": "rmse"}
+num_rounds = 100
+
+# Train a lightGBM model
+model = lgb.train(param, train_lgb_dataset, num_rounds)
+
+# COMMAND ----------
+
+# DBTITLE 1, Log model and return output.
+# Take the first row of the training dataset as the model input example.
+input_example = X_train.iloc[[0]]
+
+# Log the trained model with MLflow
+mlflow.lightgbm.log_model(
+ model,
+ artifact_path="lgb_model",
+ # The signature is automatically inferred from the input example and its predicted output.
+ input_example=input_example,
+ registered_model_name=model_name
+)
+
+# The returned model URI is needed by the model deployment notebook.
+model_version = get_latest_model_version(model_name)
+model_uri = f"models:/{model_name}/{model_version}"
+dbutils.jobs.taskValues.set("model_uri", model_uri)
+dbutils.jobs.taskValues.set("model_name", model_name)
+dbutils.jobs.taskValues.set("model_version", model_version)
+dbutils.notebook.exit(model_uri)
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/ModelValidation.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/ModelValidation.py.tmpl
new file mode 100644
index 00000000000..5d7025a94d3
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/ModelValidation.py.tmpl
@@ -0,0 +1,315 @@
+# Databricks notebook source
+##################################################################################
+# Model Validation Notebook
+##
+# This notebook uses mlflow model validation API to run mode validation after training and registering a model
+# in model registry, before deploying it to the "champion" alias.
+#
+# It runs as part of CD and by an automated model training job -> validation -> deployment job defined under ``{{template `project_name_alphanumeric_underscore` .}}/resources/model-workflow-resource.yml``
+#
+#
+# Parameters:
+#
+# * env - Name of the environment the notebook is run in (staging, or prod). Defaults to "prod".
+# * `run_mode` - The `run_mode` defines whether model validation is enabled or not. It can be one of the three values:
+# * `disabled` : Do not run the model validation notebook.
+# * `dry_run` : Run the model validation notebook. Ignore failed model validation rules and proceed to move
+# model to the "champion" alias.
+# * `enabled` : Run the model validation notebook. Move model to the "champion" alias only if all model validation
+# rules are passing.
+# * enable_baseline_comparison - Whether to load the current registered "champion" model as baseline.
+# Baseline model is a requirement for relative change and absolute change validation thresholds.
+# * validation_input - Validation input. Please refer to data parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+# * model_type - A string describing the model type. The model type can be either "regressor" and "classifier".
+# Please refer to model_type parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+# * targets - The string name of a column from data that contains evaluation labels.
+# Please refer to targets parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+#
+# Custom metrics, validation thresholds, and evaluator config are defined in validation/validation.py.
+#
+# For details on mlflow evaluate API, see doc https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+# For details and examples about performing model validation, see the Model Validation documentation https://mlflow.org/docs/latest/models.html#model-validation
+#
+##################################################################################
+
+# COMMAND ----------
+
+dbutils.widgets.text(
+ "experiment_name",
+ "/dev-{{template `experiment_base_name` .}}",
+ "Experiment Name",
+)
+dbutils.widgets.dropdown("run_mode", "disabled", ["disabled", "dry_run", "enabled"], "Run Mode")
+dbutils.widgets.dropdown("enable_baseline_comparison", "false", ["true", "false"], "Enable Baseline Comparison")
+dbutils.widgets.text("validation_input", "SELECT * FROM delta.`dbfs:/databricks-datasets/nyctaxi-with-zipcodes/subsampled`", "Validation Input")
+dbutils.widgets.text("model_type", "regressor", "Model Type")
+dbutils.widgets.text("targets", "fare_amount", "Targets")
+dbutils.widgets.text("model_name", "dev.{{ .input_schema_name }}.{{template `model_name` .}}", "Full (Three-Level) Model Name")
+
+dbutils.widgets.text("model_version", "", "Candidate Model Version")
+
+# COMMAND ----------
+run_mode = dbutils.widgets.get("run_mode").lower()
+assert run_mode == "disabled" or run_mode == "dry_run" or run_mode == "enabled"
+
+if run_mode == "disabled":
+ print(
+ "Model validation is in DISABLED mode. Exit model validation without blocking model deployment."
+ )
+ dbutils.notebook.exit(0)
+dry_run = run_mode == "dry_run"
+
+if dry_run:
+ print(
+ "Model validation is in DRY_RUN mode. Validation threshold validation failures will not block model deployment."
+ )
+else:
+ print(
+ "Model validation is in ENABLED mode. Validation threshold validation failures will block model deployment."
+ )
+
+# COMMAND ----------
+
+import mlflow
+import os
+import tempfile
+import traceback
+from mlflow import MlflowClient
+
+client = MlflowClient()
+
+# set experiment
+experiment_name = dbutils.widgets.get("experiment_name")
+mlflow.set_experiment(experiment_name)
+# set model evaluation parameters that can be inferred from the job
+model_uri = dbutils.jobs.taskValues.get("Train", "model_uri", debugValue="")
+model_name = dbutils.jobs.taskValues.get("Train", "model_name", debugValue="")
+model_version = dbutils.jobs.taskValues.get("Train", "model_version", debugValue="")
+
+if model_uri == "":
+ model_name = dbutils.widgets.get("model_name")
+ model_version = dbutils.widgets.get("model_version")
+ model_uri = "models:/" + model_name + "/" + model_version
+
+baseline_model_uri = "models:/" + model_name + "@champion"
+
+evaluators = "default"
+assert model_uri != "", "model_uri notebook parameter must be specified"
+assert model_name != "", "model_name notebook parameter must be specified"
+assert model_version != "", "model_version notebook parameter must be specified"
+
+# COMMAND ----------
+
+# take input
+enable_baseline_comparison = dbutils.widgets.get("enable_baseline_comparison")
+
+{{ if (eq .input_include_feature_store `yes`) }}
+enable_baseline_comparison = "false"
+print(
+ "Currently baseline model comparison is not supported for models registered with feature store. Please refer to "
+ "issue https://github.com/databricks/mlops-stacks/issues/70 for more details."
+)
+{{ end }}
+assert enable_baseline_comparison == "true" or enable_baseline_comparison == "false"
+enable_baseline_comparison = enable_baseline_comparison == "true"
+
+validation_input = dbutils.widgets.get("validation_input")
+assert validation_input
+data = spark.sql(validation_input)
+
+model_type = dbutils.widgets.get("model_type")
+targets = dbutils.widgets.get("targets")
+
+assert model_type
+assert targets
+
+from validation import custom_metrics as _custom_metrics, validation_thresholds as _validation_thresholds, evaluator_config as _evaluator_config
+custom_metrics = _custom_metrics()
+validation_thresholds = _validation_thresholds()
+evaluator_config = _evaluator_config()
+
+# COMMAND ----------
+
+# helper methods
+def get_run_link(run_info):
+ return "[Run](#mlflow/experiments/{0}/runs/{1})".format(
+ run_info.experiment_id, run_info.run_id
+ )
+
+
+def get_training_run(model_name, model_version):
+ version = client.get_model_version(model_name, model_version)
+ if version.run_id:
+ return mlflow.get_run(run_id=version.run_id)
+ return None
+
+
+def generate_run_name(training_run):
+ return None if not training_run else training_run.info.run_name + "-validation"
+
+
+def generate_description(training_run):
+ return (
+ None
+ if not training_run
+ else "Model Training Details: {0}\n".format(get_run_link(training_run.info))
+ )
+
+
+def log_to_model_description(run, success):
+ run_link = get_run_link(run.info)
+ description = client.get_model_version(model_name, model_version).description
+ status = "SUCCESS" if success else "FAILURE"
+ if description != "":
+ description += "\n\n---\n\n"
+ description += "Model Validation Status: {0}\nValidation Details: {1}".format(
+ status, run_link
+ )
+ client.update_model_version(
+ name=model_name, version=model_version, description=description
+ )
+
+{{ if (eq .input_include_feature_store `yes`) }}
+
+from datetime import timedelta, timezone
+import math
+import pyspark.sql.functions as F
+from pyspark.sql.types import IntegerType
+
+
+def rounded_unix_timestamp(dt, num_minutes=15):
+ """
+ Ceilings datetime dt to interval num_minutes, then returns the unix timestamp.
+ """
+ nsecs = dt.minute * 60 + dt.second + dt.microsecond * 1e-6
+ delta = math.ceil(nsecs / (60 * num_minutes)) * (60 * num_minutes) - nsecs
+ return int((dt + timedelta(seconds=delta)).replace(tzinfo=timezone.utc).timestamp())
+
+
+rounded_unix_timestamp_udf = F.udf(rounded_unix_timestamp, IntegerType())
+
+
+def rounded_taxi_data(taxi_data_df):
+ # Round the taxi data timestamp to 15 and 30 minute intervals so we can join with the pickup and dropoff features
+ # respectively.
+ taxi_data_df = (
+ taxi_data_df.withColumn(
+ "rounded_pickup_datetime",
+ F.to_timestamp(
+ rounded_unix_timestamp_udf(
+ taxi_data_df["tpep_pickup_datetime"], F.lit(15)
+ )
+ ),
+ )
+ .withColumn(
+ "rounded_dropoff_datetime",
+ F.to_timestamp(
+ rounded_unix_timestamp_udf(
+ taxi_data_df["tpep_dropoff_datetime"], F.lit(30)
+ )
+ ),
+ )
+ .drop("tpep_pickup_datetime")
+ .drop("tpep_dropoff_datetime")
+ )
+ taxi_data_df.createOrReplaceTempView("taxi_data")
+ return taxi_data_df
+
+
+data = rounded_taxi_data(data)
+
+
+{{ end }}
+
+# COMMAND ----------
+
+{{ if (eq .input_include_feature_store `yes`) }}
+# Temporary fix as FS model can't predict as a pyfunc model
+# MLflow evaluate can take a lambda function instead of a model uri for a model
+# but id does not work for the baseline model as it requires a model_uri (baseline comparison is set to false)
+
+from databricks.feature_engineering import FeatureEngineeringClient
+
+def get_fs_model(df):
+ fe_client = FeatureEngineeringClient()
+ return (
+ fe_client.score_batch(model_uri=model_uri, df=spark.createDataFrame(df))
+ .select("prediction")
+ .toPandas()
+ )
+{{ end }}
+
+training_run = get_training_run(model_name, model_version)
+
+# run evaluate
+with mlflow.start_run(
+ run_name=generate_run_name(training_run),
+ description=generate_description(training_run),
+) as run, tempfile.TemporaryDirectory() as tmp_dir:
+ validation_thresholds_file = os.path.join(tmp_dir, "validation_thresholds.txt")
+ with open(validation_thresholds_file, "w") as f:
+ if validation_thresholds:
+ for metric_name in validation_thresholds:
+ f.write(
+ "{0:30} {1}\n".format(
+ metric_name, str(validation_thresholds[metric_name])
+ )
+ )
+ mlflow.log_artifact(validation_thresholds_file)
+
+ try:
+ eval_result = mlflow.evaluate(
+ {{ if (eq .input_include_feature_store `yes`) }}
+ model=get_fs_model,
+ {{ else }}
+ model=model_uri,
+ {{ end }}
+ data=data,
+ targets=targets,
+ model_type=model_type,
+ evaluators=evaluators,
+ validation_thresholds=validation_thresholds,
+ custom_metrics=custom_metrics,
+ baseline_model=None
+ if not enable_baseline_comparison
+ else baseline_model_uri,
+ evaluator_config=evaluator_config,
+ )
+ metrics_file = os.path.join(tmp_dir, "metrics.txt")
+ with open(metrics_file, "w") as f:
+ f.write(
+ "{0:30} {1:30} {2}\n".format("metric_name", "candidate", "baseline")
+ )
+ for metric in eval_result.metrics:
+ candidate_metric_value = str(eval_result.metrics[metric])
+ baseline_metric_value = "N/A"
+ if metric in eval_result.baseline_model_metrics:
+ mlflow.log_metric(
+ "baseline_" + metric, eval_result.baseline_model_metrics[metric]
+ )
+ baseline_metric_value = str(
+ eval_result.baseline_model_metrics[metric]
+ )
+ f.write(
+ "{0:30} {1:30} {2}\n".format(
+ metric, candidate_metric_value, baseline_metric_value
+ )
+ )
+ mlflow.log_artifact(metrics_file)
+ log_to_model_description(run, True)
+ # Assign "challenger" alias to indicate model version has passed validation checks
+ print("Validation checks passed. Assigning 'challenger' alias to model version.")
+ client.set_registered_model_alias(model_name, "challenger", model_version)
+ except Exception as err:
+ log_to_model_description(run, False)
+ error_file = os.path.join(tmp_dir, "error.txt")
+ with open(error_file, "w") as f:
+ f.write("Validation failed : " + str(err) + "\n")
+ f.write(traceback.format_exc())
+ mlflow.log_artifact(error_file)
+ if not dry_run:
+ raise err
+ else:
+ print(
+ "Model validation failed in DRY_RUN. It will not block model deployment."
+ )
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/README.md.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/README.md.tmpl
new file mode 100644
index 00000000000..e5a6fec8236
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/README.md.tmpl
@@ -0,0 +1,2 @@
+# Model Validation
+To enable model validation as part of scheduled databricks workflow, please refer to [{{template `project_name_alphanumeric_underscore` .}}/resources/README.md](../resources/README.md)
diff --git a/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/validation.py.tmpl b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/validation.py.tmpl
new file mode 100644
index 00000000000..25f7bc2881c
--- /dev/null
+++ b/acceptance/bundle/deploy/mlops-stacks/template/template/{{.input_root_dir}}/{{.input_project_name}}/validation/validation.py.tmpl
@@ -0,0 +1,41 @@
+import numpy as np
+from mlflow.models import make_metric, MetricThreshold
+
+# Custom metrics to be included. Return empty list if custom metrics are not needed.
+# Please refer to custom_metrics parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+# TODO(optional) : custom_metrics
+def custom_metrics():
+
+ # TODO(optional) : define custom metric function to be included in custom_metrics.
+ def squared_diff_plus_one(eval_df, _builtin_metrics):
+ """
+ This example custom metric function creates a metric based on the ``prediction`` and
+ ``target`` columns in ``eval_df`.
+ """
+ return np.sum(np.abs(eval_df["prediction"] - eval_df["target"] + 1) ** 2)
+
+ return [make_metric(eval_fn=squared_diff_plus_one, greater_is_better=False)]
+
+
+# Define model validation rules. Return empty dict if validation rules are not needed.
+# Please refer to validation_thresholds parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+# TODO(optional) : validation_thresholds
+def validation_thresholds():
+ return {
+ "max_error": MetricThreshold(
+ threshold=500, greater_is_better=False # max_error should be <= 500
+ ),
+ "mean_squared_error": MetricThreshold(
+ threshold=500, # mean_squared_error should be <= 500
+ # min_absolute_change=0.01, # mean_squared_error should be at least 0.01 greater than baseline model accuracy
+ # min_relative_change=0.01, # mean_squared_error should be at least 1 percent greater than baseline model accuracy
+ greater_is_better=False,
+ ),
+ }
+
+
+# Define evaluator config. Return empty dict if validation rules are not needed.
+# Please refer to evaluator_config parameter in mlflow.evaluate documentation https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.evaluate
+# TODO(optional) : evaluator_config
+def evaluator_config():
+ return {}
diff --git a/acceptance/bundle/deploy/mlops-stacks/test.toml b/acceptance/bundle/deploy/mlops-stacks/test.toml
index 3b87ff1bf1f..8348f7996d0 100644
--- a/acceptance/bundle/deploy/mlops-stacks/test.toml
+++ b/acceptance/bundle/deploy/mlops-stacks/test.toml
@@ -1,5 +1,5 @@
Cloud=true
-Local=false
+Local=true
# The MLOps Stacks template registers the model in Unity Catalog and runs its
# jobs on serverless compute, so this test only works on a UC-enabled workspace
@@ -16,3 +16,8 @@ Ignore = [
[[Repls]]
Old = "aws|azure|gcp"
New = "[CLOUD_ENV_BASE]"
+
+# $TESTDIR uses backslashes on Windows, hence the either-separator match.
+[[Repls]]
+Old = 'bundle init .*[\\/]mlops-stacks/template'
+New = "bundle init mlops-stacks"
diff --git a/tools/bump_mlops_stacks.py b/tools/bump_mlops_stacks.py
new file mode 100755
index 00000000000..1f5a42c8464
--- /dev/null
+++ b/tools/bump_mlops_stacks.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Refresh the vendored mlops-stacks acceptance template to upstream HEAD.
+
+The local test leg inits from the vendored copy; the cloud leg clones upstream.
+Only the files the template renders for this test's config are vendored, and that
+set is recomputed on every bump by rendering upstream, so an upstream rename is
+picked up without a hand-maintained file list.
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from string import Template
+
+REPO_URL = "https://github.com/databricks/mlops-stacks"
+
+TEST_DIR = Path("acceptance/bundle/deploy/mlops-stacks")
+TEMPLATE_DIR = TEST_DIR / "template"
+REVISION_FILE = TEST_DIR / "template.REVISION"
+
+# Upstream's project dir name is an illegal Go module path, so we vendor it renamed.
+# The project name is lowercase-alphanumeric, so the transform is a no-op and the
+# rendered output is unchanged.
+UPSTREAM_PROJECT_DIR = "{{template `project_name_alphanumeric_underscore` .}}"
+VENDORED_PROJECT_DIR = "{{.input_project_name}}"
+
+# The test's config.json.tmpl is envsubst'd by the script; substitute the values the
+# local leg gets. CLOUD_ENV is unset locally, which acceptance_test.go maps to "aws".
+RENDER_ENV = {"UNIQUE_NAME": "x", "CLOUD_ENV_BASE": "aws"}
+
+# Everything under template/ but outside the generated project is template machinery
+# (run_validations, update_layout). It renders no output of its own, so it cannot be
+# discovered from the rendered tree and is always kept.
+PROJECT_ROOT_SEG = "{{.input_root_dir}}"
+
+
+def render_path(rel_path, config):
+ """Map a path under the upstream template/ dir to its path in the rendered project."""
+ subs = {
+ PROJECT_ROOT_SEG: config["input_root_dir"],
+ VENDORED_PROJECT_DIR: config["input_project_name"],
+ UPSTREAM_PROJECT_DIR: config["input_project_name"],
+ }
+ rendered = str(rel_path)
+ for old, new in subs.items():
+ rendered = rendered.replace(old, new)
+ return rendered.removesuffix(".tmpl")
+
+
+def render(cli, clone_dir, config_file, out_dir):
+ empty_cfg = out_dir.parent / "empty.databrickscfg"
+ empty_cfg.touch()
+ # bundle init authenticates before rendering, so drop the caller's Databricks
+ # environment and point it at a non-resolving host with an empty config file.
+ env = {k: v for k, v in os.environ.items() if not k.startswith("DATABRICKS_")}
+ env |= {
+ "DATABRICKS_HOST": "https://bump-mlops-stacks.test",
+ "DATABRICKS_TOKEN": "dummy",
+ "DATABRICKS_CONFIG_FILE": str(empty_cfg),
+ }
+ subprocess.run(
+ [str(cli), "bundle", "init", str(clone_dir), "--config-file", str(config_file)],
+ check=True,
+ cwd=out_dir,
+ stdout=subprocess.DEVNULL,
+ env=env,
+ )
+
+
+def keep_set(clone_dir, out_dir, config):
+ """List the upstream files needed to render this test's project, relative to the repo root."""
+ keep = [Path("databricks_template_schema.json")]
+ keep += sorted(p.relative_to(clone_dir) for p in (clone_dir / "library").rglob("*") if p.is_file())
+ for path in sorted((clone_dir / "template").rglob("*")):
+ if not path.is_file():
+ continue
+ rel = path.relative_to(clone_dir)
+ in_project = rel.parts[1:2] == (PROJECT_ROOT_SEG,)
+ if not in_project or (out_dir / render_path(rel.relative_to("template"), config)).exists():
+ keep.append(rel)
+ return keep
+
+
+def vendored_path(rel_path):
+ return Path(*[VENDORED_PROJECT_DIR if p == UPSTREAM_PROJECT_DIR else p for p in rel_path.parts])
+
+
+def main():
+ root = Path(__file__).resolve().parent.parent
+ config = Template((root / TEST_DIR / "config.json.tmpl").read_text()).substitute(RENDER_ENV)
+
+ with tempfile.TemporaryDirectory(prefix="mlops-stacks-") as tmp:
+ tmp = Path(tmp)
+ clone_dir, out_dir, cli = tmp / "upstream", tmp / "rendered", tmp / "cli"
+ out_dir.mkdir()
+ subprocess.run(["go", "build", "-o", str(cli), "."], check=True, cwd=root)
+ config_file = tmp / "config.json"
+ config_file.write_text(config)
+
+ subprocess.run(["git", "clone", "--depth", "1", "--quiet", REPO_URL, str(clone_dir)], check=True)
+ sha = subprocess.run(
+ ["git", "-C", str(clone_dir), "rev-parse", "HEAD"],
+ check=True,
+ capture_output=True,
+ text=True,
+ ).stdout.strip()
+
+ render(cli, clone_dir, config_file, out_dir)
+ keep = keep_set(clone_dir, out_dir, json.loads(config))
+
+ template_dir = root / TEMPLATE_DIR
+ shutil.rmtree(template_dir)
+ for rel in keep:
+ dst = template_dir / vendored_path(rel)
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(clone_dir / rel, dst)
+
+ (root / REVISION_FILE).write_text(sha + "\n")
+
+ print(f"Vendored {len(keep)} files from {REPO_URL} @ {sha}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/validate_whitespace.py b/tools/validate_whitespace.py
index b47bd3dcf54..152e52c5e6c 100755
--- a/tools/validate_whitespace.py
+++ b/tools/validate_whitespace.py
@@ -18,7 +18,9 @@ def load_ignores():
continue
if line.startswith("#"):
continue
- expanded = glob.glob(line, recursive=True)
+ # include_hidden: without it `**` skips dot-prefixed names, so a directory
+ # pattern silently fails to cover e.g. a vendored tree's .github/.
+ expanded = glob.glob(line, recursive=True, include_hidden=True)
if len(expanded) == 0:
print(f".wsignore:{ind + 1}: No matches for line: {line}")
fail = True