acc: run bundle/deploy/mlops-stacks locally

.wsignore

@@ -25,3 +25,6 @@ python/databricks/bundles/*/_models/*.py
 
 # Developed elsewhere:
 internal/genkit/tagging.py
+
+# Vendored from https://github.com/databricks/mlops-stacks:
+acceptance/bundle/deploy/mlops-stacks/template/**

Taskfile.yml

@@ -598,6 +598,15 @@ tasks:
       - task: test-update
       - task: test-update-aws
 
+  # Follows upstream HEAD, so its result changes over time: keep it out of
+  # `generate-check`, which requires byte-for-byte reproducible output.
+  bump-mlops-stacks:
+    desc: Bump the vendored mlops-stacks acceptance template to github.com/databricks/mlops-stacks HEAD
+    cmds:
+      - "./tools/bump_mlops_stacks.py"
+      # The bump rewrites a test input, so refresh that test's golden output.
+      - "go test ./acceptance -run '^TestAccept/bundle/deploy/mlops-stacks' -update -timeout=${LOCAL_TIMEOUT:-60m}"
+
   slowest:
     desc: Show 50 slowest tests from last run
     cmds:

acceptance/bundle/deploy/mlops-stacks/script

@@ -29,7 +29,13 @@ trap cleanup EXIT
 
 trace $CLI schemas create "${SCHEMA_NAME}" main | jq '{full_name}'
 
-trace $CLI bundle init mlops-stacks --config-file config.json
+# Local runs must not reach the network; cloud runs clone to keep that path covered.
+# The vendored copy in ./template/ can be updated with `task bump-mlops-stacks`
+if [ -z "$CLOUD_ENV" ]; then
+  trace $CLI bundle init "$TESTDIR/template" --config-file config.json
+else
+  trace $CLI bundle init mlops-stacks --config-file config.json
+fi
 trace cat test_repo_mlops_stacks/README.md | head -n 4
 
 cd "test_repo_mlops_stacks/project_name_${UNIQUE_NAME}" || exit 1

acceptance/bundle/deploy/mlops-stacks/template.REVISION

@@ -0,0 +1 @@
+9b387898ad10439c90581e6064f19407121dc410

acceptance/bundle/deploy/mlops-stacks/template/databricks_template_schema.json

@@ -0,0 +1,266 @@
+{
+  "welcome_message": "Welcome to MLOps Stacks. For detailed information on project generation, see the README at https://github.com/databricks/mlops-stacks/blob/main/README.md.",
+  "min_databricks_cli_version": "v0.236.0",
+  "properties": {
+    "input_setup_cicd_and_project": {
+      "order": 1,
+      "type": "string",
+      "description": "{{if false}}\n\nERROR: This template is not supported by your current Databricks CLI version.\nPlease hit control-C and go to https://docs.databricks.com/en/dev-tools/cli/install.html for instructions on upgrading the CLI to the minimum version supported by MLOps Stacks.\n\n\n{{end}}\nSelect if both CI/CD and the Project should be set up, or only one of them.\nYou can always set up the other later by running initialization again",
+      "default": "CICD_and_Project",
+      "enum": ["CICD_and_Project", "Project_Only", "CICD_Only"]
+    },
+    "input_project_name": {
+      "order": 2,
+      "type": "string",
+      "default": "my_mlops_project",
+      "description": "\nProject Name. Default",
+      "pattern": "^[^ .\\\\/]{3,}$",
+      "pattern_match_failure_message": "Project name must be at least 3 characters long and cannot contain the following characters: \"\\\", \"/\", \" \" and \".\".",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_root_dir": {
+      "order": 3,
+      "type": "string",
+      "default": "{{ .input_project_name }}",
+      "description": "\nRoot directory name.\nFor monorepos, name of the root directory that contains all the projects.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_and_Project"
+          }
+        }
+      }
+    },
+    "input_cloud": {
+      "order": 4,
+      "type": "string",
+      "description": "\nSelect cloud",
+      "default": "azure",
+      "enum": ["azure", "aws", "gcp"]
+    },
+    "input_cicd_platform": {
+      "order": 5,
+      "type": "string",
+      "description": "\nSelect CICD platform",
+      "default": "github_actions",
+      "enum": ["github_actions", "github_actions_for_github_enterprise_servers", "azure_devops", "gitlab"],
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_databricks_staging_workspace_host": {
+      "order": 6,
+      "type": "string",
+      "default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-staging-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-staging-workspace.gcp.databricks.com{{end}}",
+      "description": "\nURL of staging Databricks workspace,\nIt will run PR CI and preview changes before they're deployed to production.\nDefault",
+      "pattern": "^(https.*)?$",
+      "pattern_match_failure_message": "Databricks staging workspace host URLs must start with https. Got invalid workspace host.",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_databricks_prod_workspace_host": {
+      "order": 7,
+      "type": "string",
+      "default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-prod-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-prod-workspace.gcp.databricks.com{{end}}",
+      "description": "\nURL of production Databricks workspace.\nDefault",
+      "pattern": "^(https.*)?$",
+      "pattern_match_failure_message": "Databricks production workspace host URLs must start with https. Got invalid workspace host.",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_default_branch": {
+      "order": 8,
+      "type": "string",
+      "default": "main",
+      "description": "\nName of the default branch,\nStaging resources are deployed from this branch and stages the latest ML code.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_release_branch": {
+      "order": 9,
+      "type": "string",
+      "default": "release",
+      "description": "\nName of the release branch.\nThe training and other production jobs pull ML code from this branch.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_read_user_group": {
+      "order": 10,
+      "type": "string",
+      "default": "users",
+      "description": "\nUser group name to give READ permissions to for project resources\n(ML jobs, integration test job runs, and machine learning resources).\nA group with this name must exist in both the staging and prod workspaces.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_staging_catalog_name": {
+      "order": 12,
+      "type": "string",
+      "description": "\nName of the catalog in Unity Catalog that will host the staging UC resources. \nThis catalog must already exist and service principals must have access to it.\nDefault",
+      "default": "staging",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_prod_catalog_name": {
+      "order": 13,
+      "type": "string",
+      "description": "\nName of the catalog in Unity Catalog that will host the production UC resources.\nThis catalog must already exist and service principals must have access to it.\nDefault",
+      "default": "prod",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_test_catalog_name": {
+      "order": 14,
+      "type": "string",
+      "description": "\nName of the catalog in Unity Catalog that will be used for integration tests.\nThis catalog must already exist and service principals must have access to it.\nDefault",
+      "default": "test",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "Project_Only"
+          }
+        }
+      }
+    },
+    "input_schema_name": {
+      "order": 15,
+      "type": "string",
+      "description": "\nName of schema to use when registering a model in Unity Catalog.\nThis schema must already exist and service principals must have access.\nWe recommend using the project name.\nDefault",
+      "default": "{{ .input_project_name }}",
+      "pattern": "^[^ .\\-\\/]*$",
+      "pattern_match_failure_message": "Valid schema names cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_unity_catalog_read_user_group": {
+      "order": 16,
+      "type": "string",
+      "default": "account users",
+      "description": "\nUser group name to give EXECUTE privileges to models in Unity Catalog (UC).\nIt must exist in UC with access granted to the staging and prod workspaces.\nDefault",
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_inference_table_name": {
+      "order": 17,
+      "type": "string",
+      "description": "\nFully qualified name of inference table to attach monitoring to.\nThis table must already exist and service principals must have access.",
+      "default": "dev.{{ .input_project_name }}.predictions",
+      "pattern": "^[^ .\\-\\/]+(\\.[^ .\\-\\/]+){2}$",
+      "pattern_match_failure_message": "Fully qualified Unity Catalog table names must have catalog, schema, and table separated by \".\" and each cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
+      "skip_prompt_if": {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "CICD_Only"
+              }
+            }
+      }
+    },
+    "input_include_feature_store": {
+      "order": 18,
+      "type": "string",
+      "description": "\nWhether to include Feature Store",
+      "default": "no",
+      "enum": ["no", "yes"],
+      "skip_prompt_if": {
+        "properties": {
+          "input_setup_cicd_and_project": {
+            "const": "CICD_Only"
+          }
+        }
+      }
+    },
+    "input_docker_image": {
+      "order": 20,
+      "type": "string",
+      "description": "\nDocker image for the execution of Gitlab pipelines",
+      "default": "databricksfieldeng/mlopsstacks:latest",
+      "skip_prompt_if": {
+        "anyOf":[
+          {
+            "properties": {
+              "input_cicd_platform": {
+                "const": "github_actions"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_cicd_platform": {
+                "const": "github_actions_for_github_enterprise_servers"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_cicd_platform": {
+                "const": "azure_devops"
+              }
+            }
+          },
+          {
+            "properties": {
+              "input_setup_cicd_and_project": {
+                "const": "Project_Only"
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "success_message" : "\n*** Your MLOps Stack has been created in the '{{.input_root_dir}}{{if not (eq .input_setup_cicd_and_project `CICD_Only`) }}/{{.input_project_name}}{{end}}' directory! ***\n\nPlease refer to the README.md for further instructions on getting started."
+}

acceptance/bundle/deploy/mlops-stacks/template/library/functions.tmpl

@@ -0,0 +1,26 @@
+# define functions
+{{ define "get_host" -}}
+  {{- with url . -}}
+    {{- print .Scheme `://` .Host -}}
+  {{- end -}}
+{{- end }}
+
+{{ define "generate_doc_link" -}}
+    {{- if eq .cloud `aws` -}}
+        {{- if eq .path `repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later` -}}
+            https://docs.databricks.com/repos/git-operations-with-repos.html#add-a-repo-connected-to-a-remote-repo
+        {{- else -}}
+            https://docs.databricks.com/{{ print .path }}
+        {{- end -}}
+    {{- else if eq .cloud `gcp` -}}
+        {{- if eq .path `repos/git-operations-with-repos#add-a-repo-and-connect-remotely-later` -}}
+            https://docs.gcp.databricks.com/en/repos/git-operations-with-repos.html#clone-a-repo-connected-to-a-remote-repo
+        {{- else -}}
+            https://docs.gcp.databricks.com/{{ print .path }}
+        {{- end -}}
+    {{- else if eq .cloud `azure` -}}
+        https://learn.microsoft.com/azure/databricks/{{ (regexp `\.html`).ReplaceAllString (print .path) `` }}
+    {{- else -}}
+        {{ fail `Invalid selection of cloud in function generate_doc_link. Please choose from [azure, aws, gcp]` }}
+    {{- end -}}
+{{- end }}

acceptance/bundle/deploy/mlops-stacks/template/library/input_validation.tmpl

@@ -0,0 +1,2 @@
+{{ define `validation` }}
+{{- end -}}