Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ https://github.com/databricks/mlops-stacks/assets/87999496/0d220d55-465e-4a69-bd

### Prerequisites
- Python 3.8+
- [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/databricks-cli.html) >= v0.236.0
- [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/databricks-cli.html) >= v0.278.0

[Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/databricks-cli.html) contains [Databricks asset bundle templates](https://docs.databricks.com/en/dev-tools/bundles/templates.html) for the purpose of project creation.

Expand Down
116 changes: 86 additions & 30 deletions databricks_template_schema.json
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
{
"welcome_message": "Welcome to MLOps Stacks. For detailed information on project generation, see the README at https://github.com/databricks/mlops-stacks/blob/main/README.md.",
"min_databricks_cli_version": "v0.236.0",
"welcome_message":"Welcome to MLOps Stacks. For detailed information on project generation, see the README at https://github.com/databricks/mlops-stacks/blob/main/README.md.",
"min_databricks_cli_version": "v0.266.0",
"properties": {
"input_setup_cicd_and_project": {
"input_project_type": {
"order": 1,
"type": "string",
"description": "{{if false}}\n\nERROR: This template is not supported by your current Databricks CLI version.\nPlease hit control-C and go to https://docs.databricks.com/en/dev-tools/cli/install.html for instructions on upgrading the CLI to the minimum version supported by MLOps Stacks.\n\n\n{{end}}\nSelect if both CI/CD and the Project should be set up, or only one of them.\nYou can always set up the other later by running initialization again",
"description": "{{if false}}\n\nERROR: This template is not supported by your current Databricks CLI version.\nPlease hit control-C and go to https://docs.databricks.com/en/dev-tools/cli/install.html for instructions on upgrading the CLI to the minimum version supported by MLOps Stacks.\n\n\n{{end}}\nSelect project type",
"default": "mlops",
"enum": ["mlops", "agentops"]
},
"input_setup_cicd_and_project": {
"order": 2,
"type": "string",
"description": "Select if both CI/CD and the Project should be set up, or only one of them.\nYou can always set up the other later by running initialization again",
"default": "CICD_and_Project",
"enum": ["CICD_and_Project", "Project_Only", "CICD_Only"]
},
"input_project_name": {
"order": 2,
"order": 3,
"type": "string",
"default": "my_mlops_project",
"default": "my_{{ .input_project_type }}_project",
"description": "\nProject Name. Default",
"pattern": "^[^ .\\\\/]{3,}$",
"pattern_match_failure_message": "Project name must be at least 3 characters long and cannot contain the following characters: \"\\\", \"/\", \" \" and \".\".",
Expand All @@ -24,8 +31,21 @@
}
}
},
"input_doc_link": {
"order": 4,
"type": "string",
"description": "URL of the documentation link",
"default": "https://docs.databricks.com/en/doc-sitemap.xml",
"skip_prompt_if": {
"properties": {
"input_project_type": {
"enum": ["mlops", "agentops"]
}
}
}
},
"input_root_dir": {
"order": 3,
"order": 5,
"type": "string",
"default": "{{ .input_project_name }}",
"description": "\nRoot directory name.\nFor monorepos, name of the root directory that contains all the projects.\nDefault",
Expand All @@ -38,14 +58,14 @@
}
},
"input_cloud": {
"order": 4,
"order": 6,
"type": "string",
"description": "\nSelect cloud",
"default": "azure",
"enum": ["azure", "aws", "gcp"]
},
"input_cicd_platform": {
"order": 5,
"order": 7,
"type": "string",
"description": "\nSelect CICD platform",
"default": "github_actions",
Expand All @@ -59,7 +79,7 @@
}
},
"input_databricks_staging_workspace_host": {
"order": 6,
"order": 8,
"type": "string",
"default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-staging-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-staging-workspace.gcp.databricks.com{{end}}",
"description": "\nURL of staging Databricks workspace,\nIt will run PR CI and preview changes before they're deployed to production.\nDefault",
Expand All @@ -74,7 +94,7 @@
}
},
"input_databricks_prod_workspace_host": {
"order": 7,
"order": 9,
"type": "string",
"default": "{{if eq .input_cloud `azure`}}https://adb-xxxx.xx.azuredatabricks.net{{else if eq .input_cloud `aws`}}https://your-prod-workspace.cloud.databricks.com{{else if eq .input_cloud `gcp`}}https://your-prod-workspace.gcp.databricks.com{{end}}",
"description": "\nURL of production Databricks workspace.\nDefault",
Expand All @@ -89,7 +109,7 @@
}
},
"input_default_branch": {
"order": 8,
"order": 10,
"type": "string",
"default": "main",
"description": "\nName of the default branch,\nStaging resources are deployed from this branch and stages the latest ML code.\nDefault",
Expand All @@ -102,7 +122,7 @@
}
},
"input_release_branch": {
"order": 9,
"order": 11,
"type": "string",
"default": "release",
"description": "\nName of the release branch.\nThe training and other production jobs pull ML code from this branch.\nDefault",
Expand All @@ -115,7 +135,7 @@
}
},
"input_read_user_group": {
"order": 10,
"order": 12,
"type": "string",
"default": "users",
"description": "\nUser group name to give READ permissions to for project resources\n(ML jobs, integration test job runs, and machine learning resources).\nA group with this name must exist in both the staging and prod workspaces.\nDefault",
Expand All @@ -128,14 +148,21 @@
}
},
"input_include_models_in_unity_catalog": {
"order": 11,
"order": 13,
"type": "string",
"description": "\nWhether to use the Model Registry with Unity Catalog",
"default": "no",
"enum": ["yes", "no"]
"enum": ["yes", "no"],
"skip_prompt_if": {
"properties": {
"input_project_type": {
"const": "agentops"
}
}
}
},
"input_staging_catalog_name": {
"order": 12,
"order": 14,
"type": "string",
"description": "\nName of the catalog in Unity Catalog that will host the staging UC resources. \nThis catalog must already exist and service principals must have access to it.\nDefault",
"default": "staging",
Expand All @@ -159,7 +186,7 @@
}
},
"input_prod_catalog_name": {
"order": 13,
"order": 15,
"type": "string",
"description": "\nName of the catalog in Unity Catalog that will host the production UC resources.\nThis catalog must already exist and service principals must have access to it.\nDefault",
"default": "prod",
Expand All @@ -183,7 +210,7 @@
}
},
"input_test_catalog_name": {
"order": 14,
"order": 16,
"type": "string",
"description": "\nName of the catalog in Unity Catalog that will be used for integration tests.\nThis catalog must already exist and service principals must have access to it.\nDefault",
"default": "test",
Expand All @@ -207,7 +234,7 @@
}
},
"input_schema_name": {
"order": 15,
"order": 17,
"type": "string",
"description": "\nName of schema to use when registering a model in Unity Catalog.\nThis schema must already exist and service principals must have access.\nWe recommend using the project name.\nDefault",
"default": "{{if (eq .input_include_models_in_unity_catalog `no`)}}schema{{else}}{{ .input_project_name }}{{end}}",
Expand All @@ -233,7 +260,7 @@
}
},
"input_unity_catalog_read_user_group": {
"order": 16,
"order": 18,
"type": "string",
"default": "account users",
"description": "\nUser group name to give EXECUTE privileges to models in Unity Catalog (UC).\nIt must exist in UC with access granted to the staging and prod workspaces.\nDefault",
Expand All @@ -257,36 +284,58 @@
}
},
"input_inference_table_name": {
"order": 17,
"order": 19,
"type": "string",
"description": "\nFully qualified name of inference table to attach monitoring to.\nThis table must already exist and service principals must have access.",
"default": "dev.{{ .input_project_name }}.predictions",
"pattern": "^[^ .\\-\\/]+(\\.[^ .\\-\\/]+){2}$",
"pattern_match_failure_message": "Fully qualified Unity Catalog table names must have catalog, schema, and table separated by \".\" and each cannot contain any of the following characters: \" \", \".\", \"-\", \"\\\", \"/\"",
"skip_prompt_if": {
"anyOf":[
{
"properties": {
"input_setup_cicd_and_project": {
"const": "CICD_Only"
}
}
},
{
"properties": {
"input_project_type": {
"const": "agentops"
}
}
}
]
}
},
"input_include_feature_store": {
"order": 18,
"order": 20,
"type": "string",
"description": "\nWhether to include Feature Store",
"default": "no",
"enum": ["no", "yes"],
"skip_prompt_if": {
"properties": {
"input_setup_cicd_and_project": {
"const": "CICD_Only"
"anyOf":[
{
"properties": {
"input_setup_cicd_and_project": {
"const": "CICD_Only"
}
}
},
{
"properties": {
"input_project_type": {
"const": "agentops"
}
}
}
}
]
}
},
"input_include_mlflow_recipes": {
"order": 19,
"order": 21,
"type": "string",
"description": "\nWhether to include MLflow Recipes",
"default": "no",
Expand All @@ -313,12 +362,19 @@
"const": "CICD_Only"
}
}
},
{
"properties": {
"input_project_type": {
"const": "agentops"
}
}
}
]
}
},
"input_docker_image": {
"order": 20,
"order": 22,
"type": "string",
"description": "\nDocker image for the execution of Gitlab pipelines",
"default": "databricksfieldeng/mlopsstacks:latest",
Expand Down Expand Up @@ -357,4 +413,4 @@
}
},
"success_message" : "\n*** Your MLOps Stack has been created in the '{{.input_root_dir}}{{if not (eq .input_setup_cicd_and_project `CICD_Only`) }}/{{.input_project_name}}{{end}}' directory! ***\n\nPlease refer to the README.md for further instructions on getting started."
}
}
2 changes: 1 addition & 1 deletion library/template_variables.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
{{- end }}

{{ define `cli_version` -}}
v0.236.0
v0.278.0
{{- end }}

{{ define `stacks_version` -}}
Expand Down
24 changes: 23 additions & 1 deletion template/update_layout.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,28 @@
{{ $project_name_alphanumeric_underscore := (regexp `-`).ReplaceAllString ((regexp `[^A-Za-z0-9_-]`).ReplaceAllString .input_project_name ``) `_` -}}
{{ $root_dir := .input_root_dir}}

{{ if (eq .input_project_type `mlops`) }}
# Skip agentops-specific directories and resources
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `data_preparation`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `tests/integration`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `tests/integration/model_serving_test.py`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/data-preparation-resource.yml`) }}
{{ else if (eq .input_project_type `agentops`) }}
# Skip mlops-specific components
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `training`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `tests/training`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `feature_engineering`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `tests/feature_engineering`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `validation`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `monitoring`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `deployment`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/model-workflow-resource.yml`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/batch-inference-workflow-resource.yml`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/feature-engineering-workflow-resource.yml`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/monitoring-resource.yml`) }}
{{ skip (printf `%s/%s/%s` $root_dir $project_name_alphanumeric_underscore `resources/ml-artifacts-resource.yml`) }}
{{ end }}

{{ if (eq .input_setup_cicd_and_project `Project_Only`) }}
{{ skip (printf `%s/%s` $root_dir `.azure`) }}
{{ skip (printf `%s/%s` $root_dir `.github`) }}
Expand Down Expand Up @@ -88,4 +110,4 @@
# Remove template files
{{ skip (printf `%s/%s` $root_dir `cicd`) }}
{{ skip `update_layout` }}
{{ skip `run_validations` }}
{{ skip `run_validations` }}
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ jobs:
{{- end }}
{{ end }}

{{ if (eq .input_project_type `mlops`) }}
# Run model_training_job defined in bundle in the staging workspace
- script: |
databricks bundle run model_training_job -t test
Expand All @@ -137,3 +138,31 @@ jobs:
{{ else -}}
DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
{{- end }}
{{ else -}}
# Run data_preprocessing_job defined in bundle in the staging workspace
- script: |
databricks bundle run data_preprocessing_job -t test
workingDirectory: $(workingDirectory)
displayName: Run data preprocessing workflow for test deployment target in staging workspace
env:
{{ if (eq .input_cloud `azure`) -}}
ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
{{ else -}}
DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
{{- end }}
# Run agent_development_job defined in bundle in the staging workspace
- script: |
databricks bundle run agent_development_job -t test
workingDirectory: $(workingDirectory)
displayName: Run agent development workflow for test deployment target in staging workspace
env:
{{ if (eq .input_cloud `azure`) -}}
ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
{{ else -}}
DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
{{- end }}
{{ end }}
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,18 @@ jobs:
run: |
databricks bundle run write_feature_table_job -t test
{{- end }}
{{- if (eq .input_project_type `mlops`) }}
- name: Run Training Workflow for Test Deployment Target in Staging Workspace
id: training
run: |
databricks bundle run model_training_job -t test
{{ else -}}
- name: Run Data Preprocessing Workflow for Test Deployment Target in Staging Workspace
id: data_preprocessing
run: |
databricks bundle run data_preprocessing_job -t test
- name: Run Agent Development Workflow for Test Deployment Target in Staging Workspace
id: agent_development
run: |
databricks bundle run agent_development_job -t test
{{- end }}
Loading