Merge pull request #19 from aws-solutions/develop

Update to version v2.0.0

Author: eggoynes

CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.0] - 2022-05-31
+
+### Added
+
+- A new pipeline to train Machine Learning (ML) models using [Amazon SageMaker built-in algorithms](https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html) and [Amazon SageMaker Training Job](https://docs.aws.amazon.com/sagemaker/latest/dg/how-it-works-training.html).
+- A new pipeline to train ML models using Amazon SageMaker built-in algorithms and [Amazon Hyperparameter Tuning Job](https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-how-it-works.html).
+- A new pipeline to train ML models using Amazon SageMaker built-in algorithms and
+  [Amazon SageMaker Autopilot Job](https://docs.aws.amazon.com/sagemaker/latest/dg/autopilot-automate-model-development.html).
+- [Amazon EventBridge Rules](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-rules.html)
+  to notify the solution's administrator(s) about the status of the training jobs.
+
+### Updated
+
+- The [Amazon Simple Notification Service (SNS)](https://aws.amazon.com/sns/?whats-new-cards.sort-by=item.additionalFields.postDateTime&whats-new-cards.sort-order=desc)
+  Topic, used for pipelines notifications, was moved to the solution's main template.
+
 ## [1.5.0] - 2022-01-24
 
 ### Added

README.md

@@ -9,7 +9,7 @@ across projects and organizations, the processes contain commonalities across us
 
 The solution helps you streamline and enforce architecture best practices by providing an extendable
 framework for managing ML pipelines for Amazon Machine Learning (Amazon ML) services and third-party
-services. The solution’s template allows you to upload trained models, configure the orchestration of
+services. The solution’s template allows you to train models, upload trained models, configure the orchestration of
 the pipeline, initiate the start of the deployment process, move models through different stages of
 deployment, and monitor the successes and failures of the operations. The solution also provides a
 pipeline for building and registering Docker images for custom algorithms that can be used for model
@@ -23,7 +23,7 @@ increases your team’s agility and efficiency by allowing them to repeat succes
 #### Benefits
 
 - **Leverage a pre-configured machine learning pipeline:** Use the solution's reference architecture to initiate a pre-configured pipeline through an API call or a Git repository.
-- **Automatically deploy a trained model and inference endpoint:** Use the solution's framework to automate the model monitor pipeline or the Amazon SageMaker BYOM pipeline. Deliver an inference endpoint with model drift detection packaged as a serverless microservice.
+- **Automatically train, deploy, and monitor models:** Use the solution's pipelines to automate the model training. Deliver an inference endpoint with model drift detection packaged as a serverless microservice.
 
 ---
 

deployment/build-s3-dist.sh

@@ -129,6 +129,12 @@ echo "cdk synth BYOMCustomAlgorithmImageBuilderStack > lib/blueprints/byom/byom_
 cdk synth BYOMCustomAlgorithmImageBuilderStack > lib/blueprints/byom/byom_custom_algorithm_image_builder.yaml --path-metadata false --version-reporting false
 echo "cdk synth BYOMBatchStack > lib/blueprints/byom/byom_batch_pipeline.yaml --path-metadata false --version-reporting false"
 cdk synth BYOMBatchStack > lib/blueprints/byom/byom_batch_pipeline.yaml --path-metadata false --version-reporting false
+echo "cdk synth AutopilotJobStack > lib/blueprints/byom/autopilot_training_pipeline.yaml --path-metadata false --version-reporting false"
+cdk synth AutopilotJobStack > lib/blueprints/byom/autopilot_training_pipeline.yaml --path-metadata false --version-reporting false
+echo "cdk synth TrainingJobStack > lib/blueprints/byom/model_training_pipeline.yaml --path-metadata false --version-reporting false"
+cdk synth TrainingJobStack > lib/blueprints/byom/model_training_pipeline.yaml --path-metadata false --version-reporting false
+echo "cdk synth HyperparamaterTunningJobStack > lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml --path-metadata false --version-reporting false"
+cdk synth HyperparamaterTunningJobStack > lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml --path-metadata false --version-reporting false
 
 # Replace %%VERSION%% in other templates
 replace="s/%%VERSION%%/$3/g"
@@ -150,6 +156,12 @@ echo "sed -i -e $replace lib/blueprints/byom/byom_custom_algorithm_image_builder
 sed -i -e $replace lib/blueprints/byom/byom_custom_algorithm_image_builder.yaml
 echo "sed -i -e $replace lib/blueprints/byom/byom_batch_pipeline.yaml"
 sed -i -e $replace lib/blueprints/byom/byom_batch_pipeline.yaml
+echo "sed -i -e $replace lib/blueprints/byom/autopilot_training_pipeline.yaml"
+sed -i -e $replace lib/blueprints/byom/autopilot_training_pipeline.yaml
+echo "sed -i -e $replace lib/blueprints/byom/model_training_pipeline.yaml"
+sed -i -e $replace lib/blueprints/byom/model_training_pipeline.yaml
+echo "sed -i -e $replace lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml"
+sed -i -e $replace lib/blueprints/byom/model_hyperparameter_tunning_pipeline.yaml
 
 # Run 'cdk synth' for main templates to generate raw solution outputs
 echo "cdk synth mlops-workload-orchestrator-single-account --path-metadata false --version-reporting false --output=$staging_dist_dir"

source/app.py

@@ -19,6 +19,8 @@
 from lib.blueprints.byom.single_account_codepipeline import SingleAccountCodePipelineStack
 from lib.blueprints.byom.multi_account_codepipeline import MultiAccountCodePipelineStack
 from lib.blueprints.byom.byom_custom_algorithm_image_builder import BYOMCustomAlgorithmImageBuilderStack
+from lib.blueprints.byom.autopilot_training_pipeline import AutopilotJobStack
+from lib.blueprints.byom.model_training_pipeline import TrainingJobStack
 from lib.aws_sdk_config_aspect import AwsSDKConfigAspect
 from lib.blueprints.byom.pipeline_definitions.cdk_context_value import get_cdk_context_value
 
@@ -56,7 +58,9 @@
 batch_stack = BYOMBatchStack(
     app,
     "BYOMBatchStack",
-    description=(f"({solution_id}byom-bt) - BYOM Batch Transform pipeline in AWS MLOps Framework. Version {version}"),
+    description=(
+        f"({solution_id}byom-bt) - BYOM Batch Transform pipeline in MLOps Workload Orchestrator. Version {version}"
+    ),
 )
 
 core.Aspects.of(batch_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentBatch", solution_id, version))
@@ -113,6 +117,38 @@
 
 core.Aspects.of(realtime_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentRealtime", solution_id, version))
 
+
+autopilot_stack = AutopilotJobStack(
+    app,
+    "AutopilotJobStack",
+    description=(f"({solution_id}-autopilot) - Autopilot training pipeline. Version {version}"),
+)
+
+core.Aspects.of(autopilot_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentAutopilot", solution_id, version))
+
+
+training_stack = TrainingJobStack(
+    app,
+    "TrainingJobStack",
+    training_type="TrainingJob",
+    description=(f"({solution_id}-training) - Model Training pipeline. Version {version}"),
+)
+
+core.Aspects.of(training_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentTraining", solution_id, version))
+
+
+hyperparameter_tunning_stack = TrainingJobStack(
+    app,
+    "HyperparamaterTunningJobStack",
+    training_type="HyperparameterTuningJob",
+    description=(f"({solution_id}-tuner) - Model Hyperparameter Tunning pipeline. Version {version}"),
+)
+
+core.Aspects.of(hyperparameter_tunning_stack).add(
+    AwsSDKConfigAspect(app, "SDKUserAgentHyperparamater", solution_id, version)
+)
+
+
 SingleAccountCodePipelineStack(
     app,
     "SingleAccountCodePipelineStack",

source/lambdas/pipeline_orchestration/index.py

@@ -106,14 +106,16 @@ def provision_pipeline(
         pipeline_template_url = (
             template_url("multi_account_codepipeline")
             if is_multi_account == "True"
+            # training pipelines are deployed in the account where the main template is deployed
+            and pipeline_type not in ["model_training_builtin", "model_tuner_builtin", "model_autopilot_training"]
             else template_url("single_account_codepipeline")
         )
 
         template_zip_name = f"mlops_provisioned_pipelines/{provisioned_pipeline_stack_name}/template.zip"
         template_file_name = provisioned_pipeline_template_url.split("/")[-1]
         # get the codepipeline parameters
         codepipeline_params = get_codepipeline_params(
-            is_multi_account, provisioned_pipeline_stack_name, template_zip_name, template_file_name
+            is_multi_account, pipeline_type, provisioned_pipeline_stack_name, template_zip_name, template_file_name
         )
         # format the params (the format is the same for multi-account parameters)
         formatted_codepipeline_params = format_template_parameters(codepipeline_params, "True")