diff --git a/CHANGELOG.md b/CHANGELOG.md index f4dd309..7ca0fa1 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.1] - 2022-08-12 + +### Updated + +- The AWS IAM Role permissions with the new naming convention for the temporary Amazon SageMaker endpoints used by the Amazon SageMaker Clarify Model Bias Monitor and Amazon SageMaker Clarify Model Explainability Monitor pipelines. + +### Fixed + +- Environment variables of lambda functions by adding `PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python` to handle `protobuf` library breaking changes in versions greater than `3.20.1`. + +- Empty string image url for the model training pipelines when using Amazon SageMaker Model Registry option. + ## [2.0.0] - 2022-05-31 ### Added diff --git a/source/app.py b/source/app.py index 5678c38..776ed74 100644 --- a/source/app.py +++ b/source/app.py @@ -22,6 +22,7 @@ from lib.blueprints.byom.autopilot_training_pipeline import AutopilotJobStack from lib.blueprints.byom.model_training_pipeline import TrainingJobStack from lib.aws_sdk_config_aspect import AwsSDKConfigAspect +from lib.protobuf_config_aspect import ProtobufConfigAspect from lib.blueprints.byom.pipeline_definitions.cdk_context_value import get_cdk_context_value app = core.App() @@ -37,6 +38,10 @@ # add AWS_SDK_USER_AGENT env variable to Lambda functions core.Aspects.of(mlops_stack_single).add(AwsSDKConfigAspect(app, "SDKUserAgentSingle", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(mlops_stack_single).add(ProtobufConfigAspect(app, "ProtobufConfigSingle")) + + mlops_stack_multi = MLOpsStack( app, "mlops-workload-orchestrator-multi-account", @@ -46,6 +51,9 @@ core.Aspects.of(mlops_stack_multi).add(AwsSDKConfigAspect(app, "SDKUserAgentMulti", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(mlops_stack_multi).add(ProtobufConfigAspect(app, "ProtobufConfigMulti")) + BYOMCustomAlgorithmImageBuilderStack( app, "BYOMCustomAlgorithmImageBuilderStack", @@ -65,6 +73,9 @@ core.Aspects.of(batch_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentBatch", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(batch_stack).add(ProtobufConfigAspect(app, "ProtobufConfigBatch")) + data_quality_monitor_stack = ModelMonitorStack( app, "DataQualityModelMonitorStack", @@ -76,6 +87,10 @@ AwsSDKConfigAspect(app, "SDKUserAgentDataMonitor", solution_id, version) ) + +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(data_quality_monitor_stack).add(ProtobufConfigAspect(app, "ProtobufConfigDataMonitor")) + model_quality_monitor_stack = ModelMonitorStack( app, "ModelQualityModelMonitorStack", @@ -87,6 +102,9 @@ AwsSDKConfigAspect(app, "SDKUserAgentModelQuality", solution_id, version) ) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(model_quality_monitor_stack).add(ProtobufConfigAspect(app, "ProtobufConfigModelQuality")) + model_bias_monitor_stack = ModelMonitorStack( app, "ModelBiasModelMonitorStack", @@ -96,6 +114,8 @@ core.Aspects.of(model_bias_monitor_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentModelBias", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(model_bias_monitor_stack).add(ProtobufConfigAspect(app, "ProtobufConfigModelBias")) model_explainability_monitor_stack = ModelMonitorStack( app, @@ -108,6 +128,8 @@ AwsSDKConfigAspect(app, "SDKUserAgentModelExplainability", solution_id, version) ) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(model_explainability_monitor_stack).add(ProtobufConfigAspect(app, "ProtobufConfigModelExplainability")) realtime_stack = BYOMRealtimePipelineStack( app, @@ -117,6 +139,8 @@ core.Aspects.of(realtime_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentRealtime", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(realtime_stack).add(ProtobufConfigAspect(app, "ProtobufConfigRealtime")) autopilot_stack = AutopilotJobStack( app, @@ -126,6 +150,8 @@ core.Aspects.of(autopilot_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentAutopilot", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(autopilot_stack).add(ProtobufConfigAspect(app, "ProtobufConfigAutopilot")) training_stack = TrainingJobStack( app, @@ -136,6 +162,8 @@ core.Aspects.of(training_stack).add(AwsSDKConfigAspect(app, "SDKUserAgentTraining", solution_id, version)) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(training_stack).add(ProtobufConfigAspect(app, "ProtobufConfigTraining")) hyperparameter_tunning_stack = TrainingJobStack( app, @@ -148,6 +176,8 @@ AwsSDKConfigAspect(app, "SDKUserAgentHyperparamater", solution_id, version) ) +# add PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python to handle protobuf breaking changes +core.Aspects.of(hyperparameter_tunning_stack).add(ProtobufConfigAspect(app, "ProtobufConfigHyperparamater")) SingleAccountCodePipelineStack( app, diff --git a/source/lambdas/pipeline_orchestration/lambda_helpers.py b/source/lambdas/pipeline_orchestration/lambda_helpers.py index 7b19a97..9a5d32d 100644 --- a/source/lambdas/pipeline_orchestration/lambda_helpers.py +++ b/source/lambdas/pipeline_orchestration/lambda_helpers.py @@ -447,12 +447,7 @@ def get_model_training_specifc_params(event: Dict[str, Any], job_name: str) -> L return [ ("NotificationsSNSTopicArn", os.environ["MLOPS_NOTIFICATIONS_SNS_TOPIC"]), ("JobName", job_name), - ( - "ImageUri", - get_image_uri(event.get("pipeline_type"), event, os.environ["REGION"]) - if os.environ["USE_MODEL_REGISTRY"] == "No" - else "", - ), + ("ImageUri", get_image_uri(event.get("pipeline_type"), event, os.environ["REGION"])), ("InstanceType", event.get("instance_type", "ml.m4.xlarge")), ("JobInstanceCount", str(event.get("instance_count", "1"))), ("InstanceVolumeSize", str(event.get("instance_volume_size", "20"))), diff --git a/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py b/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py index ac6f8f0..92e93c2 100644 --- a/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py +++ b/source/lib/blueprints/byom/pipeline_definitions/iam_policies.py @@ -29,7 +29,9 @@ def sagemaker_policy_statement(is_realtime_pipeline, endpoint_name, endpoint_nam "sagemaker:DescribeModel", # NOSONAR: permission needs to be repeated for clarity "sagemaker:DeleteModel", ] - resources = [f"{sagemaker_arn_prefix}:model/mlopssagemakermodel*"] + resources = [ + f"{sagemaker_arn_prefix}:model/mlopssagemakermodel*" # NOSONAR: permission needs to be repeated for clarity + ] if is_realtime_pipeline: # extend actions @@ -95,8 +97,9 @@ def sagemaker_model_bias_explainability_baseline_job_policy(): "sagemaker:InvokeEndpoint", # NOSONAR: permission needs to be repeated for clarity ], resources=[ - f"{sagemaker_arn_prefix}:endpoint-config/sagemaker-clarify-endpoint-config*", - f"{sagemaker_arn_prefix}:endpoint/sagemaker-clarify-endpoint*", + f"{sagemaker_arn_prefix}:model/mlopssagemakermodel*", + f"{sagemaker_arn_prefix}:endpoint-config/sm-clarify-config*", + f"{sagemaker_arn_prefix}:endpoint/sm-clarify-*", ], ) @@ -174,6 +177,7 @@ def create_service_role(scope, id, service, description): def sagemaker_monitor_policy_statement(baseline_job_name, monitoring_schedule_name, endpoint_name, monitoring_type): # common permissions actions = [ + "sagemaker:DescribeModel", "sagemaker:DescribeEndpointConfig", "sagemaker:DescribeEndpoint", "sagemaker:CreateMonitoringSchedule", @@ -184,6 +188,7 @@ def sagemaker_monitor_policy_statement(baseline_job_name, monitoring_schedule_na ] # common resources resources = [ + f"{sagemaker_arn_prefix}:model/mlopssagemakermodel*", f"{sagemaker_arn_prefix}:endpoint-config/mlopssagemakerendpointconfig*", f"{sagemaker_arn_prefix}:endpoint/{endpoint_name}", f"{sagemaker_arn_prefix}:monitoring-schedule/{monitoring_schedule_name}", diff --git a/source/lib/protobuf_config_aspect.py b/source/lib/protobuf_config_aspect.py new file mode 100644 index 0000000..393bca0 --- /dev/null +++ b/source/lib/protobuf_config_aspect.py @@ -0,0 +1,26 @@ +# ##################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. A copy of the License is located at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # +# and limitations under the License. # +# ##################################################################################################################### +import jsii +from aws_cdk.core import IAspect, IConstruct, Construct +from aws_cdk.aws_lambda import Function + + +@jsii.implements(IAspect) +class ProtobufConfigAspect(Construct): + def __init__(self, scope: Construct, id: str): + super().__init__(scope, id) + + def visit(self, node: IConstruct): + if isinstance(node, Function): + # this is to handle the protobuf package breaking changes. + node.add_environment(key="PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", value="python") diff --git a/source/requirements-test.txt b/source/requirements-test.txt index fef7085..478ae34 100644 --- a/source/requirements-test.txt +++ b/source/requirements-test.txt @@ -3,4 +3,5 @@ boto3==1.17.23 crhelper==2.0.6 pytest==6.1.2 pytest-cov==2.10.1 -moto[all]==2.0.2 \ No newline at end of file +moto[all]==2.0.2 +protobuf==3.20.* \ No newline at end of file diff --git a/source/requirements.txt b/source/requirements.txt index 5f72451..e72a00e 100644 --- a/source/requirements.txt +++ b/source/requirements.txt @@ -31,4 +31,4 @@ aws-solutions-constructs.aws-apigateway-lambda==1.126.0 aws-solutions-constructs.aws-lambda-sagemakerendpoint==1.126.0 aws-solutions-constructs.core==1.126.0 aws-cdk.cloudformation-include==1.126.0 -aws-cdk.aws-cloudformation==1.126.0 +aws-cdk.aws-cloudformation==1.126.0 \ No newline at end of file