From faf8fc68dcdc2bdec6ff77fccecc78c059752568 Mon Sep 17 00:00:00 2001 From: alinadima Date: Fri, 18 Aug 2023 16:10:40 +0200 Subject: [PATCH 1/7] Add Lambda to check ECR before running build. --- demos-pipeline/lib/lib/demo-pipeline.ts | 63 +++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/demos-pipeline/lib/lib/demo-pipeline.ts b/demos-pipeline/lib/lib/demo-pipeline.ts index 6978f27..a7884de 100644 --- a/demos-pipeline/lib/lib/demo-pipeline.ts +++ b/demos-pipeline/lib/lib/demo-pipeline.ts @@ -2,6 +2,12 @@ import * as cdk from "aws-cdk-lib"; import { Construct } from "constructs"; import * as codepipeline from "aws-cdk-lib/aws-codepipeline"; import * as codepipeline_actions from "aws-cdk-lib/aws-codepipeline-actions"; +import * as events from "aws-cdk-lib/aws-events"; +import * as targets from "aws-cdk-lib/aws-events-targets"; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as efs from "aws-cdk-lib/aws-efs"; + import { BuildSpec, ComputeType, @@ -10,7 +16,7 @@ import { PipelineProject, } from "aws-cdk-lib/aws-codebuild"; import { IRepository } from "aws-cdk-lib/aws-ecr"; -import * as efs from "aws-cdk-lib/aws-efs"; + import { ISecurityGroup, IVpc, @@ -20,6 +26,7 @@ import { } from "aws-cdk-lib/aws-ec2"; import { Bucket } from "aws-cdk-lib/aws-s3"; import { SourceRepo, DistributionKind } from "./constructs/source-repo"; +import {CodeCommitTrigger} from "aws-cdk-lib/aws-codepipeline-actions"; /** * Properties to allow customizing the build. @@ -70,6 +77,7 @@ export class DemoPipelineStack extends cdk.Stack { const sourceOutput = new codepipeline.Artifact(); const sourceAction = new codepipeline_actions.CodeCommitSourceAction({ + trigger: CodeCommitTrigger.NONE, output: sourceOutput, actionName: "Source", repository: sourceRepo.repo, @@ -121,7 +129,6 @@ export class DemoPipelineStack extends cdk.Stack { versioned: true, enforceSSL: true, }); - const artifactAction = new codepipeline_actions.S3DeployAction({ actionName: "Demo-Artifact", input: buildOutput, @@ -129,7 +136,7 @@ export class DemoPipelineStack extends cdk.Stack { }); /** Now create the actual Pipeline */ - new codepipeline.Pipeline(this, "DemoPipeline", { + const pipeline = new codepipeline.Pipeline(this, "DemoPipeline", { restartExecutionOnUpdate: true, stages: [ { @@ -146,6 +153,56 @@ export class DemoPipelineStack extends cdk.Stack { }, ], }); + + /** Here we create the logic to check for presence of ECR image on CodeCommit repo creation, + * and only start pipeline if image exists. On CodeCommit repo updates, just trigger pipeline. */ + const fn = new lambda.Function(this, 'OSImageCheck', { + runtime: lambda.Runtime.PYTHON_3_10, + handler: 'index.handler', + code: lambda.Code.fromInline(` +import boto3 + +codecommit_client = boto3.client('codecommit') +ecr_client = boto3.client('ecr') +codepipeline_client = boto3.client('codepipeline') + +def handler(event, context): + response = ecr_client.describe_images(repositoryName=${props.imageRepo.repositoryName}, filter={'tagStatus': 'TAGGED'}) + for i in response['imageDetails']: + if ${props.imageTag} in i['imageTags']: + codepipeline_response = codepipeline_client.start_pipeline_execution(name=${pipeline.pipelineName}) + break + ` + ), + }); + + const startPipelinePolicy = new iam.PolicyStatement({ + actions: ['codepipeline:StartPipelineExecution'], + resources: [pipeline.pipelineArn], + }); + const ecrPolicy = new iam.PolicyStatement({ + actions: ['ecr:DescribeImages'], + resources: [props.imageRepo.repositoryArn], + }); + + fn.role?.attachInlinePolicy( + new iam.Policy(this, 'CheckOSAndStart', { + statements: [startPipelinePolicy, ecrPolicy], + }), + ); + + const ruleOnCreateOrUpdate = new events.Rule(this, 'BuildTriggerRule', { + eventPattern: { + source: ['aws.codecommit'], + resources: [sourceRepo.repo.repositoryArn], + detail: { + event: ['referenceCreated', 'referenceUpdated'], + referenceName: ['main'] + }, + detailType: ['CodeCommit Repository State Change'], + }, + }); + ruleOnCreateOrUpdate.addTarget(new targets.LambdaFunction(fn)); } /** From ae690b5969b447c7d7e588b9c0bd7a825ecf1b55 Mon Sep 17 00:00:00 2001 From: alinadima Date: Fri, 18 Aug 2023 17:21:13 +0200 Subject: [PATCH 2/7] Fix to Python indentation. --- demos-pipeline/lib/lib/demo-pipeline.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/demos-pipeline/lib/lib/demo-pipeline.ts b/demos-pipeline/lib/lib/demo-pipeline.ts index a7884de..4655c9b 100644 --- a/demos-pipeline/lib/lib/demo-pipeline.ts +++ b/demos-pipeline/lib/lib/demo-pipeline.ts @@ -167,11 +167,11 @@ ecr_client = boto3.client('ecr') codepipeline_client = boto3.client('codepipeline') def handler(event, context): - response = ecr_client.describe_images(repositoryName=${props.imageRepo.repositoryName}, filter={'tagStatus': 'TAGGED'}) - for i in response['imageDetails']: - if ${props.imageTag} in i['imageTags']: - codepipeline_response = codepipeline_client.start_pipeline_execution(name=${pipeline.pipelineName}) - break + response = ecr_client.describe_images(repositoryName='${props.imageRepo.repositoryName}', filter={'tagStatus': 'TAGGED'}) + for i in response['imageDetails']: + if '${props.imageTag}' in i['imageTags']: + codepipeline_response = codepipeline_client.start_pipeline_execution(name='${pipeline.pipelineName}') + break ` ), }); From d6a7694ebbd15bdfb4fb11d9570e9020d15f05c1 Mon Sep 17 00:00:00 2001 From: alinadima Date: Mon, 21 Aug 2023 15:29:58 +0200 Subject: [PATCH 3/7] Incercept pipeline creation triggered start and check for image presence. --- demos-pipeline/lib/lib/demo-pipeline.ts | 130 +++++++++++++++++------- demos-pipeline/lib/lib/network.ts | 2 +- 2 files changed, 95 insertions(+), 37 deletions(-) diff --git a/demos-pipeline/lib/lib/demo-pipeline.ts b/demos-pipeline/lib/lib/demo-pipeline.ts index 4655c9b..8af24f8 100644 --- a/demos-pipeline/lib/lib/demo-pipeline.ts +++ b/demos-pipeline/lib/lib/demo-pipeline.ts @@ -4,8 +4,8 @@ import * as codepipeline from "aws-cdk-lib/aws-codepipeline"; import * as codepipeline_actions from "aws-cdk-lib/aws-codepipeline-actions"; import * as events from "aws-cdk-lib/aws-events"; import * as targets from "aws-cdk-lib/aws-events-targets"; -import * as lambda from 'aws-cdk-lib/aws-lambda'; -import * as iam from 'aws-cdk-lib/aws-iam'; +import * as lambda from "aws-cdk-lib/aws-lambda"; +import * as iam from "aws-cdk-lib/aws-iam"; import * as efs from "aws-cdk-lib/aws-efs"; import { @@ -26,7 +26,7 @@ import { } from "aws-cdk-lib/aws-ec2"; import { Bucket } from "aws-cdk-lib/aws-s3"; import { SourceRepo, DistributionKind } from "./constructs/source-repo"; -import {CodeCommitTrigger} from "aws-cdk-lib/aws-codepipeline-actions"; +import { CodeCommitTrigger } from "aws-cdk-lib/aws-codepipeline-actions"; /** * Properties to allow customizing the build. @@ -77,7 +77,7 @@ export class DemoPipelineStack extends cdk.Stack { const sourceOutput = new codepipeline.Artifact(); const sourceAction = new codepipeline_actions.CodeCommitSourceAction({ - trigger: CodeCommitTrigger.NONE, + // trigger: CodeCommitTrigger.NONE, output: sourceOutput, actionName: "Source", repository: sourceRepo.repo, @@ -135,6 +135,68 @@ export class DemoPipelineStack extends cdk.Stack { bucket: artifactBucket, }); + /** Here we create the logic to check for presence of ECR image on the CodePipeline automatic triggering upon resource creation, + * and stop the execution if the image does not exist. */ + const fnOnPipelineCreate = new lambda.Function( + this, + "OSImageCheckOnStart", + { + runtime: lambda.Runtime.PYTHON_3_10, + handler: "index.handler", + code: lambda.Code.fromInline(` +import boto3 +import json + +ecr_client = boto3.client('ecr') +codepipeline_client = boto3.client('codepipeline') + +def handler(event, context): + print("Received event: " + json.dumps(event, indent=2)) + response = ecr_client.describe_images(repositoryName='${props.imageRepo.repositoryName}', filter={'tagStatus': 'TAGGED'}) + for i in response['imageDetails']: + if '${props.imageTag}' in i['imageTags']: + break + else: + print('OS image not found. Stopping execution.') + response = codepipeline_client.stop_pipeline_execution( + pipelineName=event['detail']['pipeline'], + pipelineExecutionId=event['detail']['execution-id'], + abandon=False, + reason='OS image not found in ECR repository. Stopping pipeline until image is present.') + `), + } + ); + const stopPipelinePolicy = new iam.PolicyStatement({ + actions: ["codepipeline:StopPipelineExecution"], + resources: ["*"], //TODO: fix. + }); + + const ecrPolicy = new iam.PolicyStatement({ + actions: ["ecr:DescribeImages"], + resources: [props.imageRepo.repositoryArn], + }); + fnOnPipelineCreate.role?.attachInlinePolicy( + new iam.Policy(this, "CheckOSAndStop", { + statements: [stopPipelinePolicy, ecrPolicy], + }) + ); + + const pipelineCreateRule = new events.Rule(this, "OnPipelineStartRule", { + eventPattern: { + detailType: ["CodePipeline Pipeline Execution State Change"], + source: ["aws.codepipeline"], + detail: { + state: ["STARTED"], + "execution-trigger": { + "trigger-type": ["CreatePipeline"], + }, + }, + }, + }); + pipelineCreateRule.addTarget( + new targets.LambdaFunction(fnOnPipelineCreate) + ); + /** Now create the actual Pipeline */ const pipeline = new codepipeline.Pipeline(this, "DemoPipeline", { restartExecutionOnUpdate: true, @@ -156,10 +218,10 @@ export class DemoPipelineStack extends cdk.Stack { /** Here we create the logic to check for presence of ECR image on CodeCommit repo creation, * and only start pipeline if image exists. On CodeCommit repo updates, just trigger pipeline. */ - const fn = new lambda.Function(this, 'OSImageCheck', { - runtime: lambda.Runtime.PYTHON_3_10, - handler: 'index.handler', - code: lambda.Code.fromInline(` + /** const fn = new lambda.Function(this, "OSImageCheck", { + runtime: lambda.Runtime.PYTHON_3_10, + handler: "index.handler", + code: lambda.Code.fromInline(` import boto3 codecommit_client = boto3.client('codecommit') @@ -172,37 +234,33 @@ def handler(event, context): if '${props.imageTag}' in i['imageTags']: codepipeline_response = codepipeline_client.start_pipeline_execution(name='${pipeline.pipelineName}') break - ` - ), - }); + `), + });**/ - const startPipelinePolicy = new iam.PolicyStatement({ - actions: ['codepipeline:StartPipelineExecution'], - resources: [pipeline.pipelineArn], - }); - const ecrPolicy = new iam.PolicyStatement({ - actions: ['ecr:DescribeImages'], - resources: [props.imageRepo.repositoryArn], - }); + /** const startPipelinePolicy = new iam.PolicyStatement({ + actions: ["codepipeline:StartPipelineExecution"], + resources: [pipeline.pipelineArn], + });**/ - fn.role?.attachInlinePolicy( - new iam.Policy(this, 'CheckOSAndStart', { - statements: [startPipelinePolicy, ecrPolicy], - }), - ); + /** fn.role?.attachInlinePolicy( + new iam.Policy(this, "CheckOSAndStart", { + statements: [startPipelinePolicy, ecrPolicy], + }) + ); +**/ - const ruleOnCreateOrUpdate = new events.Rule(this, 'BuildTriggerRule', { - eventPattern: { - source: ['aws.codecommit'], - resources: [sourceRepo.repo.repositoryArn], - detail: { - event: ['referenceCreated', 'referenceUpdated'], - referenceName: ['main'] - }, - detailType: ['CodeCommit Repository State Change'], - }, - }); - ruleOnCreateOrUpdate.addTarget(new targets.LambdaFunction(fn)); + /** const ruleOnCreateOrUpdate = new events.Rule(this, "BuildTriggerRule", { + eventPattern: { + source: ["aws.codecommit"], + resources: [sourceRepo.repo.repositoryArn], + detail: { + event: ["referenceCreated", "referenceUpdated"], + referenceName: ["main"], + }, + detailType: ["CodeCommit Repository State Change"], + }, + }); + ruleOnCreateOrUpdate.addTarget(new targets.LambdaFunction(fn));**/ } /** diff --git a/demos-pipeline/lib/lib/network.ts b/demos-pipeline/lib/lib/network.ts index c28b2d4..87000b7 100644 --- a/demos-pipeline/lib/lib/network.ts +++ b/demos-pipeline/lib/lib/network.ts @@ -10,7 +10,7 @@ export class PipelineNetworkStack extends cdk.Stack { public readonly vpc: ec2.IVpc; constructor(scope: Construct, props?: cdk.StackProps) { - super(scope, 'PipelineNetwork', props); + super(scope, "PipelineNetwork", props); // We will create a VPC with 3 Private and Public subnets for AWS // Resources that have network interfaces (e.g. Connecting and EFS From ab88e28afd27fae712affe3a085f76b8819cf4c1 Mon Sep 17 00:00:00 2001 From: alinadima Date: Mon, 21 Aug 2023 15:32:59 +0200 Subject: [PATCH 4/7] Fix eslint --- demos-pipeline/example/bin/poky-pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos-pipeline/example/bin/poky-pipeline.ts b/demos-pipeline/example/bin/poky-pipeline.ts index b0fd1d1..5eacc44 100644 --- a/demos-pipeline/example/bin/poky-pipeline.ts +++ b/demos-pipeline/example/bin/poky-pipeline.ts @@ -50,7 +50,7 @@ new BuildImagePipelineStack(app, "BuildImagePipeline", { /** * Set up networking to allow us to securely attach EFS to our CodeBuild instances. */ -const vpc = new PipelineNetworkStack(app, { +const vpc = new PipelineNetworkStack(app, { ...defaultProps, }); From 9957f29c28df758ff93cb1e1bbf63edb2fcd1ad1 Mon Sep 17 00:00:00 2001 From: alinadima Date: Mon, 21 Aug 2023 15:38:20 +0200 Subject: [PATCH 5/7] Remove commented code for now. --- demos-pipeline/lib/lib/demo-pipeline.ts | 46 ------------------------- 1 file changed, 46 deletions(-) diff --git a/demos-pipeline/lib/lib/demo-pipeline.ts b/demos-pipeline/lib/lib/demo-pipeline.ts index 8af24f8..1c8a19f 100644 --- a/demos-pipeline/lib/lib/demo-pipeline.ts +++ b/demos-pipeline/lib/lib/demo-pipeline.ts @@ -215,52 +215,6 @@ def handler(event, context): }, ], }); - - /** Here we create the logic to check for presence of ECR image on CodeCommit repo creation, - * and only start pipeline if image exists. On CodeCommit repo updates, just trigger pipeline. */ - /** const fn = new lambda.Function(this, "OSImageCheck", { - runtime: lambda.Runtime.PYTHON_3_10, - handler: "index.handler", - code: lambda.Code.fromInline(` -import boto3 - -codecommit_client = boto3.client('codecommit') -ecr_client = boto3.client('ecr') -codepipeline_client = boto3.client('codepipeline') - -def handler(event, context): - response = ecr_client.describe_images(repositoryName='${props.imageRepo.repositoryName}', filter={'tagStatus': 'TAGGED'}) - for i in response['imageDetails']: - if '${props.imageTag}' in i['imageTags']: - codepipeline_response = codepipeline_client.start_pipeline_execution(name='${pipeline.pipelineName}') - break - `), - });**/ - - /** const startPipelinePolicy = new iam.PolicyStatement({ - actions: ["codepipeline:StartPipelineExecution"], - resources: [pipeline.pipelineArn], - });**/ - - /** fn.role?.attachInlinePolicy( - new iam.Policy(this, "CheckOSAndStart", { - statements: [startPipelinePolicy, ecrPolicy], - }) - ); -**/ - - /** const ruleOnCreateOrUpdate = new events.Rule(this, "BuildTriggerRule", { - eventPattern: { - source: ["aws.codecommit"], - resources: [sourceRepo.repo.repositoryArn], - detail: { - event: ["referenceCreated", "referenceUpdated"], - referenceName: ["main"], - }, - detailType: ["CodeCommit Repository State Change"], - }, - }); - ruleOnCreateOrUpdate.addTarget(new targets.LambdaFunction(fn));**/ } /** From 961383829e652fa28d7a607ff6be546fd31e90c0 Mon Sep 17 00:00:00 2001 From: alinadima Date: Mon, 21 Aug 2023 16:11:13 +0200 Subject: [PATCH 6/7] Fix policy. Abandon build if image is not present. Do not wait. --- demos-pipeline/lib/lib/demo-pipeline.ts | 32 ++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/demos-pipeline/lib/lib/demo-pipeline.ts b/demos-pipeline/lib/lib/demo-pipeline.ts index 1c8a19f..7da8d3e 100644 --- a/demos-pipeline/lib/lib/demo-pipeline.ts +++ b/demos-pipeline/lib/lib/demo-pipeline.ts @@ -161,25 +161,12 @@ def handler(event, context): response = codepipeline_client.stop_pipeline_execution( pipelineName=event['detail']['pipeline'], pipelineExecutionId=event['detail']['execution-id'], - abandon=False, + abandon=True, reason='OS image not found in ECR repository. Stopping pipeline until image is present.') `), } ); - const stopPipelinePolicy = new iam.PolicyStatement({ - actions: ["codepipeline:StopPipelineExecution"], - resources: ["*"], //TODO: fix. - }); - const ecrPolicy = new iam.PolicyStatement({ - actions: ["ecr:DescribeImages"], - resources: [props.imageRepo.repositoryArn], - }); - fnOnPipelineCreate.role?.attachInlinePolicy( - new iam.Policy(this, "CheckOSAndStop", { - statements: [stopPipelinePolicy, ecrPolicy], - }) - ); const pipelineCreateRule = new events.Rule(this, "OnPipelineStartRule", { eventPattern: { @@ -215,8 +202,25 @@ def handler(event, context): }, ], }); + + const stopPipelinePolicy = new iam.PolicyStatement({ + actions: ["codepipeline:StopPipelineExecution"], + resources: [pipeline.pipelineArn], + }); + + const ecrPolicy = new iam.PolicyStatement({ + actions: ["ecr:DescribeImages"], + resources: [props.imageRepo.repositoryArn], + }); + fnOnPipelineCreate.role?.attachInlinePolicy( + new iam.Policy(this, "CheckOSAndStop", { + statements: [stopPipelinePolicy, ecrPolicy], + }) + ); } + + /** * Adds an EFS FileSystem to the VPC and SecurityGroup. * From b8bc2613d4aeaf94ca39602705ac77dbbb1b6441 Mon Sep 17 00:00:00 2001 From: alinadima Date: Mon, 21 Aug 2023 16:13:04 +0200 Subject: [PATCH 7/7] And lint again --- demos-pipeline/lib/lib/demo-pipeline.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/demos-pipeline/lib/lib/demo-pipeline.ts b/demos-pipeline/lib/lib/demo-pipeline.ts index 7da8d3e..b7766a4 100644 --- a/demos-pipeline/lib/lib/demo-pipeline.ts +++ b/demos-pipeline/lib/lib/demo-pipeline.ts @@ -167,7 +167,6 @@ def handler(event, context): } ); - const pipelineCreateRule = new events.Rule(this, "OnPipelineStartRule", { eventPattern: { detailType: ["CodePipeline Pipeline Execution State Change"], @@ -213,14 +212,12 @@ def handler(event, context): resources: [props.imageRepo.repositoryArn], }); fnOnPipelineCreate.role?.attachInlinePolicy( - new iam.Policy(this, "CheckOSAndStop", { - statements: [stopPipelinePolicy, ecrPolicy], - }) + new iam.Policy(this, "CheckOSAndStop", { + statements: [stopPipelinePolicy, ecrPolicy], + }) ); } - - /** * Adds an EFS FileSystem to the VPC and SecurityGroup. *