feat: Notification on runner image build failures (CloudSnorkel#278)

```
const runners = new GitHubRunners(this, 'runners');
const topic = runners.failedImageBuildsTopic();
topic.addSubscription(new subs.EmailSubscription('[email protected]'))
```

Fix CloudSnorkel#277

src/providers/image-builders/aws-image-builder/builder.ts

@@ -9,17 +9,20 @@ import {
   aws_imagebuilder as imagebuilder,
   aws_logs as logs,
   aws_s3_assets as s3_assets,
+  aws_sns as sns,
+  aws_sns_subscriptions as subs,
   CustomResource,
   Duration,
   RemovalPolicy,
   Stack,
 } from 'aws-cdk-lib';
 import { TagMutability, TagStatus } from 'aws-cdk-lib/aws-ecr';
 import { RetentionDays } from 'aws-cdk-lib/aws-logs';
-import { Construct } from 'constructs';
+import { Construct, IConstruct } from 'constructs';
 import { AmiRecipe, defaultBaseAmi } from './ami';
 import { ImageBuilderObjectBase } from './common';
 import { ContainerRecipe, defaultBaseDockerImage } from './container';
+import { FilterFailedBuildsFunction } from './filter-failed-builds-function';
 import { BuildImageFunction } from '../../../lambdas/build-image-function';
 import { DeleteAmiFunction } from '../../../lambdas/delete-ami-function';
 import { singletonLambda } from '../../../utils';
@@ -643,3 +646,34 @@ export class AwsImageBuilderRunnerImageBuilder extends RunnerImageBuilderBase {
     return this.boundComponents;
   }
 }
+
+/**
+ * @internal
+ */
+export class AwsImageBuilderFailedBuildNotifier implements cdk.IAspect {
+  public static createFilteringTopic(scope: Construct, targetTopic: sns.Topic) {
+    const topic = new sns.Topic(scope, 'Image Builder Builds');
+    const filter = new FilterFailedBuildsFunction(scope, 'Image Builder Builds Filter', {
+      logRetention: logs.RetentionDays.ONE_MONTH,
+      environment: {
+        TARGET_TOPIC_ARN: targetTopic.topicArn,
+      },
+    });
+
+    topic.addSubscription(new subs.LambdaSubscription(filter));
+    targetTopic.grantPublish(filter);
+
+    return topic;
+  }
+
+  constructor(private topic: sns.ITopic) {
+  }
+
+  public visit(node: IConstruct): void {
+    if (node instanceof AwsImageBuilderRunnerImageBuilder) {
+      const builder = node as AwsImageBuilderRunnerImageBuilder;
+      const infra = builder.node.findChild('Infrastructure') as imagebuilder.CfnInfrastructureConfiguration;
+      infra.snsTopicArn = this.topic.topicArn;
+    }
+  }
+}

src/providers/image-builders/aws-image-builder/filter-failed-builds.lambda.ts

@@ -0,0 +1,19 @@
+/* eslint-disable-next-line import/no-extraneous-dependencies,import/no-unresolved */
+import * as AWSLambda from 'aws-lambda';
+/* eslint-disable-next-line import/no-extraneous-dependencies */
+import * as AWS from 'aws-sdk';
+
+const sns = new AWS.SNS();
+
+exports.handler = async function(event: AWSLambda.SNSEvent) {
+  console.log(JSON.stringify(event));
+  for (const record of event.Records) {
+    let message = JSON.parse(record.Sns.Message);
+    if (message.state.status === 'FAILED') {
+      await sns.publish({
+        TopicArn: process.env.TARGET_TOPIC_ARN,
+        Message: record.Sns.Message,
+      }).promise();
+    }
+  }
+};

src/providers/image-builders/codebuild.ts

@@ -9,14 +9,15 @@ import {
   aws_iam as iam,
   aws_logs as logs,
   aws_s3_assets as s3_assets,
+  aws_sns as sns,
   CustomResource,
   Duration,
   RemovalPolicy,
 } from 'aws-cdk-lib';
 import { ComputeType } from 'aws-cdk-lib/aws-codebuild';
 import { TagMutability, TagStatus } from 'aws-cdk-lib/aws-ecr';
 import { RetentionDays } from 'aws-cdk-lib/aws-logs';
-import { Construct } from 'constructs';
+import { Construct, IConstruct } from 'constructs';
 import { defaultBaseDockerImage } from './aws-image-builder';
 import { RunnerImageBuilderBase, RunnerImageBuilderProps } from './common';
 import { BuildImageFunction } from '../../lambdas/build-image-function';
@@ -362,3 +363,19 @@ export class CodeBuildRunnerImageBuilder extends RunnerImageBuilderBase {
     return this.role;
   }
 }
+
+/**
+ * @internal
+ */
+export class CodeBuildImageBuilderFailedBuildNotifier implements cdk.IAspect {
+  constructor(private topic: sns.ITopic) {
+  }
+
+  public visit(node: IConstruct): void {
+    if (node instanceof CodeBuildRunnerImageBuilder) {
+      const builder = node as CodeBuildRunnerImageBuilder;
+      const project = builder.node.findChild('CodeBuild') as codebuild.Project;
+      project.notifyOnBuildFailed('BuildFailed', this.topic);
+    }
+  }
+}

src/runner.ts

@@ -6,6 +6,7 @@ import {
   aws_iam as iam,
   aws_lambda as lambda,
   aws_logs as logs,
+  aws_sns as sns,
   aws_stepfunctions as stepfunctions,
   aws_stepfunctions_tasks as stepfunctions_tasks,
   RemovalPolicy,
@@ -16,6 +17,7 @@ import { DeleteRunnerFunction } from './lambdas/delete-runner-function';
 import { SetupFunction } from './lambdas/setup-function';
 import { StatusFunction } from './lambdas/status-function';
 import { TokenRetrieverFunction } from './lambdas/token-retriever-function';
+import { AwsImageBuilderFailedBuildNotifier, CodeBuildImageBuilderFailedBuildNotifier } from './providers';
 import { CodeBuildRunnerProvider } from './providers/codebuild';
 import { IRunnerProvider } from './providers/common';
 import { FargateRunnerProvider } from './providers/fargate';
@@ -614,4 +616,23 @@ export class GitHubRunners extends Construct {
   public metricTime(props?: cloudwatch.MetricProps): cloudwatch.Metric {
     return this.orchestrator.metricTime(props);
   }
+
+  /**
+   * Creates a topic for notifications when a runner image build fails.
+   *
+   * Runner images are rebuilt every week by default. This provides the latest GitHub Runner version and software updates.
+   *
+   * If you want to be sure you are using the latest runner version, you can use this topic to be notified when a build fails.
+   */
+  public failedImageBuildsTopic() {
+    const topic = new sns.Topic(this, 'Failed Runner Image Builds');
+    const stack = cdk.Stack.of(this);
+    cdk.Aspects.of(stack).add(new CodeBuildImageBuilderFailedBuildNotifier(topic));
+    cdk.Aspects.of(stack).add(
+      new AwsImageBuilderFailedBuildNotifier(
+        AwsImageBuilderFailedBuildNotifier.createFilteringTopic(this, topic),
+      ),
+    );
+    return topic;
+  }
 }

test/default.integ.snapshot/github-runners-test.assets.json

@@ -183,15 +183,28 @@
         }
       }
     },
-    "d683e78c33c4de0163529ce1ae47b62cbdea581ddc99af53dff5e914efc37944": {
+    "a03b764f5d4bc761c5335ccb74f4496f0ac4044feac5c489f1b70e2f082f25f8": {
+      "source": {
+        "path": "asset.a03b764f5d4bc761c5335ccb74f4496f0ac4044feac5c489f1b70e2f082f25f8.lambda",
+        "packaging": "zip"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "a03b764f5d4bc761c5335ccb74f4496f0ac4044feac5c489f1b70e2f082f25f8.zip",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    },
+    "3eae1c989c4929e82fc388bb698cb8ef1310f800f06fee5e92b0ed0824cc1245": {
       "source": {
         "path": "github-runners-test.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "d683e78c33c4de0163529ce1ae47b62cbdea581ddc99af53dff5e914efc37944.json",
+          "objectKey": "3eae1c989c4929e82fc388bb698cb8ef1310f800f06fee5e92b0ed0824cc1245.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }