From bb1c9896ac2b708ac7e41bd2649db7e095896d9a Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Fri, 15 Nov 2024 15:31:00 +0200 Subject: [PATCH 1/3] feat: adding alarm that monitors %xx errors in IAM --- infra/aws/index.ts | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/infra/aws/index.ts b/infra/aws/index.ts index 16982cc0ca..8d9ad18d8f 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -3,7 +3,7 @@ import * as aws from "@pulumi/aws"; import * as op from "@1password/op-js"; import * as cloudflare from "@pulumi/cloudflare"; import { secretsManager, amplify } from "infra-libs"; -import {stack, defaultTags} from "../lib/tags"; +import { stack, defaultTags } from "../lib/tags"; const current = aws.getCallerIdentity({}); const regionData = aws.getRegion({}); @@ -40,8 +40,6 @@ const snsAlertsTopicArn = coreInfraStack.getOutput("snsAlertsTopicArn"); const passportXyzDomainName = coreInfraStack.getOutput("passportXyzDomainName"); const passportXyzHostedZoneId = coreInfraStack.getOutput("passportXyzHostedZoneId"); - - const containerInsightsStatus = stack == "production" ? "enabled" : "disabled"; // Manage secrets & envs for Passport XYZ @@ -309,6 +307,30 @@ const albPassportXyzTargetGroup = new aws.lb.TargetGroup(`passport-xyz-iam`, { }, }); +/* + * Alarm for monitoring target 5XX errors + */ +const httpsListenerAlbPrefix = albHttpsListenerArn.apply((arn) => arn.split(":").pop()); +const http5xxTargetAlarm = new aws.cloudwatch.MetricAlarm(`HTTP-Target-5XX-passport-xyz-iam`, { + tags: { ...defaultTags, Name: `HTTP-Target-5XX-passport-xyz-iam` }, + name: `HTTP-Target-5XX-passport-xyz-iam`, + alarmActions: [snsAlertsTopicArn], + okActions: [snsAlertsTopicArn], + + period: 60, + statistic: "Sum", + + datapointsToAlarm: 3, + evaluationPeriods: 5, + + metricName: "HTTPCode_Target_5XX_Count", + namespace: "AWS/ApplicationELB", + + comparisonOperator: "GreaterThanThreshold", + threshold: 0, + treatMissingData: "notBreaching", +}); + const albPassportXyzListenerRule = new aws.lb.ListenerRule(`passport-xyz-iam-https`, { listenerArn: albHttpsListenerArn, priority: 102, // This needs to be grater than the priority number for passport-scroll-badge-service @@ -660,7 +682,7 @@ const ecsAutoScalingTargetXyz = new aws.appautoscaling.Target("autoscaling_targe tags: { ...defaultTags, Name: "autoscaling_target_xyz", - } + }, }); const ecsAutoScalingPolicyXyz = new aws.appautoscaling.Policy("passport-autoscaling-policy-xyz", { @@ -793,7 +815,7 @@ const gitcoinEcsAutoScalingTarget = new aws.appautoscaling.Target("autoscaling_t tags: { ...defaultTags, Name: "autoscaling_target", - } + }, }); const gitcoinEcsAutoScalingPolicy = new aws.appautoscaling.Policy("passport-autoscaling-policy", { @@ -852,7 +874,8 @@ const amplifyAppInfo = coreInfraStack.getOutput("newPassportDomain").apply((doma branchName: passportBranches[stack], environmentVariables: passportXyzAppEnvironment, tags: { ...defaultTags, Name: `${prefix}.${domainName}` }, - buildCommand: "npm install --g lerna@6.6.2 && lerna bootstrap && rm -rf ../node_modules/@tendermint && npm run build", + buildCommand: + "npm install --g lerna@6.6.2 && lerna bootstrap && rm -rf ../node_modules/@tendermint && npm run build", preBuildCommand: "nvm use 20.9.0", artifactsBaseDirectory: "out", customRules: [ From 4e592f4c1214e8abb194121ee2df2b4d92b664ea Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Fri, 15 Nov 2024 15:48:10 +0200 Subject: [PATCH 2/3] fix: alarm config for IAM --- infra/aws/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/infra/aws/index.ts b/infra/aws/index.ts index 8d9ad18d8f..99a45a9507 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -31,6 +31,7 @@ const redisConnectionUrl = pulumi.interpolate`${coreInfraStack.getOutput("static const albDnsName = coreInfraStack.getOutput("coreAlbDns"); const albZoneId = coreInfraStack.getOutput("coreAlbZoneId"); const albHttpsListenerArn = coreInfraStack.getOutput("coreAlbHttpsListenerArn"); +const coreAlbArn = coreInfraStack.getOutput("coreAlbArn"); const passportDataScienceStack = new pulumi.StackReference(`passportxyz/passport-data/${stack}`); const passportDataScienceEndpoint = passportDataScienceStack.getOutput("internalAlbBaseUrl"); @@ -310,7 +311,7 @@ const albPassportXyzTargetGroup = new aws.lb.TargetGroup(`passport-xyz-iam`, { /* * Alarm for monitoring target 5XX errors */ -const httpsListenerAlbPrefix = albHttpsListenerArn.apply((arn) => arn.split(":").pop()); +const coreAlbArnPrefix = coreAlbArn.apply((arn) => arn.split(":").pop()); const http5xxTargetAlarm = new aws.cloudwatch.MetricAlarm(`HTTP-Target-5XX-passport-xyz-iam`, { tags: { ...defaultTags, Name: `HTTP-Target-5XX-passport-xyz-iam` }, name: `HTTP-Target-5XX-passport-xyz-iam`, @@ -326,6 +327,11 @@ const http5xxTargetAlarm = new aws.cloudwatch.MetricAlarm(`HTTP-Target-5XX-passp metricName: "HTTPCode_Target_5XX_Count", namespace: "AWS/ApplicationELB", + dimensions: { + LoadBalancer: coreAlbArnPrefix, + TargetGroup: albPassportXyzTargetGroup.arnSuffix, + }, + comparisonOperator: "GreaterThanThreshold", threshold: 0, treatMissingData: "notBreaching", From babcb01ae72c2668abfa46b9f0a16e9563dfb190 Mon Sep 17 00:00:00 2001 From: Gerald Iakobinyi-Pich Date: Fri, 15 Nov 2024 17:04:05 +0200 Subject: [PATCH 3/3] fix: naming of variable --- infra/aws/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/aws/index.ts b/infra/aws/index.ts index 99a45a9507..aca2e905c1 100644 --- a/infra/aws/index.ts +++ b/infra/aws/index.ts @@ -311,7 +311,7 @@ const albPassportXyzTargetGroup = new aws.lb.TargetGroup(`passport-xyz-iam`, { /* * Alarm for monitoring target 5XX errors */ -const coreAlbArnPrefix = coreAlbArn.apply((arn) => arn.split(":").pop()); +const coreAlbArnSuffix = coreAlbArn.apply((arn) => arn.split(":").pop()); const http5xxTargetAlarm = new aws.cloudwatch.MetricAlarm(`HTTP-Target-5XX-passport-xyz-iam`, { tags: { ...defaultTags, Name: `HTTP-Target-5XX-passport-xyz-iam` }, name: `HTTP-Target-5XX-passport-xyz-iam`, @@ -328,7 +328,7 @@ const http5xxTargetAlarm = new aws.cloudwatch.MetricAlarm(`HTTP-Target-5XX-passp namespace: "AWS/ApplicationELB", dimensions: { - LoadBalancer: coreAlbArnPrefix, + LoadBalancer: coreAlbArnSuffix, TargetGroup: albPassportXyzTargetGroup.arnSuffix, },