Bump docker tag by cloudnativedaysjp/dreamkast#2445

commit: cloudnativedaysjp/dreamkast@fae1523
cloudnativedaysjp · Nov 11, 2024 · 27e7c64 · 27e7c64
1 parent e4d46cc
commit 27e7c64
Show file tree

Hide file tree

Showing 18 changed files with 438 additions and 0 deletions.
diff --git a/ecspresso/reviewapps/dk-2445/cleanup.sh b/ecspresso/reviewapps/dk-2445/cleanup.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+cd $(dirname $0)
+
+find . -name "ecspresso.yml" | xargs -I{} -P10 ecspresso --config={} delete --force --terminate ||:
+sleep 10 # wait for ECS Services to be deleted
+aws events delete-rule --name dk-2445-harvestjob
+aws ecs deregister-task-definition --task-definition dreamkast-dev-dk-2445-harvestjob
+aws servicediscovery get-service --id srv-s3oyoi7hl3c5dsab &>/dev/null && aws servicediscovery delete-service --id srv-s3oyoi7hl3c5dsab
+aws servicediscovery get-service --id srv-6afgopoy3yvjzhld &>/dev/null && aws servicediscovery delete-service --id srv-6afgopoy3yvjzhld
+aws elbv2 describe-rules --rule-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:listener-rule/app/dreamkast-dev/122c5b4a47b64f9d/bc86e7b2e4bca8f5/f3279a80d5d6108e &>/dev/null && aws elbv2 delete-rule --rule-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:listener-rule/app/dreamkast-dev/122c5b4a47b64f9d/bc86e7b2e4bca8f5/f3279a80d5d6108e
+aws elbv2 describe-target-groups --target-group-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:targetgroup/dev-dk-2445/f9d4a2f3bd520442 &>/dev/null && aws elbv2 delete-target-group --target-group-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:targetgroup/dev-dk-2445/f9d4a2f3bd520442
+:
diff --git a/ecspresso/reviewapps/dk-2445/const.libsonnet b/ecspresso/reviewapps/dk-2445/const.libsonnet
@@ -0,0 +1,45 @@
+{
+  PR_NAME: 'dk-2445',
+  externalEndpoints: {
+    dkApi: 'https://api.dev.cloudnativedays.jp',
+    dkWeaver: 'https://dkw.dev.cloudnativedays.jp',
+  },
+  imageTags: {
+    dreamkast_ecs: 'fae1523c58feb0d412b0b45dd76635c313254cec',
+    mysql: '8.0.33',
+    redis: '6.0',
+  },
+  internalEndpoints: {
+    rdb: 'mysql-dk-2445.development.local',
+    redis: 'redis://redis-dk-2445.development.local',
+  },
+  publicSubnetIDs: [
+    'subnet-00709135a42bf907e',
+    'subnet-0d07831c8fc073511',
+    'subnet-033491d41490494b6',
+  ],
+  region: 'us-west-2',
+  s3: {
+    dreamkast: {
+      name: 'dreamkast-dev-bucket',
+      region: 'us-west-2',
+    },
+  },
+  secretManager: {
+    dk: 'dreamkast/reviewapp-env-yGJKrj',
+    railsApp: 'dreamkast/rails-app-secret-SqidNC',
+  },
+  sentry: {
+    dsn: 'TODO',
+  },
+  serviceDiscovery: {
+    mysql: 'srv-s3oyoi7hl3c5dsab',
+    redis: 'srv-6afgopoy3yvjzhld',
+  },
+  sqs: {
+    fifo: 'dreamkast-stg-fifo-queue',
+  },
+  targetGroupArn: {
+    dk: 'arn:aws:elasticloadbalancing:us-west-2:607167088920:targetgroup/dev-dk-2445/f9d4a2f3bd520442',
+  },
+}
diff --git a/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/ecspresso.yml b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/ecspresso.yml
@@ -0,0 +1,6 @@
+region: us-west-2
+cluster: dreamkast-dev
+service: dk-2445-dreamkast-fifo-worker
+service_definition: service-def.jsonnet
+task_definition: task-def.jsonnet
+timeout: 10m
diff --git a/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/service-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/service-def.jsonnet
@@ -0,0 +1,8 @@
+local dreamkast_fifo_worker = import '../../../base/dreamkast-fifo-worker.libsonnet';
+local const = import '../const.libsonnet';
+
+dreamkast_fifo_worker.serviceDef(
+  region=const.region,
+  subnetIDs=const.publicSubnetIDs,
+  securityGroupID='sg-0140d2aeaaa5d6d07',  // dreamkast-dev-ecs-dreamkast-fifo-worker
+)
diff --git a/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/task-def.jsonnet
@@ -0,0 +1,26 @@
+local dreamkast_fifo_worker = import '../../../base/dreamkast-fifo-worker.libsonnet';
+local const = import '../const.libsonnet';
+
+dreamkast_fifo_worker.taskDef(
+  family='dreamkast-dev-%s-fifo-worker' % [const.PR_NAME],
+  taskRoleName='dreamkast-dev-ecs-dreamkast-fifo-worker',
+  imageTag=const.imageTags.dreamkast_ecs,
+
+  region=const.region,
+  rdbInternalEndpoint=const.internalEndpoints.rdb,
+  redisInternalEndpoint=const.internalEndpoints.redis,
+
+  s3BucketName=const.s3.dreamkast.name,
+  s3BucketRegion=const.s3.dreamkast.region,
+
+  sqsFifoQueueName=const.sqs.fifo,
+
+  sentryDsn=const.sentry.dsn,
+
+  railsAppSecretManagerName=const.secretManager.railsApp,
+  rdsSecretManagerName=const.secretManager.rds,
+  dreamkastSecretManagerName=const.secretManager.dk,
+
+  enableLogging=true,
+  reviewapp=true,
+)
diff --git a/ecspresso/reviewapps/dk-2445/dreamkast/ecspresso.yml b/ecspresso/reviewapps/dk-2445/dreamkast/ecspresso.yml
@@ -0,0 +1,6 @@
+region: us-west-2
+cluster: dreamkast-dev
+service: dk-2445-dreamkast
+service_definition: service-def.jsonnet
+task_definition: task-def.jsonnet
+timeout: 10m
diff --git a/ecspresso/reviewapps/dk-2445/dreamkast/service-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast/service-def.jsonnet
@@ -0,0 +1,9 @@
+local dreamkast_dk = import '../../../base/dreamkast.libsonnet';
+local const = import '../const.libsonnet';
+
+dreamkast_dk.serviceDef(
+  region=const.region,
+  subnetIDs=const.publicSubnetIDs,
+  securityGroupID='sg-00e734fea020b954d',  // dreamkast-dev-ecs-dreamkast
+  targetGroupArn=const.targetGroupArn.dk,
+)
diff --git a/ecspresso/reviewapps/dk-2445/dreamkast/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast/task-def.jsonnet
@@ -0,0 +1,28 @@
+local dreamkast = import '../../../base/dreamkast.libsonnet';
+local const = import '../const.libsonnet';
+
+dreamkast.taskDef(
+  family='dreamkast-dev-%s-dk' % [const.PR_NAME],
+  taskRoleName='dreamkast-dev-ecs-dreamkast',
+  imageTag=const.imageTags.dreamkast_ecs,
+
+  region=const.region,
+  dkApiEndpoint=const.externalEndpoints.dkApi,
+  dkWeaverEndpoint=const.externalEndpoints.dkWeaver,
+  rdbInternalEndpoint=const.internalEndpoints.rdb,
+  redisInternalEndpoint=const.internalEndpoints.redis,
+
+  s3BucketName=const.s3.dreamkast.name,
+  s3BucketRegion=const.s3.dreamkast.region,
+
+  sqsFifoQueueName=const.sqs.fifo,
+
+  sentryDsn=const.sentry.dsn,
+
+  railsAppSecretManagerName=const.secretManager.railsApp,
+  rdsSecretManagerName=const.secretManager.rds,
+  dreamkastSecretManagerName=const.secretManager.dk,
+
+  enableLogging=true,
+  reviewapp=true,
+)
diff --git a/ecspresso/reviewapps/dk-2445/harvestjob/ecschedule.yml b/ecspresso/reviewapps/dk-2445/harvestjob/ecschedule.yml
@@ -0,0 +1,19 @@
+region: us-west-2
+cluster: dreamkast-dev
+role: dreamkast-dev-ecs-scheduled-task-target-role
+rules:
+- name: dk-2445-harvestjob
+  scheduleExpression: "cron(*/3 * * * ? *)"
+  taskDefinition: dreamkast-dev-dk-2445-harvestjob
+  launch_type: FARGATE
+  platform_version: LATEST
+  network_configuration:
+    aws_vpc_configuration:
+      subnets:
+      - 'subnet-00709135a42bf907e' # dreamkast-dev-vpc-public-us-west-2a
+      - 'subnet-0d07831c8fc073511' # dreamkast-dev-vpc-public-us-west-2b
+      - 'subnet-033491d41490494b6' # dreamkast-dev-vpc-public-us-west-2c
+      security_groups:
+      - sg-05592a72e569c245b # dreamkast-dev-ecs-harvestjob
+      assign_public_ip: ENABLED
+
diff --git a/ecspresso/reviewapps/dk-2445/harvestjob/ecspresso.taskdef.yml b/ecspresso/reviewapps/dk-2445/harvestjob/ecspresso.taskdef.yml
@@ -0,0 +1,4 @@
+region: us-west-2
+cluster: dreamkast-dev
+task_definition: task-def.jsonnet
+timeout: 5m
diff --git a/ecspresso/reviewapps/dk-2445/harvestjob/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/harvestjob/task-def.jsonnet
@@ -0,0 +1,115 @@
+local const = import '../const.libsonnet';
+local family = 'dreamkast-dev-%s-harvestjob' % [const.PR_NAME];
+local executionRoleName = 'dreamkast-dev-ecs-task-execution-role';
+local roleName = 'dreamkast-dev-ecs-harvestjob';
+
+{
+  containerDefinitions: [
+    {
+      local container = self,
+
+      name: 'dreamkast',
+      image: '607167088920.dkr.ecr.%s.amazonaws.com/dreamkast-ecs:%s' % [const.region, const.imageTags.dreamkast_ecs],
+      essential: true,
+      entryPoint: [
+        '/bin/bash',
+        '-c',
+      ],
+      command: [
+        'bundle exec rake util:polling_harvest_job_and_update_video',
+      ],
+      environment: [
+        {
+          name: 'RAILS_ENV',
+          value: 'production',
+        },
+        {
+          name: 'MYSQL_HOST',
+          value: const.internalEndpoints.rdb,
+        },
+        {
+          name: 'MYSQL_DATABASE',
+          value: 'dreamkast',
+        },
+        {
+          name: 'REDIS_URL',
+          value: const.internalEndpoints.redis,
+        },
+        {
+          name: 'SENTRY_DSN',
+          value: const.sentry.dsn,
+        },
+        {
+          name: 'S3_BUCKET',
+          value: 'dreamkast-dev-bucket',
+        },
+        {
+          name: 'S3_REGION',
+          value: const.region,
+        },
+        {
+          name: 'SQS_MAIL_QUEUE_URL',
+          value: 'https://sqs.%s.amazonaws.com/607167088920/%s.fifo' % [const.region, const.sqs.fifo],
+        },
+        {
+          name: 'DREAMKAST_NAMESPACE',
+          value: 'dreamkast-dev',
+        },
+        {
+          name: 'REVIEW_APP',
+          value: 'true',
+        },
+      ],
+      secrets: [
+        {
+          name: 'SLACK_WEBHOOK_URL',
+          valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:SLACK_CHANNEL_FOR_HARVEST_JOB_NOTIFICATION::' % [const.region, const.secretManager.dk],
+        },
+        {
+          name: 'SLACK_CHANNEL',
+          valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:SLACK_CHANNEL_FOR_HARVEST_JOB_NOTIFICATION::' % [const.region, const.secretManager.dk],
+        },
+        {
+          name: 'RAILS_MASTER_KEY',
+          valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s' % [const.region, const.secretManager.railsApp],
+        },
+        {
+          name: 'AUTH0_CLIENT_ID',
+          valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:AUTH0_CLIENT_ID::' % [const.region, const.secretManager.dk],
+        },
+        {
+          name: 'AUTH0_CLIENT_SECRET',
+          valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:AUTH0_CLIENT_SECRET::' % [const.region, const.secretManager.dk],
+        },
+        {
+          name: 'AUTH0_DOMAIN',
+          valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:AUTH0_DOMAIN::' % [const.region, const.secretManager.dk],
+        },
+      ],
+
+      logConfiguration: {
+        logDriver: 'awslogs',
+        options: {
+          'awslogs-create-group': 'true',
+          'awslogs-group': family,
+          'awslogs-region': const.region,
+          'awslogs-stream-prefix': container.name,
+        },
+      },
+
+      cpu: 256,
+      memory: 512,
+      memoryReservation: 512,
+    },
+  ],
+  family: family,
+  cpu: '256',
+  memory: '512',
+  executionRoleArn: 'arn:aws:iam::607167088920:role/%s' % [executionRoleName],
+  taskRoleArn: 'arn:aws:iam::607167088920:role/%s' % [roleName],
+  networkMode: 'awsvpc',
+  requiresCompatibilities: [
+    'FARGATE',
+  ],
+  volumes: [],
+}
diff --git a/ecspresso/reviewapps/dk-2445/initialize.sh b/ecspresso/reviewapps/dk-2445/initialize.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+cd $(dirname $0)
+
+# required the following commands:
+# * aws
+# * jq
+# * jsonnet
+# * jsonnetfmt
+
+# variables
+PR_NAME=${PR_NAME:?"PR_NAME must be specified"}
+PR_NUMBER=${PR_NUMBER:?"PR_NUMBER must be specified"}
+IMAGE_TAG=${IMAGE_TAG:?"IMAGE_TAG must be specified"}
+LISTENER_RULE_PRIORITY_BASE=30000
+LISTENER_RULE_PRIORITY=$(( LISTENER_RULE_PRIORITY_BASE + PR_NUMBER ))
+
+VPC_ID="vpc-0f0d012967c635f34"
+LISTENER_ARN="arn:aws:elasticloadbalancing:us-west-2:607167088920:listener/app/dreamkast-dev/122c5b4a47b64f9d/bc86e7b2e4bca8f5"
+SERVICE_DISCOVERY_NAMESPACE="ns-53ijjrlltqf5r2gm"
+
+
+if [ ! -f "cleanup.sh" ]; then
+# create TargetGroup
+TARGET_GROUP_ARN=$(aws elbv2 create-target-group \
+  --name "dev-${PR_NAME}" \
+  --target-type ip \
+  --protocol HTTP \
+  --port 3000 \
+  --vpc-id ${VPC_ID} \
+  --ip-address-type ipv4 \
+  | jq -r ".TargetGroups[0].TargetGroupArn")
+
+# create ALB ListenerRule
+LISTENER_RULE_ARN=$(aws elbv2 create-rule --listener-arn ${LISTENER_ARN} \
+  --priority ${LISTENER_RULE_PRIORITY} \
+  --conditions Field=host-header,Values="dreamkast-${PR_NAME}.dev.cloudnativedays.jp" \
+  --actions Type=forward,TargetGroupArn=${TARGET_GROUP_ARN} \
+  | jq -r ".Rules[] | select(.Priority == \"${LISTENER_RULE_PRIORITY}\") | .RuleArn")
+
+# create ServiceDiscovery Services
+SERVICE_ID_MYSQL=$(aws servicediscovery create-service \
+  --name "mysql-${PR_NAME}" \
+  --dns-config "NamespaceId="${SERVICE_DISCOVERY_NAMESPACE}",DnsRecords=[{Type="A",TTL="10"}]" \
+  --health-check-custom-config FailureThreshold=1 \
+  | jq -r ".Service.Id")
+
+SERVICE_ID_REDIS=$(aws servicediscovery create-service \
+  --name "redis-${PR_NAME}" \
+  --dns-config "NamespaceId="${SERVICE_DISCOVERY_NAMESPACE}",DnsRecords=[{Type="A",TTL="10"}]" \
+  --health-check-custom-config FailureThreshold=1 \
+  | jq -r ".Service.Id")
+
+# replace variables in each ecspresso.yml
+find . -name ecspresso.yml | xargs -I{} sed -i -e 's/__PR_NAME__/'${PR_NAME}'/g' {}
+# replace variables in each ecschedule.yml
+find . -name ecschedule.yml | xargs -I{} sed -i -e 's/__PR_NAME__/'${PR_NAME}'/g' {}
+
+# replace variables in const.libsonnet
+cat << _EOL_ | jsonnet - > ./const.libsonnet.tmp
+local const = import './const.libsonnet';
+
+const + {
+  PR_NAME: "${PR_NAME}",
+  targetGroupArn: {
+    dk: "${TARGET_GROUP_ARN}",
+  },
+  serviceDiscovery: {
+    mysql: "${SERVICE_ID_MYSQL}",
+    redis: "${SERVICE_ID_REDIS}",
+  },
+  imageTags: const.imageTags + {
+    dreamkast_ecs: "${IMAGE_TAG}",
+  },
+}
+_EOL_
+mv const.libsonnet.tmp const.libsonnet
+
+# create cleanup.sh
+cat << _EOF_ > ./cleanup.sh
+#!/usr/bin/env bash
+set -e -o pipefail
+cd \$(dirname \$0)
+
+find . -name "ecspresso.yml" | xargs -I{} -P10 ecspresso --config={} delete --force --terminate ||:
+sleep 10 # wait for ECS Services to be deleted
+aws events delete-rule --name ${PR_NAME}-harvestjob
+aws ecs deregister-task-definition --task-definition dreamkast-dev-${PR_NAME}-harvestjob
+aws servicediscovery get-service --id ${SERVICE_ID_MYSQL} &>/dev/null && aws servicediscovery delete-service --id ${SERVICE_ID_MYSQL}
+aws servicediscovery get-service --id ${SERVICE_ID_REDIS} &>/dev/null && aws servicediscovery delete-service --id ${SERVICE_ID_REDIS}
+aws elbv2 describe-rules --rule-arn ${LISTENER_RULE_ARN} &>/dev/null && aws elbv2 delete-rule --rule-arn ${LISTENER_RULE_ARN}
+aws elbv2 describe-target-groups --target-group-arn ${TARGET_GROUP_ARN} &>/dev/null && aws elbv2 delete-target-group --target-group-arn ${TARGET_GROUP_ARN}
+:
+_EOF_
+
+else
+# update imageTags
+jsonnet const.libsonnet \
+  | jq ".imageTags.dreamkast_ecs|=\"${IMAGE_TAG}\"" \
+  > const.libsonnet.tmp
+mv const.libsonnet.tmp const.libsonnet
+
+fi
+
+jsonnetfmt -i const.libsonnet