diff --git a/ecspresso/reviewapps/dk-2445/cleanup.sh b/ecspresso/reviewapps/dk-2445/cleanup.sh new file mode 100644 index 000000000..08f97a2df --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/cleanup.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -e -o pipefail +cd $(dirname $0) + +find . -name "ecspresso.yml" | xargs -I{} -P10 ecspresso --config={} delete --force --terminate ||: +sleep 10 # wait for ECS Services to be deleted +aws events delete-rule --name dk-2445-harvestjob +aws ecs deregister-task-definition --task-definition dreamkast-dev-dk-2445-harvestjob +aws servicediscovery get-service --id srv-s3oyoi7hl3c5dsab &>/dev/null && aws servicediscovery delete-service --id srv-s3oyoi7hl3c5dsab +aws servicediscovery get-service --id srv-6afgopoy3yvjzhld &>/dev/null && aws servicediscovery delete-service --id srv-6afgopoy3yvjzhld +aws elbv2 describe-rules --rule-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:listener-rule/app/dreamkast-dev/122c5b4a47b64f9d/bc86e7b2e4bca8f5/f3279a80d5d6108e &>/dev/null && aws elbv2 delete-rule --rule-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:listener-rule/app/dreamkast-dev/122c5b4a47b64f9d/bc86e7b2e4bca8f5/f3279a80d5d6108e +aws elbv2 describe-target-groups --target-group-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:targetgroup/dev-dk-2445/f9d4a2f3bd520442 &>/dev/null && aws elbv2 delete-target-group --target-group-arn arn:aws:elasticloadbalancing:us-west-2:607167088920:targetgroup/dev-dk-2445/f9d4a2f3bd520442 +: diff --git a/ecspresso/reviewapps/dk-2445/const.libsonnet b/ecspresso/reviewapps/dk-2445/const.libsonnet new file mode 100644 index 000000000..2bf07bb6d --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/const.libsonnet @@ -0,0 +1,45 @@ +{ + PR_NAME: 'dk-2445', + externalEndpoints: { + dkApi: 'https://api.dev.cloudnativedays.jp', + dkWeaver: 'https://dkw.dev.cloudnativedays.jp', + }, + imageTags: { + dreamkast_ecs: 'fae1523c58feb0d412b0b45dd76635c313254cec', + mysql: '8.0.33', + redis: '6.0', + }, + internalEndpoints: { + rdb: 'mysql-dk-2445.development.local', + redis: 'redis://redis-dk-2445.development.local', + }, + publicSubnetIDs: [ + 'subnet-00709135a42bf907e', + 'subnet-0d07831c8fc073511', + 'subnet-033491d41490494b6', + ], + region: 'us-west-2', + s3: { + dreamkast: { + name: 'dreamkast-dev-bucket', + region: 'us-west-2', + }, + }, + secretManager: { + dk: 'dreamkast/reviewapp-env-yGJKrj', + railsApp: 'dreamkast/rails-app-secret-SqidNC', + }, + sentry: { + dsn: 'TODO', + }, + serviceDiscovery: { + mysql: 'srv-s3oyoi7hl3c5dsab', + redis: 'srv-6afgopoy3yvjzhld', + }, + sqs: { + fifo: 'dreamkast-stg-fifo-queue', + }, + targetGroupArn: { + dk: 'arn:aws:elasticloadbalancing:us-west-2:607167088920:targetgroup/dev-dk-2445/f9d4a2f3bd520442', + }, +} diff --git a/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/ecspresso.yml b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/ecspresso.yml new file mode 100644 index 000000000..2ea35b989 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/ecspresso.yml @@ -0,0 +1,6 @@ +region: us-west-2 +cluster: dreamkast-dev +service: dk-2445-dreamkast-fifo-worker +service_definition: service-def.jsonnet +task_definition: task-def.jsonnet +timeout: 10m diff --git a/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/service-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/service-def.jsonnet new file mode 100644 index 000000000..e5a4be2c2 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/service-def.jsonnet @@ -0,0 +1,8 @@ +local dreamkast_fifo_worker = import '../../../base/dreamkast-fifo-worker.libsonnet'; +local const = import '../const.libsonnet'; + +dreamkast_fifo_worker.serviceDef( + region=const.region, + subnetIDs=const.publicSubnetIDs, + securityGroupID='sg-0140d2aeaaa5d6d07', // dreamkast-dev-ecs-dreamkast-fifo-worker +) diff --git a/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/task-def.jsonnet new file mode 100644 index 000000000..5b6b01021 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/dreamkast-fifo-worker/task-def.jsonnet @@ -0,0 +1,26 @@ +local dreamkast_fifo_worker = import '../../../base/dreamkast-fifo-worker.libsonnet'; +local const = import '../const.libsonnet'; + +dreamkast_fifo_worker.taskDef( + family='dreamkast-dev-%s-fifo-worker' % [const.PR_NAME], + taskRoleName='dreamkast-dev-ecs-dreamkast-fifo-worker', + imageTag=const.imageTags.dreamkast_ecs, + + region=const.region, + rdbInternalEndpoint=const.internalEndpoints.rdb, + redisInternalEndpoint=const.internalEndpoints.redis, + + s3BucketName=const.s3.dreamkast.name, + s3BucketRegion=const.s3.dreamkast.region, + + sqsFifoQueueName=const.sqs.fifo, + + sentryDsn=const.sentry.dsn, + + railsAppSecretManagerName=const.secretManager.railsApp, + rdsSecretManagerName=const.secretManager.rds, + dreamkastSecretManagerName=const.secretManager.dk, + + enableLogging=true, + reviewapp=true, +) diff --git a/ecspresso/reviewapps/dk-2445/dreamkast/ecspresso.yml b/ecspresso/reviewapps/dk-2445/dreamkast/ecspresso.yml new file mode 100644 index 000000000..587e254c1 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/dreamkast/ecspresso.yml @@ -0,0 +1,6 @@ +region: us-west-2 +cluster: dreamkast-dev +service: dk-2445-dreamkast +service_definition: service-def.jsonnet +task_definition: task-def.jsonnet +timeout: 10m diff --git a/ecspresso/reviewapps/dk-2445/dreamkast/service-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast/service-def.jsonnet new file mode 100644 index 000000000..8ee603369 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/dreamkast/service-def.jsonnet @@ -0,0 +1,9 @@ +local dreamkast_dk = import '../../../base/dreamkast.libsonnet'; +local const = import '../const.libsonnet'; + +dreamkast_dk.serviceDef( + region=const.region, + subnetIDs=const.publicSubnetIDs, + securityGroupID='sg-00e734fea020b954d', // dreamkast-dev-ecs-dreamkast + targetGroupArn=const.targetGroupArn.dk, +) diff --git a/ecspresso/reviewapps/dk-2445/dreamkast/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/dreamkast/task-def.jsonnet new file mode 100644 index 000000000..fba481657 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/dreamkast/task-def.jsonnet @@ -0,0 +1,28 @@ +local dreamkast = import '../../../base/dreamkast.libsonnet'; +local const = import '../const.libsonnet'; + +dreamkast.taskDef( + family='dreamkast-dev-%s-dk' % [const.PR_NAME], + taskRoleName='dreamkast-dev-ecs-dreamkast', + imageTag=const.imageTags.dreamkast_ecs, + + region=const.region, + dkApiEndpoint=const.externalEndpoints.dkApi, + dkWeaverEndpoint=const.externalEndpoints.dkWeaver, + rdbInternalEndpoint=const.internalEndpoints.rdb, + redisInternalEndpoint=const.internalEndpoints.redis, + + s3BucketName=const.s3.dreamkast.name, + s3BucketRegion=const.s3.dreamkast.region, + + sqsFifoQueueName=const.sqs.fifo, + + sentryDsn=const.sentry.dsn, + + railsAppSecretManagerName=const.secretManager.railsApp, + rdsSecretManagerName=const.secretManager.rds, + dreamkastSecretManagerName=const.secretManager.dk, + + enableLogging=true, + reviewapp=true, +) diff --git a/ecspresso/reviewapps/dk-2445/harvestjob/ecschedule.yml b/ecspresso/reviewapps/dk-2445/harvestjob/ecschedule.yml new file mode 100644 index 000000000..6b39bfb57 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/harvestjob/ecschedule.yml @@ -0,0 +1,19 @@ +region: us-west-2 +cluster: dreamkast-dev +role: dreamkast-dev-ecs-scheduled-task-target-role +rules: +- name: dk-2445-harvestjob + scheduleExpression: "cron(*/3 * * * ? *)" + taskDefinition: dreamkast-dev-dk-2445-harvestjob + launch_type: FARGATE + platform_version: LATEST + network_configuration: + aws_vpc_configuration: + subnets: + - 'subnet-00709135a42bf907e' # dreamkast-dev-vpc-public-us-west-2a + - 'subnet-0d07831c8fc073511' # dreamkast-dev-vpc-public-us-west-2b + - 'subnet-033491d41490494b6' # dreamkast-dev-vpc-public-us-west-2c + security_groups: + - sg-05592a72e569c245b # dreamkast-dev-ecs-harvestjob + assign_public_ip: ENABLED + diff --git a/ecspresso/reviewapps/dk-2445/harvestjob/ecspresso.taskdef.yml b/ecspresso/reviewapps/dk-2445/harvestjob/ecspresso.taskdef.yml new file mode 100644 index 000000000..dab13b8eb --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/harvestjob/ecspresso.taskdef.yml @@ -0,0 +1,4 @@ +region: us-west-2 +cluster: dreamkast-dev +task_definition: task-def.jsonnet +timeout: 5m diff --git a/ecspresso/reviewapps/dk-2445/harvestjob/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/harvestjob/task-def.jsonnet new file mode 100644 index 000000000..dc2cda46c --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/harvestjob/task-def.jsonnet @@ -0,0 +1,115 @@ +local const = import '../const.libsonnet'; +local family = 'dreamkast-dev-%s-harvestjob' % [const.PR_NAME]; +local executionRoleName = 'dreamkast-dev-ecs-task-execution-role'; +local roleName = 'dreamkast-dev-ecs-harvestjob'; + +{ + containerDefinitions: [ + { + local container = self, + + name: 'dreamkast', + image: '607167088920.dkr.ecr.%s.amazonaws.com/dreamkast-ecs:%s' % [const.region, const.imageTags.dreamkast_ecs], + essential: true, + entryPoint: [ + '/bin/bash', + '-c', + ], + command: [ + 'bundle exec rake util:polling_harvest_job_and_update_video', + ], + environment: [ + { + name: 'RAILS_ENV', + value: 'production', + }, + { + name: 'MYSQL_HOST', + value: const.internalEndpoints.rdb, + }, + { + name: 'MYSQL_DATABASE', + value: 'dreamkast', + }, + { + name: 'REDIS_URL', + value: const.internalEndpoints.redis, + }, + { + name: 'SENTRY_DSN', + value: const.sentry.dsn, + }, + { + name: 'S3_BUCKET', + value: 'dreamkast-dev-bucket', + }, + { + name: 'S3_REGION', + value: const.region, + }, + { + name: 'SQS_MAIL_QUEUE_URL', + value: 'https://sqs.%s.amazonaws.com/607167088920/%s.fifo' % [const.region, const.sqs.fifo], + }, + { + name: 'DREAMKAST_NAMESPACE', + value: 'dreamkast-dev', + }, + { + name: 'REVIEW_APP', + value: 'true', + }, + ], + secrets: [ + { + name: 'SLACK_WEBHOOK_URL', + valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:SLACK_CHANNEL_FOR_HARVEST_JOB_NOTIFICATION::' % [const.region, const.secretManager.dk], + }, + { + name: 'SLACK_CHANNEL', + valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:SLACK_CHANNEL_FOR_HARVEST_JOB_NOTIFICATION::' % [const.region, const.secretManager.dk], + }, + { + name: 'RAILS_MASTER_KEY', + valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s' % [const.region, const.secretManager.railsApp], + }, + { + name: 'AUTH0_CLIENT_ID', + valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:AUTH0_CLIENT_ID::' % [const.region, const.secretManager.dk], + }, + { + name: 'AUTH0_CLIENT_SECRET', + valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:AUTH0_CLIENT_SECRET::' % [const.region, const.secretManager.dk], + }, + { + name: 'AUTH0_DOMAIN', + valueFrom: 'arn:aws:secretsmanager:%s:607167088920:secret:%s:AUTH0_DOMAIN::' % [const.region, const.secretManager.dk], + }, + ], + + logConfiguration: { + logDriver: 'awslogs', + options: { + 'awslogs-create-group': 'true', + 'awslogs-group': family, + 'awslogs-region': const.region, + 'awslogs-stream-prefix': container.name, + }, + }, + + cpu: 256, + memory: 512, + memoryReservation: 512, + }, + ], + family: family, + cpu: '256', + memory: '512', + executionRoleArn: 'arn:aws:iam::607167088920:role/%s' % [executionRoleName], + taskRoleArn: 'arn:aws:iam::607167088920:role/%s' % [roleName], + networkMode: 'awsvpc', + requiresCompatibilities: [ + 'FARGATE', + ], + volumes: [], +} diff --git a/ecspresso/reviewapps/dk-2445/initialize.sh b/ecspresso/reviewapps/dk-2445/initialize.sh new file mode 100755 index 000000000..bd48677a3 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/initialize.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -e -o pipefail +cd $(dirname $0) + +# required the following commands: +# * aws +# * jq +# * jsonnet +# * jsonnetfmt + +# variables +PR_NAME=${PR_NAME:?"PR_NAME must be specified"} +PR_NUMBER=${PR_NUMBER:?"PR_NUMBER must be specified"} +IMAGE_TAG=${IMAGE_TAG:?"IMAGE_TAG must be specified"} +LISTENER_RULE_PRIORITY_BASE=30000 +LISTENER_RULE_PRIORITY=$(( LISTENER_RULE_PRIORITY_BASE + PR_NUMBER )) + +VPC_ID="vpc-0f0d012967c635f34" +LISTENER_ARN="arn:aws:elasticloadbalancing:us-west-2:607167088920:listener/app/dreamkast-dev/122c5b4a47b64f9d/bc86e7b2e4bca8f5" +SERVICE_DISCOVERY_NAMESPACE="ns-53ijjrlltqf5r2gm" + + +if [ ! -f "cleanup.sh" ]; then +# create TargetGroup +TARGET_GROUP_ARN=$(aws elbv2 create-target-group \ + --name "dev-${PR_NAME}" \ + --target-type ip \ + --protocol HTTP \ + --port 3000 \ + --vpc-id ${VPC_ID} \ + --ip-address-type ipv4 \ + | jq -r ".TargetGroups[0].TargetGroupArn") + +# create ALB ListenerRule +LISTENER_RULE_ARN=$(aws elbv2 create-rule --listener-arn ${LISTENER_ARN} \ + --priority ${LISTENER_RULE_PRIORITY} \ + --conditions Field=host-header,Values="dreamkast-${PR_NAME}.dev.cloudnativedays.jp" \ + --actions Type=forward,TargetGroupArn=${TARGET_GROUP_ARN} \ + | jq -r ".Rules[] | select(.Priority == \"${LISTENER_RULE_PRIORITY}\") | .RuleArn") + +# create ServiceDiscovery Services +SERVICE_ID_MYSQL=$(aws servicediscovery create-service \ + --name "mysql-${PR_NAME}" \ + --dns-config "NamespaceId="${SERVICE_DISCOVERY_NAMESPACE}",DnsRecords=[{Type="A",TTL="10"}]" \ + --health-check-custom-config FailureThreshold=1 \ + | jq -r ".Service.Id") + +SERVICE_ID_REDIS=$(aws servicediscovery create-service \ + --name "redis-${PR_NAME}" \ + --dns-config "NamespaceId="${SERVICE_DISCOVERY_NAMESPACE}",DnsRecords=[{Type="A",TTL="10"}]" \ + --health-check-custom-config FailureThreshold=1 \ + | jq -r ".Service.Id") + +# replace variables in each ecspresso.yml +find . -name ecspresso.yml | xargs -I{} sed -i -e 's/__PR_NAME__/'${PR_NAME}'/g' {} +# replace variables in each ecschedule.yml +find . -name ecschedule.yml | xargs -I{} sed -i -e 's/__PR_NAME__/'${PR_NAME}'/g' {} + +# replace variables in const.libsonnet +cat << _EOL_ | jsonnet - > ./const.libsonnet.tmp +local const = import './const.libsonnet'; + +const + { + PR_NAME: "${PR_NAME}", + targetGroupArn: { + dk: "${TARGET_GROUP_ARN}", + }, + serviceDiscovery: { + mysql: "${SERVICE_ID_MYSQL}", + redis: "${SERVICE_ID_REDIS}", + }, + imageTags: const.imageTags + { + dreamkast_ecs: "${IMAGE_TAG}", + }, +} +_EOL_ +mv const.libsonnet.tmp const.libsonnet + +# create cleanup.sh +cat << _EOF_ > ./cleanup.sh +#!/usr/bin/env bash +set -e -o pipefail +cd \$(dirname \$0) + +find . -name "ecspresso.yml" | xargs -I{} -P10 ecspresso --config={} delete --force --terminate ||: +sleep 10 # wait for ECS Services to be deleted +aws events delete-rule --name ${PR_NAME}-harvestjob +aws ecs deregister-task-definition --task-definition dreamkast-dev-${PR_NAME}-harvestjob +aws servicediscovery get-service --id ${SERVICE_ID_MYSQL} &>/dev/null && aws servicediscovery delete-service --id ${SERVICE_ID_MYSQL} +aws servicediscovery get-service --id ${SERVICE_ID_REDIS} &>/dev/null && aws servicediscovery delete-service --id ${SERVICE_ID_REDIS} +aws elbv2 describe-rules --rule-arn ${LISTENER_RULE_ARN} &>/dev/null && aws elbv2 delete-rule --rule-arn ${LISTENER_RULE_ARN} +aws elbv2 describe-target-groups --target-group-arn ${TARGET_GROUP_ARN} &>/dev/null && aws elbv2 delete-target-group --target-group-arn ${TARGET_GROUP_ARN} +: +_EOF_ + +else +# update imageTags +jsonnet const.libsonnet \ + | jq ".imageTags.dreamkast_ecs|=\"${IMAGE_TAG}\"" \ + > const.libsonnet.tmp +mv const.libsonnet.tmp const.libsonnet + +fi + +jsonnetfmt -i const.libsonnet diff --git a/ecspresso/reviewapps/dk-2445/mysql/ecspresso.yml b/ecspresso/reviewapps/dk-2445/mysql/ecspresso.yml new file mode 100644 index 000000000..6a568bffa --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/mysql/ecspresso.yml @@ -0,0 +1,6 @@ +region: us-west-2 +cluster: dreamkast-dev +service: dk-2445-mysql +service_definition: service-def.jsonnet +task_definition: task-def.jsonnet +timeout: 10m diff --git a/ecspresso/reviewapps/dk-2445/mysql/service-def.jsonnet b/ecspresso/reviewapps/dk-2445/mysql/service-def.jsonnet new file mode 100644 index 000000000..91214d765 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/mysql/service-def.jsonnet @@ -0,0 +1,9 @@ +local mysql = import '../../../base/mysql.libsonnet'; +local const = import '../const.libsonnet'; + +mysql.serviceDef( + region=const.region, + subnetIDs=const.publicSubnetIDs, + securityGroupID='sg-0e0029eb49f4d0455', // dreamkast-dev-ecs-mysql + serviceDiscoveryID=const.serviceDiscovery.mysql, +) diff --git a/ecspresso/reviewapps/dk-2445/mysql/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/mysql/task-def.jsonnet new file mode 100644 index 000000000..e8a42b6ab --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/mysql/task-def.jsonnet @@ -0,0 +1,12 @@ +local mysql = import '../../../base/mysql.libsonnet'; +local const = import '../const.libsonnet'; + +mysql.taskDef( + family='dreamkast-dev-%s-mysql' % [const.PR_NAME], + taskRoleName='dreamkast-dev-ecs-mysql', + imageTag=const.imageTags.mysql, + + region=const.region, + + enableLogging=false, +) diff --git a/ecspresso/reviewapps/dk-2445/redis/ecspresso.yml b/ecspresso/reviewapps/dk-2445/redis/ecspresso.yml new file mode 100644 index 000000000..29debc985 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/redis/ecspresso.yml @@ -0,0 +1,6 @@ +region: us-west-2 +cluster: dreamkast-dev +service: dk-2445-redis +service_definition: service-def.jsonnet +task_definition: task-def.jsonnet +timeout: 10m diff --git a/ecspresso/reviewapps/dk-2445/redis/service-def.jsonnet b/ecspresso/reviewapps/dk-2445/redis/service-def.jsonnet new file mode 100644 index 000000000..d70462f23 --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/redis/service-def.jsonnet @@ -0,0 +1,9 @@ +local redis = import '../../../base/redis.libsonnet'; +local const = import '../const.libsonnet'; + +redis.serviceDef( + region=const.region, + subnetIDs=const.publicSubnetIDs, + securityGroupID='sg-0ab649652e2dd6c9c', // dreamkast-dev-ecs-redis + serviceDiscoveryID=const.serviceDiscovery.redis, +) diff --git a/ecspresso/reviewapps/dk-2445/redis/task-def.jsonnet b/ecspresso/reviewapps/dk-2445/redis/task-def.jsonnet new file mode 100644 index 000000000..e2ed63cba --- /dev/null +++ b/ecspresso/reviewapps/dk-2445/redis/task-def.jsonnet @@ -0,0 +1,12 @@ +local redis = import '../../../base/redis.libsonnet'; +local const = import '../const.libsonnet'; + +redis.taskDef( + family='dreamkast-dev-%s-redis' % [const.PR_NAME], + taskRoleName='dreamkast-dev-ecs-redis', + imageTag=const.imageTags.redis, + + region=const.region, + + enableLogging=false, +)