From fac93bb18a0d4103877d281fbe1939c5733dd6f3 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 2 Sep 2024 00:54:33 +0200 Subject: [PATCH 01/46] Move off lambda and to ECS --- infrastructure/applications/applications.tf | 5 ++ .../applications/pycon_backend/main.tf | 21 +++-- .../applications/pycon_backend/server.tf | 83 ++++++++++++++++++ infrastructure/applications/server/db.tf | 7 ++ infrastructure/applications/server/dns.tf | 0 infrastructure/applications/server/ecs.tf | 3 + infrastructure/applications/server/emails.tf | 3 + .../applications/server/env_vars.tf | 3 + infrastructure/applications/server/main.tf | 61 +++++++++++++ infrastructure/applications/server/role.tf | 45 ++++++++++ infrastructure/applications/server/secrets.tf | 3 + .../applications/server/security.tf | 40 +++++++++ .../applications/server/server_user_data.sh | 23 +++++ .../applications/server/task_traefik.tf | 87 +++++++++++++++++++ .../applications/server/variables.tf | 1 + infrastructure/applications/server/vpc.tf | 20 +++++ infrastructure/components/cloudfront/main.tf | 18 ++-- .../components/cloudfront/variables.tf | 2 +- infrastructure/global/ecr_repos/main.tf | 4 - 19 files changed, 412 insertions(+), 17 deletions(-) create mode 100644 infrastructure/applications/pycon_backend/server.tf create mode 100644 infrastructure/applications/server/db.tf create mode 100644 infrastructure/applications/server/dns.tf create mode 100644 infrastructure/applications/server/ecs.tf create mode 100644 infrastructure/applications/server/emails.tf create mode 100644 infrastructure/applications/server/env_vars.tf create mode 100644 infrastructure/applications/server/main.tf create mode 100644 infrastructure/applications/server/role.tf create mode 100644 infrastructure/applications/server/secrets.tf create mode 100644 infrastructure/applications/server/security.tf create mode 100644 infrastructure/applications/server/server_user_data.sh create mode 100644 infrastructure/applications/server/task_traefik.tf create mode 100644 infrastructure/applications/server/variables.tf create mode 100644 infrastructure/applications/server/vpc.tf diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf index 56d4288f43..d58b6e4540 100644 --- a/infrastructure/applications/applications.tf +++ b/infrastructure/applications/applications.tf @@ -50,3 +50,8 @@ module "emails" { aws.us = aws.us } } + +module "server" { + source = "./server" + ecs_arm_ami = local.ecs_arm_ami +} diff --git a/infrastructure/applications/pycon_backend/main.tf b/infrastructure/applications/pycon_backend/main.tf index 60fbb125bc..c9d2359c2f 100644 --- a/infrastructure/applications/pycon_backend/main.tf +++ b/infrastructure/applications/pycon_backend/main.tf @@ -1,7 +1,6 @@ locals { is_prod = terraform.workspace == "production" - admin_domain = "admin" - full_admin_domain = local.is_prod ? "${local.admin_domain}.pycon.it" : "${terraform.workspace}-${local.admin_domain}.pycon.it" + admin_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" db_connection = var.enable_proxy ? "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_proxy.proxy[0].endpoint}:${data.aws_db_instance.database.port}/pycon" : "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" cdn_url = local.is_prod ? "cdn.pycon.it" : "${terraform.workspace}-cdn.pycon.it" } @@ -126,13 +125,25 @@ module "lambda" { } } +data "aws_instance" "server" { + instance_tags = { + Name = "${terraform.workspace}-server" + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + module "admin_distribution" { source = "../../components/cloudfront" application = local.application zone_name = "pycon.it" - domain = local.full_admin_domain + domain = local.admin_domain certificate_arn = data.aws_acm_certificate.cert.arn - origin_url = module.lambda.cloudfront_friendly_lambda_url - forward_host_header_lambda_arn = data.aws_lambda_function.forward_host_header.qualified_arn + # origin_url = module.lambda.cloudfront_friendly_lambda_url + origin_url = data.aws_instance.server.public_dns + # forward_host_header_lambda_arn = data.aws_lambda_function.forward_host_header.qualified_arn } diff --git a/infrastructure/applications/pycon_backend/server.tf b/infrastructure/applications/pycon_backend/server.tf new file mode 100644 index 0000000000..a962443c7c --- /dev/null +++ b/infrastructure/applications/pycon_backend/server.tf @@ -0,0 +1,83 @@ +data "aws_ecs_cluster" "server" { + cluster_name = "${terraform.workspace}-server" +} + +resource "aws_cloudwatch_log_group" "backend" { + name = "/ecs/pythonit-${terraform.workspace}-backend" + retention_in_days = 3 +} + +resource "aws_ecs_task_definition" "backend" { + family = "pythonit-${terraform.workspace}-backend" + container_definitions = jsonencode([ + { + name = "backend" + image = "${data.aws_ecr_repository.be_repo.repository_url}@${data.aws_ecr_image.be_arm_image.image_digest}" + memoryReservation = 200 + essential = true + + entrypoint = [ + "/home/app/.venv/bin/uwsgi", + ] + + command = [ + "--http", ":8000", "--module", "pycon.wsgi:application" + ] + + dockerLabels = { + "traefik.enable" = "true" + "traefik.http.routers.backend.rule" = "Host(`${local.admin_domain}`)" + } + + environment = local.env_vars + portMappings = [ + { + containerPort = 8000 + hostPort = 0 + }, + ] + + mountPoints = [] + systemControls = [ + { + "namespace" : "net.core.somaxconn", + "value" : "4096" + } + ] + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.backend.name + "awslogs-region" = "eu-central-1" + "awslogs-stream-prefix" = "ecs" + } + } + + healthCheck = { + retries = 3 + command = [ + "CMD-SHELL", + "echo 1" + ] + timeout = 3 + interval = 10 + } + + stopTimeout = 300 + } + ]) + + requires_compatibilities = [] + tags = {} +} + + +resource "aws_ecs_service" "backend" { + name = "pythonit-${terraform.workspace}-backend" + cluster = data.aws_ecs_cluster.server.id + task_definition = aws_ecs_task_definition.backend.arn + desired_count = 1 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 +} diff --git a/infrastructure/applications/server/db.tf b/infrastructure/applications/server/db.tf new file mode 100644 index 0000000000..654129144f --- /dev/null +++ b/infrastructure/applications/server/db.tf @@ -0,0 +1,7 @@ +locals { + db_connection = "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" +} + +data "aws_db_instance" "database" { + db_instance_identifier = "pythonit-${terraform.workspace}" +} diff --git a/infrastructure/applications/server/dns.tf b/infrastructure/applications/server/dns.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/infrastructure/applications/server/ecs.tf b/infrastructure/applications/server/ecs.tf new file mode 100644 index 0000000000..996e935cdf --- /dev/null +++ b/infrastructure/applications/server/ecs.tf @@ -0,0 +1,3 @@ +resource "aws_ecs_cluster" "server" { + name = "${terraform.workspace}-server" +} diff --git a/infrastructure/applications/server/emails.tf b/infrastructure/applications/server/emails.tf new file mode 100644 index 0000000000..d2b01058aa --- /dev/null +++ b/infrastructure/applications/server/emails.tf @@ -0,0 +1,3 @@ +data "aws_sesv2_configuration_set" "main" { + configuration_set_name = "pythonit-${terraform.workspace}" +} diff --git a/infrastructure/applications/server/env_vars.tf b/infrastructure/applications/server/env_vars.tf new file mode 100644 index 0000000000..e4a72a1ef0 --- /dev/null +++ b/infrastructure/applications/server/env_vars.tf @@ -0,0 +1,3 @@ +locals { + is_prod = terraform.workspace == "prod" +} diff --git a/infrastructure/applications/server/main.tf b/infrastructure/applications/server/main.tf new file mode 100644 index 0000000000..cb4b72b385 --- /dev/null +++ b/infrastructure/applications/server/main.tf @@ -0,0 +1,61 @@ +data "template_file" "server_user_data" { + template = file("${path.module}/server_user_data.sh") + vars = { + ecs_cluster = aws_ecs_cluster.server.name + } +} + +data "aws_security_group" "tempone" { + name = "pythonit-${terraform.workspace}-worker-instance" +} + +resource "aws_launch_template" "server" { + name = "pythonit-${terraform.workspace}-server" + image_id = var.ecs_arm_ami + instance_type = "t4g.medium" + user_data = base64encode(data.template_file.server_user_data.rendered) + key_name = "pretix" + + iam_instance_profile { + name = aws_iam_instance_profile.server.name + } + + network_interfaces { + associate_public_ip_address = true + security_groups = [ + data.aws_security_group.rds.id, + data.aws_security_group.lambda.id, + data.aws_security_group.tempone.id, + aws_security_group.server.id, + ] + subnet_id = data.aws_subnet.public_1a.id + } +} + +resource "aws_autoscaling_group" "server" { + name = "pythonit-${terraform.workspace}-server" + vpc_zone_identifier = [data.aws_subnet.public_1a.id] + desired_capacity = 1 + max_size = 1 + min_size = 1 + termination_policies = ["OldestInstance"] + + instance_refresh { + strategy = "Rolling" + preferences { + min_healthy_percentage = 100 + max_healthy_percentage = 110 + } + } + + launch_template { + id = aws_launch_template.server.id + version = aws_launch_template.server.latest_version + } + + tag { + key = "Name" + value = "${terraform.workspace}-server" + propagate_at_launch = true + } +} diff --git a/infrastructure/applications/server/role.tf b/infrastructure/applications/server/role.tf new file mode 100644 index 0000000000..62a3d3f51e --- /dev/null +++ b/infrastructure/applications/server/role.tf @@ -0,0 +1,45 @@ +data "aws_iam_policy_document" "server_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com", "ecs-tasks.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +data "aws_iam_policy_document" "server_role_policy" { + statement { + effect = "Allow" + actions = ["ecs:*", "ecr:*", "ec2:DescribeInstances"] + resources = [ + "*" + ] + } + + statement { + effect = "Allow" + actions = ["cloudwatch:PutMetricData", "logs:*"] + resources = ["*"] + } +} + + +resource "aws_iam_role" "server" { + name = "pythonit-${terraform.workspace}-server" + assume_role_policy = data.aws_iam_policy_document.server_assume_role.json +} + +resource "aws_iam_instance_profile" "server" { + name = "pythonit-${terraform.workspace}-server" + role = aws_iam_role.server.name +} + +resource "aws_iam_role_policy" "server" { + name = "pythonit-${terraform.workspace}-server-policy" + role = aws_iam_role.server.id + policy = data.aws_iam_policy_document.server_role_policy.json +} diff --git a/infrastructure/applications/server/secrets.tf b/infrastructure/applications/server/secrets.tf new file mode 100644 index 0000000000..127691dca2 --- /dev/null +++ b/infrastructure/applications/server/secrets.tf @@ -0,0 +1,3 @@ +module "common_secrets" { + source = "../../components/secrets" +} diff --git a/infrastructure/applications/server/security.tf b/infrastructure/applications/server/security.tf new file mode 100644 index 0000000000..ae2fb1b588 --- /dev/null +++ b/infrastructure/applications/server/security.tf @@ -0,0 +1,40 @@ +data "aws_security_group" "lambda" { + name = "pythonit-lambda-security-group" +} + +data "aws_security_group" "rds" { + name = "pythonit-rds-security-group" +} + +resource "aws_security_group" "server" { + name = "${terraform.workspace}-server" + description = "${terraform.workspace} server" + vpc_id = data.aws_vpc.default.id +} + +resource "aws_security_group_rule" "out_all" { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "all" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.server.id +} + +resource "aws_security_group_rule" "web_http" { + type = "ingress" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.server.id +} + +resource "aws_security_group_rule" "web_dashboard" { + type = "ingress" + from_port = 8080 + to_port = 8080 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.server.id +} diff --git a/infrastructure/applications/server/server_user_data.sh b/infrastructure/applications/server/server_user_data.sh new file mode 100644 index 0000000000..65a15dc837 --- /dev/null +++ b/infrastructure/applications/server/server_user_data.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -x + +# Config ECS agent +echo "ECS_CLUSTER=${ecs_cluster}" > /etc/ecs/ecs.config + +# Reclaim unused Docker disk space +cat << "EOF" > /usr/local/bin/claimspace.sh +#!/bin/bash +# Run fstrim on the host OS periodically to reclaim the unused container data blocks +docker ps -q | xargs docker inspect --format='{{ .State.Pid }}' | xargs -IZ sudo fstrim /proc/Z/root/ +exit $? +EOF + +chmod +x /usr/local/bin/claimspace.sh +echo "0 0 * * * root /usr/local/bin/claimspace.sh" > /etc/cron.d/claimspace + +sudo su +sudo dd if=/dev/zero of=/swapfile bs=8GB count=32 +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +sudo echo "/swapfile swap swap defaults 0 0" >> /etc/fstab diff --git a/infrastructure/applications/server/task_traefik.tf b/infrastructure/applications/server/task_traefik.tf new file mode 100644 index 0000000000..a1bdf86f39 --- /dev/null +++ b/infrastructure/applications/server/task_traefik.tf @@ -0,0 +1,87 @@ +resource "aws_cloudwatch_log_group" "traefik" { + name = "/ecs/pythonit-${terraform.workspace}-traefik" + retention_in_days = 3 +} + +resource "aws_ecs_task_definition" "traefik" { + family = "pythonit-${terraform.workspace}-traefik" + container_definitions = jsonencode([ + { + name = "traefik" + image = "traefik:v3.1.2" + memoryReservation = 200 + essential = true + + environment = [ + { + name = "TRAEFIK_PROVIDERS_ECS_CLUSTERS" + value = aws_ecs_cluster.server.name + }, + { + name = "TRAEFIK_PROVIDERS_ECS_AUTODISCOVERCLUSTERS" + value = "false", + }, + { + name = "TRAEFIK_PROVIDERS_ECS_EXPOSEDBYDEFAULT", + value = "false", + }, + { + name = "TRAEFIK_ENTRYPOINTS_WEB_ADDRESS", + value = ":80" + }, + { + name = "TRAEFIK_LOG_LEVEL", + value = "DEBUG" + } + ] + + portMappings = [ + { + containerPort = 80 + hostPort = 80 + }, + ] + + mountPoints = [] + systemControls = [ + { + "namespace" : "net.core.somaxconn", + "value" : "4096" + } + ] + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.traefik.name + "awslogs-region" = "eu-central-1" + "awslogs-stream-prefix" = "ecs" + } + } + + healthCheck = { + retries = 3 + command = [ + "CMD-SHELL", + "echo 4" + ] + timeout = 3 + interval = 10 + } + + stopTimeout = 300 + } + ]) + + requires_compatibilities = [] + tags = {} +} + +resource "aws_ecs_service" "traefik" { + name = "pythonit-${terraform.workspace}-traefik" + cluster = aws_ecs_cluster.server.id + task_definition = aws_ecs_task_definition.traefik.arn + desired_count = 1 + deployment_minimum_healthy_percent = 0 + deployment_maximum_percent = 100 +} diff --git a/infrastructure/applications/server/variables.tf b/infrastructure/applications/server/variables.tf new file mode 100644 index 0000000000..15b58209f2 --- /dev/null +++ b/infrastructure/applications/server/variables.tf @@ -0,0 +1 @@ +variable "ecs_arm_ami" {} diff --git a/infrastructure/applications/server/vpc.tf b/infrastructure/applications/server/vpc.tf new file mode 100644 index 0000000000..7162daf1c0 --- /dev/null +++ b/infrastructure/applications/server/vpc.tf @@ -0,0 +1,20 @@ +data "aws_vpc" "default" { + filter { + name = "tag:Name" + values = ["pythonit-vpc"] + } +} + +data "aws_subnet" "public_1a" { + vpc_id = data.aws_vpc.default.id + + filter { + name = "tag:Type" + values = ["public"] + } + + filter { + name = "tag:AZ" + values = ["eu-central-1a"] + } +} diff --git a/infrastructure/components/cloudfront/main.tf b/infrastructure/components/cloudfront/main.tf index 1aaf319877..340f4d1d97 100644 --- a/infrastructure/components/cloudfront/main.tf +++ b/infrastructure/components/cloudfront/main.tf @@ -6,6 +6,10 @@ data "aws_cloudfront_origin_request_policy" "all_viewer_except_host_header" { name = "Managed-AllViewerExceptHostHeader" } +data "aws_cloudfront_origin_request_policy" "all_viewer" { + name = "Managed-AllViewer" +} + resource "aws_cloudfront_distribution" "application" { enabled = true is_ipv6_enabled = true @@ -18,7 +22,7 @@ resource "aws_cloudfront_distribution" "application" { origin_id = "default" custom_origin_config { - origin_protocol_policy = "https-only" + origin_protocol_policy = "http-only" http_port = "80" https_port = "443" origin_ssl_protocols = ["TLSv1"] @@ -38,16 +42,16 @@ resource "aws_cloudfront_distribution" "application" { target_origin_id = "default" cache_policy_id = data.aws_cloudfront_cache_policy.caching_disabled.id - origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer_except_host_header.id + origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id viewer_protocol_policy = "redirect-to-https" compress = true - lambda_function_association { - event_type = "viewer-request" - lambda_arn = var.forward_host_header_lambda_arn - include_body = false - } + # lambda_function_association { + # event_type = "viewer-request" + # lambda_arn = var.forward_host_header_lambda_arn + # include_body = false + # } } restrictions { diff --git a/infrastructure/components/cloudfront/variables.tf b/infrastructure/components/cloudfront/variables.tf index 2713c97095..9c91434b52 100644 --- a/infrastructure/components/cloudfront/variables.tf +++ b/infrastructure/components/cloudfront/variables.tf @@ -3,4 +3,4 @@ variable "origin_url" {} variable "domain" {} variable "certificate_arn" {} variable "zone_name" {} -variable "forward_host_header_lambda_arn" {} +# variable "forward_host_header_lambda_arn" {} diff --git a/infrastructure/global/ecr_repos/main.tf b/infrastructure/global/ecr_repos/main.tf index 7d890757dd..2277c3c424 100644 --- a/infrastructure/global/ecr_repos/main.tf +++ b/infrastructure/global/ecr_repos/main.tf @@ -1,11 +1,7 @@ locals { services = [ "pycon-backend", - "users-backend", - "gateway", - "association-backend", "pretix", - "cms" ] } From b3483b4239d0c1587ef516f52094f27bbcc7b7ea Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 2 Sep 2024 00:56:20 +0200 Subject: [PATCH 02/46] change test --- backend/custom_admin/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/custom_admin/admin.py b/backend/custom_admin/admin.py index 53eb999a6a..34ac164699 100644 --- a/backend/custom_admin/admin.py +++ b/backend/custom_admin/admin.py @@ -6,7 +6,7 @@ from django.contrib import admin from django.urls import path -SITE_NAME = "PyCon Italia" +SITE_NAME = "Change Test PyCon Italia" admin.site.site_header = SITE_NAME admin.site.site_title = SITE_NAME From 1ecca8a14746f4ef0916de870176b77860a32685 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 2 Sep 2024 01:32:46 +0200 Subject: [PATCH 03/46] test --- infrastructure/applications/server/task_traefik.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/infrastructure/applications/server/task_traefik.tf b/infrastructure/applications/server/task_traefik.tf index a1bdf86f39..cd452afe8a 100644 --- a/infrastructure/applications/server/task_traefik.tf +++ b/infrastructure/applications/server/task_traefik.tf @@ -29,10 +29,6 @@ resource "aws_ecs_task_definition" "traefik" { name = "TRAEFIK_ENTRYPOINTS_WEB_ADDRESS", value = ":80" }, - { - name = "TRAEFIK_LOG_LEVEL", - value = "DEBUG" - } ] portMappings = [ From 60959a2c1ed1abc1a0c6841530c10257df1a542e Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 2 Sep 2024 02:04:47 +0200 Subject: [PATCH 04/46] s --- infrastructure/applications/applications.tf | 4 +-- .../applications/pycon_backend/worker.tf | 4 +-- infrastructure/applications/server/role.tf | 27 ++++++++++++++----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf index d58b6e4540..16fe5592ea 100644 --- a/infrastructure/applications/applications.tf +++ b/infrastructure/applications/applications.tf @@ -52,6 +52,6 @@ module "emails" { } module "server" { - source = "./server" - ecs_arm_ami = local.ecs_arm_ami + source = "./server" + ecs_arm_ami = local.ecs_arm_ami } diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/worker.tf index 718c060f01..b02985718b 100644 --- a/infrastructure/applications/pycon_backend/worker.tf +++ b/infrastructure/applications/pycon_backend/worker.tf @@ -417,7 +417,7 @@ resource "aws_ecs_task_definition" "beat" { resource "aws_ecs_service" "worker" { name = "pythonit-${terraform.workspace}-worker" - cluster = aws_ecs_cluster.worker.id + cluster = data.aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.worker.arn desired_count = 1 deployment_minimum_healthy_percent = 0 @@ -426,7 +426,7 @@ resource "aws_ecs_service" "worker" { resource "aws_ecs_service" "beat" { name = "pythonit-${terraform.workspace}-beat" - cluster = aws_ecs_cluster.worker.id + cluster = data.aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.beat.arn desired_count = 1 deployment_minimum_healthy_percent = 0 diff --git a/infrastructure/applications/server/role.tf b/infrastructure/applications/server/role.tf index 62a3d3f51e..291d983f72 100644 --- a/infrastructure/applications/server/role.tf +++ b/infrastructure/applications/server/role.tf @@ -14,22 +14,37 @@ data "aws_iam_policy_document" "server_assume_role" { data "aws_iam_policy_document" "server_role_policy" { statement { effect = "Allow" - actions = ["ecs:*", "ecr:*", "ec2:DescribeInstances"] + actions = [ + "iam:PassRole", + "ses:*", + "ecs:*", + "ecr:*", + "ec2:DescribeInstances", + ] resources = [ "*" ] } statement { - effect = "Allow" - actions = ["cloudwatch:PutMetricData", "logs:*"] + effect = "Allow" + actions = ["cloudwatch:PutMetricData", "logs:*"] resources = ["*"] } + + statement { + effect = "Allow" + actions = ["s3:*"] + resources = [ + "arn:aws:s3:::${terraform.workspace}-pycon-backend-media", + "arn:aws:s3:::${terraform.workspace}-pycon-backend-media/*", + ] + } } resource "aws_iam_role" "server" { - name = "pythonit-${terraform.workspace}-server" + name = "pythonit-${terraform.workspace}-server" assume_role_policy = data.aws_iam_policy_document.server_assume_role.json } @@ -39,7 +54,7 @@ resource "aws_iam_instance_profile" "server" { } resource "aws_iam_role_policy" "server" { - name = "pythonit-${terraform.workspace}-server-policy" - role = aws_iam_role.server.id + name = "pythonit-${terraform.workspace}-server-policy" + role = aws_iam_role.server.id policy = data.aws_iam_policy_document.server_role_policy.json } From 963c982ee20d13c46f6f4f0ce71792d7d76ce234 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 6 Sep 2024 01:20:27 +0200 Subject: [PATCH 05/46] changes --- infrastructure/applications/applications.tf | 5 + infrastructure/applications/pretix_arm/ecr.tf | 13 ++ infrastructure/applications/pretix_arm/ecs.tf | 141 ++++++++++++++++++ .../{server/dns.tf => pretix_arm/main.tf} | 0 infrastructure/applications/pretix_arm/rds.tf | 3 + .../applications/pretix_arm/secrets.tf | 8 + .../applications/pretix_arm/variables.tf | 1 + .../applications/pycon_backend/main.tf | 73 +++++++-- .../applications/pycon_backend/worker.tf | 8 +- infrastructure/applications/server/ecs.tf | 30 ++++ infrastructure/applications/server/main.tf | 11 +- .../applications/server/task_traefik.tf | 4 +- 12 files changed, 282 insertions(+), 15 deletions(-) create mode 100644 infrastructure/applications/pretix_arm/ecr.tf create mode 100644 infrastructure/applications/pretix_arm/ecs.tf rename infrastructure/applications/{server/dns.tf => pretix_arm/main.tf} (100%) create mode 100644 infrastructure/applications/pretix_arm/rds.tf create mode 100644 infrastructure/applications/pretix_arm/secrets.tf create mode 100644 infrastructure/applications/pretix_arm/variables.tf diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf index 16fe5592ea..d28f390fb7 100644 --- a/infrastructure/applications/applications.tf +++ b/infrastructure/applications/applications.tf @@ -55,3 +55,8 @@ module "server" { source = "./server" ecs_arm_ami = local.ecs_arm_ami } + +module "pretix_arm" { + source = "./pretix_arm" + ecs_arm_ami = local.ecs_arm_ami +} diff --git a/infrastructure/applications/pretix_arm/ecr.tf b/infrastructure/applications/pretix_arm/ecr.tf new file mode 100644 index 0000000000..71dfb57c58 --- /dev/null +++ b/infrastructure/applications/pretix_arm/ecr.tf @@ -0,0 +1,13 @@ +data "aws_ecr_repository" "repo" { + name = "pythonit/pretix" +} + +data "aws_ecr_image" "image" { + repository_name = data.aws_ecr_repository.repo.name + image_tag = "arm-${data.external.githash.result.githash}" +} + +data "external" "githash" { + program = ["python", abspath("${path.module}/githash.py")] + working_dir = abspath("${path.root}/../../pretix") +} diff --git a/infrastructure/applications/pretix_arm/ecs.tf b/infrastructure/applications/pretix_arm/ecs.tf new file mode 100644 index 0000000000..0f26cfdb43 --- /dev/null +++ b/infrastructure/applications/pretix_arm/ecs.tf @@ -0,0 +1,141 @@ +data "aws_ecs_cluster" "server" { + cluster_name = "${terraform.workspace}-server" +} + +data "aws_instance" "redis" { + instance_tags = { + Name = "pythonit-production-redis" + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +resource "aws_cloudwatch_log_group" "pretix" { + name = "/ecs/pythonit-${terraform.workspace}-pretix-arm" + retention_in_days = 7 +} + +resource "aws_ecs_task_definition" "pretix_service" { + family = "pythonit-${terraform.workspace}-pretix" + container_definitions = jsonencode([ + { + name = "pretix" + image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" + memoryReservation = 200 + essential = true + environment = [ + { + name = "DATABASE_NAME" + value = "pretix" + }, + { + name = "DATABASE_USERNAME" + value = data.aws_db_instance.database.master_username + }, + { + name = "DATABASE_PASSWORD" + value = module.common_secrets.value.database_password + }, + { + name = "DATABASE_HOST" + value = data.aws_db_instance.database.address + }, + { + name = "MAIL_USER" + value = module.secrets.value.mail_user + }, + { + name = "MAIL_PASSWORD" + value = module.secrets.value.mail_password + }, + { + name = "PRETIX_SENTRY_DSN" + value = module.secrets.value.sentry_dsn + }, + { + name = "SECRET_KEY" + value = module.secrets.value.secret_key + }, + { + name = "PRETIX_REDIS_LOCATION", + value = "redis://${data.aws_instance.redis.private_ip}/0" + }, + { + name = "PRETIX_REDIS_SESSIONS", + value = "false" + }, + { + name = "PRETIX_CELERY_BROKER", + value = "redis://${data.aws_instance.redis.private_ip}/1" + }, + { + name = "PRETIX_CELERY_BACKEND", + value = "redis://${data.aws_instance.redis.private_ip}/2" + }, + { + name = "PRETIX_PRETIX_URL", + value = "https://tickets.pycon.it/" + }, + { + name = "PRETIX_PRETIX_TRUST_X_FORWARDED_PROTO", + value = "true" + } + ] + portMappings = [ + { + containerPort = 80 + hostPort = 0 + } + ] + mountPoints = [ + { + sourceVolume = "media" + containerPath = "/data/media" + }, + { + sourceVolume = "data" + containerPath = "/var/pretix-data" + } + ] + systemControls = [ + { + "namespace" : "net.core.somaxconn", + "value" : "4096" + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.pretix.name + "awslogs-region" = "eu-central-1" + "awslogs-stream-prefix" = "ecs" + } + } + }, + ]) + + volume { + name = "media" + host_path = "/var/pretix/data/media" + } + + volume { + name = "data" + host_path = "/var/pretix-data" + } + + requires_compatibilities = [] + tags = {} +} + +resource "aws_ecs_service" "pretix" { + name = "pythonit-${terraform.workspace}-pretix" + cluster = data.aws_ecs_cluster.server.id + task_definition = aws_ecs_task_definition.pretix_service.arn + desired_count = 1 + deployment_minimum_healthy_percent = 0 + deployment_maximum_percent = 100 +} diff --git a/infrastructure/applications/server/dns.tf b/infrastructure/applications/pretix_arm/main.tf similarity index 100% rename from infrastructure/applications/server/dns.tf rename to infrastructure/applications/pretix_arm/main.tf diff --git a/infrastructure/applications/pretix_arm/rds.tf b/infrastructure/applications/pretix_arm/rds.tf new file mode 100644 index 0000000000..41dd040f4e --- /dev/null +++ b/infrastructure/applications/pretix_arm/rds.tf @@ -0,0 +1,3 @@ +data "aws_db_instance" "database" { + db_instance_identifier = "pythonit-${terraform.workspace}" +} diff --git a/infrastructure/applications/pretix_arm/secrets.tf b/infrastructure/applications/pretix_arm/secrets.tf new file mode 100644 index 0000000000..b18cdce2fc --- /dev/null +++ b/infrastructure/applications/pretix_arm/secrets.tf @@ -0,0 +1,8 @@ +module "secrets" { + source = "../../components/secrets" + service = "pretix" +} + +module "common_secrets" { + source = "../../components/secrets" +} diff --git a/infrastructure/applications/pretix_arm/variables.tf b/infrastructure/applications/pretix_arm/variables.tf new file mode 100644 index 0000000000..15b58209f2 --- /dev/null +++ b/infrastructure/applications/pretix_arm/variables.tf @@ -0,0 +1 @@ +variable "ecs_arm_ami" {} diff --git a/infrastructure/applications/pycon_backend/main.tf b/infrastructure/applications/pycon_backend/main.tf index c9d2359c2f..cbc8f2e378 100644 --- a/infrastructure/applications/pycon_backend/main.tf +++ b/infrastructure/applications/pycon_backend/main.tf @@ -127,7 +127,7 @@ module "lambda" { data "aws_instance" "server" { instance_tags = { - Name = "${terraform.workspace}-server" + Name = "pythonit-${terraform.workspace}-server" } filter { @@ -136,14 +136,67 @@ data "aws_instance" "server" { } } -module "admin_distribution" { - source = "../../components/cloudfront" +data "aws_cloudfront_origin_request_policy" "all_viewer" { + name = "Managed-AllViewer" +} + +data "aws_cloudfront_cache_policy" "caching_disabled" { + name = "Managed-CachingDisabled" +} + +resource "aws_cloudfront_distribution" "application" { + enabled = true + is_ipv6_enabled = true + comment = "${terraform.workspace}-${local.application}" + wait_for_deployment = false + aliases = [local.admin_domain] + + origin { + domain_name = data.aws_instance.server.public_dns + origin_id = "default" + + custom_origin_config { + origin_protocol_policy = "http-only" + http_port = "80" + https_port = "443" + origin_ssl_protocols = ["TLSv1"] + } + } + + viewer_certificate { + cloudfront_default_certificate = false + minimum_protocol_version = "TLSv1" + ssl_support_method = "sni-only" + acm_certificate_arn = data.aws_acm_certificate.cert.arn + } + + default_cache_behavior { + allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] + cached_methods = ["GET", "HEAD"] + target_origin_id = "default" - application = local.application - zone_name = "pycon.it" - domain = local.admin_domain - certificate_arn = data.aws_acm_certificate.cert.arn - # origin_url = module.lambda.cloudfront_friendly_lambda_url - origin_url = data.aws_instance.server.public_dns - # forward_host_header_lambda_arn = data.aws_lambda_function.forward_host_header.qualified_arn + cache_policy_id = data.aws_cloudfront_cache_policy.caching_disabled.id + origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id + + viewer_protocol_policy = "redirect-to-https" + compress = true + } + + restrictions { + geo_restriction { + restriction_type = "none" + } + } +} + +resource "aws_route53_record" "record" { + zone_id = data.aws_route53_zone.pycon_zone.zone_id + name = local.admin_domain + type = "A" + + alias { + name = aws_cloudfront_distribution.application.domain_name + zone_id = aws_cloudfront_distribution.application.hosted_zone_id + evaluate_target_health = false + } } diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/worker.tf index b02985718b..540273aeb2 100644 --- a/infrastructure/applications/pycon_backend/worker.tf +++ b/infrastructure/applications/pycon_backend/worker.tf @@ -198,6 +198,10 @@ locals { { name = "AWS_SES_CONFIGURATION_SET" value = data.aws_sesv2_configuration_set.main.configuration_set_name + }, + { + name = "SNS_WEBHOOK_SECRET" + value = module.common_secrets.value.sns_webhook_secret } ] } @@ -420,8 +424,8 @@ resource "aws_ecs_service" "worker" { cluster = data.aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.worker.arn desired_count = 1 - deployment_minimum_healthy_percent = 0 - deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 } resource "aws_ecs_service" "beat" { diff --git a/infrastructure/applications/server/ecs.tf b/infrastructure/applications/server/ecs.tf index 996e935cdf..8ee260eaab 100644 --- a/infrastructure/applications/server/ecs.tf +++ b/infrastructure/applications/server/ecs.tf @@ -1,3 +1,33 @@ resource "aws_ecs_cluster" "server" { name = "${terraform.workspace}-server" } + +resource "aws_ecs_capacity_provider" "server" { + name = "pythonit-${terraform.workspace}-server" + + auto_scaling_group_provider { + auto_scaling_group_arn = aws_autoscaling_group.server.arn + managed_termination_protection = "ENABLED" + + managed_scaling { + maximum_scaling_step_size = 2 + minimum_scaling_step_size = 1 + status = "ENABLED" + target_capacity = 1 + instance_warmup_period = 60 + } + } +} + +resource "aws_ecs_cluster_capacity_providers" "server" { + cluster_name = aws_ecs_cluster.server.name + capacity_providers = [ + aws_ecs_capacity_provider.server.name, + ] + + default_capacity_provider_strategy { + base = 1 + weight = 100 + capacity_provider = aws_ecs_capacity_provider.server.name + } +} diff --git a/infrastructure/applications/server/main.tf b/infrastructure/applications/server/main.tf index cb4b72b385..0ce8cf4d57 100644 --- a/infrastructure/applications/server/main.tf +++ b/infrastructure/applications/server/main.tf @@ -39,12 +39,15 @@ resource "aws_autoscaling_group" "server" { max_size = 1 min_size = 1 termination_policies = ["OldestInstance"] + protect_from_scale_in = true instance_refresh { strategy = "Rolling" preferences { min_healthy_percentage = 100 max_healthy_percentage = 110 + scale_in_protected_instances = "Refresh" + instance_warmup = 30 } } @@ -55,7 +58,13 @@ resource "aws_autoscaling_group" "server" { tag { key = "Name" - value = "${terraform.workspace}-server" + value = "pythonit-${terraform.workspace}-server" + propagate_at_launch = true + } + + tag { + key = "AmazonECSManaged" + value = true propagate_at_launch = true } } diff --git a/infrastructure/applications/server/task_traefik.tf b/infrastructure/applications/server/task_traefik.tf index cd452afe8a..2e46911752 100644 --- a/infrastructure/applications/server/task_traefik.tf +++ b/infrastructure/applications/server/task_traefik.tf @@ -78,6 +78,6 @@ resource "aws_ecs_service" "traefik" { cluster = aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.traefik.arn desired_count = 1 - deployment_minimum_healthy_percent = 0 - deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 } From 11702389c004c99df3d2fbdf5a2ae6837a8be049 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 6 Sep 2024 01:24:27 +0200 Subject: [PATCH 06/46] test --- .github/workflows/build-images.yml | 156 +++++++++++++++++++++++++++++ .github/workflows/deploy.yml | 154 +--------------------------- 2 files changed, 159 insertions(+), 151 deletions(-) create mode 100644 .github/workflows/build-images.yml diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 0000000000..3de3d5bee6 --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,156 @@ +name: Build images + +on: + workflow_call: + +jobs: + build-and-push-service: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + + strategy: + fail-fast: false + matrix: + service: + - name: pycon-backend + dir: backend + - name: pretix + dir: pretix + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi + - name: Set up QEMU dependency + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-qemu-action@v3 + - name: Login to GitHub Packages + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Amazon ECR + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.service.name }} + - name: Build and push + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + with: + context: ./${{ matrix.service.dir }} + file: ./${{ matrix.service.dir }}/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} + ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/amd64 + + build-and-push-arm-service: + runs-on: [self-hosted] + permissions: + packages: write + contents: read + strategy: + fail-fast: false + matrix: + service: + - name: pycon-backend + dir: backend + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=arm-${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi + - name: Set up QEMU dependency + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-qemu-action@v3 + - name: Login to GitHub Packages + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Amazon ECR + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-buildx-action@v3 + - name: Build and push + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + with: + context: ./${{ matrix.service.dir }} + file: ./${{ matrix.service.dir }}/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} + ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 400a8df807..4dce53c42e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,156 +50,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-central-1 - build-and-push-service: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - - strategy: - fail-fast: false - matrix: - service: - - name: pycon-backend - dir: backend - - name: pretix - dir: pretix - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-qemu-action@v3 - - name: Login to GitHub Packages - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-buildx-action@v3 - - name: Cache Docker layers - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.service.name }} - - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/build-push-action@v6 - with: - context: ./${{ matrix.service.dir }} - file: ./${{ matrix.service.dir }}/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} - ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/amd64 - - build-and-push-arm-service: - runs-on: [self-hosted] - permissions: - packages: write - contents: read - strategy: - fail-fast: false - matrix: - service: - - name: pycon-backend - dir: backend - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=arm-${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-qemu-action@v3 - - name: Login to GitHub Packages - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-buildx-action@v3 - - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/build-push-action@v6 - with: - context: ./${{ matrix.service.dir }} - file: ./${{ matrix.service.dir }}/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} - ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 + build-images: + uses: ./.github/workflows/build-images.yml build-emails: runs-on: ubuntu-latest @@ -226,7 +78,7 @@ jobs: terraform: runs-on: ubuntu-latest - needs: [build-emails, build-and-push-service, build-and-push-arm-service, create-db] + needs: [build-emails, build-images, create-db] environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: From 3760d50eed0bf6a398d3763c6faef2c5d9058aaa Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 6 Sep 2024 02:10:03 +0200 Subject: [PATCH 07/46] changes --- .github/workflows/deploy.yml | 154 ++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4dce53c42e..400a8df807 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,8 +50,156 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-central-1 - build-images: - uses: ./.github/workflows/build-images.yml + build-and-push-service: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + + strategy: + fail-fast: false + matrix: + service: + - name: pycon-backend + dir: backend + - name: pretix + dir: pretix + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi + - name: Set up QEMU dependency + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-qemu-action@v3 + - name: Login to GitHub Packages + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Amazon ECR + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.service.name }} + - name: Build and push + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + with: + context: ./${{ matrix.service.dir }} + file: ./${{ matrix.service.dir }}/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} + ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/amd64 + + build-and-push-arm-service: + runs-on: [self-hosted] + permissions: + packages: write + contents: read + strategy: + fail-fast: false + matrix: + service: + - name: pycon-backend + dir: backend + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=arm-${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi + - name: Set up QEMU dependency + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-qemu-action@v3 + - name: Login to GitHub Packages + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Amazon ECR + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-buildx-action@v3 + - name: Build and push + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + with: + context: ./${{ matrix.service.dir }} + file: ./${{ matrix.service.dir }}/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} + ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 build-emails: runs-on: ubuntu-latest @@ -78,7 +226,7 @@ jobs: terraform: runs-on: ubuntu-latest - needs: [build-emails, build-images, create-db] + needs: [build-emails, build-and-push-service, build-and-push-arm-service, create-db] environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: From bc76668aec842113986da1e48a7d0edceda24e1f Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 6 Sep 2024 02:11:28 +0200 Subject: [PATCH 08/46] pretix arm --- .github/workflows/deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 400a8df807..a665012412 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -141,6 +141,8 @@ jobs: service: - name: pycon-backend dir: backend + - name: pretix + dir: pretix steps: - uses: actions/checkout@v4 From b548a1f58b8303dcf47f3870bf53a0fd40cb5bac Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Fri, 6 Sep 2024 14:54:10 +0200 Subject: [PATCH 09/46] changes --- .github/workflows/deploy.yml | 82 +---------- .../applications/.terraform.lock.hcl | 34 ++--- infrastructure/applications/config.tf | 2 +- .../applications/pretix_arm/githash.py | 12 ++ .../pretix_arm/{ecs.tf => task_web.tf} | 93 ++++++++----- .../applications/pretix_arm/task_worker.tf | 131 ++++++++++++++++++ .../applications/pycon_backend/providers.tf | 2 +- .../applications/pycon_backend/server.tf | 8 +- .../applications/pycon_backend/worker.tf | 16 ++- .../applications/server/task_traefik.tf | 8 +- 10 files changed, 250 insertions(+), 138 deletions(-) create mode 100644 infrastructure/applications/pretix_arm/githash.py rename infrastructure/applications/pretix_arm/{ecs.tf => task_web.tf} (80%) create mode 100644 infrastructure/applications/pretix_arm/task_worker.tf diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a665012412..360065d993 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,86 +50,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-central-1 - build-and-push-service: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - - strategy: - fail-fast: false - matrix: - service: - - name: pycon-backend - dir: backend - - name: pretix - dir: pretix - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-qemu-action@v3 - - name: Login to GitHub Packages - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-buildx-action@v3 - - name: Cache Docker layers - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.service.name }} - - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/build-push-action@v6 - with: - context: ./${{ matrix.service.dir }} - file: ./${{ matrix.service.dir }}/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} - ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/amd64 - build-and-push-arm-service: runs-on: [self-hosted] permissions: @@ -228,7 +148,7 @@ jobs: terraform: runs-on: ubuntu-latest - needs: [build-emails, build-and-push-service, build-and-push-arm-service, create-db] + needs: [build-emails, build-and-push-arm-service, create-db] environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: diff --git a/infrastructure/applications/.terraform.lock.hcl b/infrastructure/applications/.terraform.lock.hcl index 7fadab67ea..22194db8c0 100644 --- a/infrastructure/applications/.terraform.lock.hcl +++ b/infrastructure/applications/.terraform.lock.hcl @@ -2,25 +2,25 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.64.0" - constraints = "5.64.0" + version = "5.66.0" + constraints = "5.66.0" hashes = [ - "h1:Xasb457vfMG/1SGu6KSApCzAqUHMlsL028OQu3dZVv8=", - "zh:1d361f8062c68c9d5ac14b0aa8390709542129b8a9b258e61bbbabc706078b44", - "zh:39dcbf53e3896bdd77071384c8fad4a5862c222c73f3bcf356aca488101f22fd", - "zh:3fad63505f0c5b6f01cc9a6ef02b2226983b79424126a9caf6eb724f654299f4", - "zh:53a8b90d00829cc27e3171a13a8ff1404ee0ea018e73f31d3f916d246cc39613", - "zh:5734c25ef5a04b40f3c1ac5f817f11e42ee3328f74dbc141c0e64afbb0acc834", - "zh:66ea14dbd87f291ce4a877123363933d3ca4022f209f885807a6689c22c24e80", - "zh:68e79654ad0894a3d93134c3377748ace3058d5fad5ec09d1e9a8f8f9b8a47ea", - "zh:7b74259d0ceef0c49cea6bcd171df997b6bad141085bbadded15b440faeb0eee", - "zh:988ebfb5d115dc57070b5abf2e4200ad49cde535f27fd2ba5e34cf9ab336a57f", + "h1:RHs4rOiKrKJqr8UhVW7yqfoMVwaofQ+9ChP41rAzc1A=", + "zh:071c908eb18627f4becdaf0a9fe95d7a61f69be365080aba2ef5e24f6314392b", + "zh:3dea2a474c6ad4be5b508de4e90064ec485e3fbcebb264cb6c4dec660e3ea8b5", + "zh:56c0b81e3bbf4e9ccb2efb984f8758e2bc563ce179ff3aecc1145df268b046d1", + "zh:5f34b75a9ef69cad8c79115ecc0697427d7f673143b81a28c3cf8d5decfd7f93", + "zh:65632bc2c408775ee44cb32a72e7c48376001a9a7b3adbc2c9b4d088a7d58650", + "zh:6d0550459941dfb39582fadd20bfad8816255a827bfaafb932d51d66030fcdd5", + "zh:7f1811ef179e507fdcc9776eb8dc3d650339f8b84dd084642cf7314c5ca26745", + "zh:8a793d816d7ef57e71758fe95bf830cfca70d121df70778b65cc11065ad004fd", + "zh:8c7cda08adba01b5ae8cc4e5fbf16761451f0fab01327e5f44fc47b7248ba653", + "zh:96d855f1771342771855c0fb2d47ff6a731e8f2fa5d242b18037c751fd63e6c3", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:a0a2d4efe2835f0101a0a5024e044a3f28c00e10a8d87fce89c707ef6db75cea", - "zh:aecb3e4b9121771dee9cac7975bf5d0657b5f3e8b57788c455beaeb0f3c48d93", - "zh:d2d3393170b8ef761d3146f39f6788c4a3e876e6c5d4cedca4870c2680688ae6", - "zh:daba5a005c1baa4a5eefbfb86d43ccf880eb5b42e8136f0d932f55886d72bda0", - "zh:de16a6ff3baacdaf9609a0a89aa1913fc19cccaf5ee0fc1c49c5a075baa47c02", + "zh:b2a62669b72c2471820410b58d764102b11c24e326831ddcfae85c7d20795acf", + "zh:b4a6b251ac24c8f5522581f8d55238d249d0008d36f64475beefc3791f229e1d", + "zh:ca519fa7ee1cac30439c7e2d311a0ecea6a5dae2d175fe8440f30133688b6272", + "zh:fbcd54e7d65806b0038fc8a0fbdc717e1284298ff66e22aac39dcc5a22cc99e5", ] } diff --git a/infrastructure/applications/config.tf b/infrastructure/applications/config.tf index 4628880be0..380484e9b8 100644 --- a/infrastructure/applications/config.tf +++ b/infrastructure/applications/config.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.64.0" + version = "5.66.0" configuration_aliases = [aws.us] } } diff --git a/infrastructure/applications/pretix_arm/githash.py b/infrastructure/applications/pretix_arm/githash.py new file mode 100644 index 0000000000..14cef63fd8 --- /dev/null +++ b/infrastructure/applications/pretix_arm/githash.py @@ -0,0 +1,12 @@ +import json +import subprocess +import sys + +git_output = subprocess.check_output( + ["git", "rev-list", "-1", "HEAD", "--", "."], +) +githash = git_output.decode().strip() + +output = {"githash": githash} +output_json = json.dumps(output) +sys.stdout.write(output_json) diff --git a/infrastructure/applications/pretix_arm/ecs.tf b/infrastructure/applications/pretix_arm/task_web.tf similarity index 80% rename from infrastructure/applications/pretix_arm/ecs.tf rename to infrastructure/applications/pretix_arm/task_web.tf index 0f26cfdb43..8368ddf047 100644 --- a/infrastructure/applications/pretix_arm/ecs.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -1,32 +1,13 @@ -data "aws_ecs_cluster" "server" { - cluster_name = "${terraform.workspace}-server" -} - -data "aws_instance" "redis" { - instance_tags = { - Name = "pythonit-production-redis" - } - - filter { - name = "instance-state-name" - values = ["running"] - } -} - -resource "aws_cloudwatch_log_group" "pretix" { - name = "/ecs/pythonit-${terraform.workspace}-pretix-arm" - retention_in_days = 7 -} - -resource "aws_ecs_task_definition" "pretix_service" { - family = "pythonit-${terraform.workspace}-pretix" - container_definitions = jsonencode([ +locals { + env_vars = [ { - name = "pretix" - image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" - memoryReservation = 200 - essential = true - environment = [ + name = "VIRTUAL_ENV", + value = "/var/pretix/venv" + }, + { + name = "PATH", + value = "/var/pretix/venv/bin:/usr/local/bin:/usr/bin:/bin" + }, { name = "DATABASE_NAME" value = "pretix" @@ -84,12 +65,50 @@ resource "aws_ecs_task_definition" "pretix_service" { value = "true" } ] +} +data "aws_ecs_cluster" "server" { + cluster_name = "${terraform.workspace}-server" +} + +data "aws_instance" "redis" { + instance_tags = { + Name = "pythonit-production-redis" + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +resource "aws_cloudwatch_log_group" "pretix" { + name = "/ecs/pythonit-${terraform.workspace}-pretix-arm" + retention_in_days = 7 +} + +resource "aws_ecs_task_definition" "pretix_web" { + family = "pythonit-${terraform.workspace}-pretix" + container_definitions = jsonencode([ + { + name = "pretix" + image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" + memoryReservation = 200 + essential = true + environment = local.env_vars portMappings = [ { - containerPort = 80 + containerPort = 8000 hostPort = 0 } ] + command = ["webworker"] + + workingDirectory = "/var/pretix" + + dockerLabels = { + "traefik.enable" = "true" + "traefik.http.routers.backend.rule" = "Host(`tickets.pycon.it`)" + } mountPoints = [ { sourceVolume = "media" @@ -131,11 +150,17 @@ resource "aws_ecs_task_definition" "pretix_service" { tags = {} } -resource "aws_ecs_service" "pretix" { - name = "pythonit-${terraform.workspace}-pretix" +resource "aws_ecs_service" "pretix_web" { + name = "pretix-web" cluster = data.aws_ecs_cluster.server.id - task_definition = aws_ecs_task_definition.pretix_service.arn + task_definition = aws_ecs_task_definition.pretix_web.arn desired_count = 1 - deployment_minimum_healthy_percent = 0 - deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + + lifecycle { + ignore_changes = [ + capacity_provider_strategy + ] + } } diff --git a/infrastructure/applications/pretix_arm/task_worker.tf b/infrastructure/applications/pretix_arm/task_worker.tf new file mode 100644 index 0000000000..1847079956 --- /dev/null +++ b/infrastructure/applications/pretix_arm/task_worker.tf @@ -0,0 +1,131 @@ +# resource "aws_ecs_task_definition" "pretix_web" { +# family = "pythonit-${terraform.workspace}-pretix" +# container_definitions = jsonencode([ +# { +# name = "pretix" +# image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" +# memoryReservation = 200 +# essential = true +# environment = [ +# { +# name = "DATABASE_NAME" +# value = "pretix" +# }, +# { +# name = "DATABASE_USERNAME" +# value = data.aws_db_instance.database.master_username +# }, +# { +# name = "DATABASE_PASSWORD" +# value = module.common_secrets.value.database_password +# }, +# { +# name = "DATABASE_HOST" +# value = data.aws_db_instance.database.address +# }, +# { +# name = "MAIL_USER" +# value = module.secrets.value.mail_user +# }, +# { +# name = "MAIL_PASSWORD" +# value = module.secrets.value.mail_password +# }, +# { +# name = "PRETIX_SENTRY_DSN" +# value = module.secrets.value.sentry_dsn +# }, +# { +# name = "SECRET_KEY" +# value = module.secrets.value.secret_key +# }, +# { +# name = "PRETIX_REDIS_LOCATION", +# value = "redis://${data.aws_instance.redis.private_ip}/0" +# }, +# { +# name = "PRETIX_REDIS_SESSIONS", +# value = "false" +# }, +# { +# name = "PRETIX_CELERY_BROKER", +# value = "redis://${data.aws_instance.redis.private_ip}/1" +# }, +# { +# name = "PRETIX_CELERY_BACKEND", +# value = "redis://${data.aws_instance.redis.private_ip}/2" +# }, +# { +# name = "PRETIX_PRETIX_URL", +# value = "https://tickets.pycon.it/" +# }, +# { +# name = "PRETIX_PRETIX_TRUST_X_FORWARDED_PROTO", +# value = "true" +# } +# ] +# portMappings = [ +# { +# containerPort = 80 +# hostPort = 0 +# } +# ] +# dockerLabels = { +# "traefik.enable" = "true" +# "traefik.http.routers.backend.rule" = "Host(`tickets.pycon.it`)" +# } +# mountPoints = [ +# { +# sourceVolume = "media" +# containerPath = "/data/media" +# }, +# { +# sourceVolume = "data" +# containerPath = "/var/pretix-data" +# } +# ] +# systemControls = [ +# { +# "namespace" : "net.core.somaxconn", +# "value" : "4096" +# } +# ] +# logConfiguration = { +# logDriver = "awslogs" +# options = { +# "awslogs-group" = aws_cloudwatch_log_group.pretix.name +# "awslogs-region" = "eu-central-1" +# "awslogs-stream-prefix" = "ecs" +# } +# } +# }, +# ]) + +# volume { +# name = "media" +# host_path = "/var/pretix/data/media" +# } + +# volume { +# name = "data" +# host_path = "/var/pretix-data" +# } + +# requires_compatibilities = [] +# tags = {} +# } + +# resource "aws_ecs_service" "pretix_web" { +# name = "pretix-worker" +# cluster = data.aws_ecs_cluster.server.id +# task_definition = aws_ecs_task_definition.pretix_web.arn +# desired_count = 1 +# deployment_minimum_healthy_percent = 100 +# deployment_maximum_percent = 200 + +# lifecycle { +# ignore_changes = [ +# capacity_provider_strategy +# ] +# } +# } diff --git a/infrastructure/applications/pycon_backend/providers.tf b/infrastructure/applications/pycon_backend/providers.tf index 4776eb5bb5..53481f218f 100644 --- a/infrastructure/applications/pycon_backend/providers.tf +++ b/infrastructure/applications/pycon_backend/providers.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.64.0" + version = "5.66.0" configuration_aliases = [aws.us] } } diff --git a/infrastructure/applications/pycon_backend/server.tf b/infrastructure/applications/pycon_backend/server.tf index a962443c7c..7098c74de1 100644 --- a/infrastructure/applications/pycon_backend/server.tf +++ b/infrastructure/applications/pycon_backend/server.tf @@ -74,10 +74,16 @@ resource "aws_ecs_task_definition" "backend" { resource "aws_ecs_service" "backend" { - name = "pythonit-${terraform.workspace}-backend" + name = "backend-web" cluster = data.aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.backend.arn desired_count = 1 deployment_minimum_healthy_percent = 100 deployment_maximum_percent = 200 + + lifecycle { + ignore_changes = [ + capacity_provider_strategy + ] + } } diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/worker.tf index 540273aeb2..b75e07e988 100644 --- a/infrastructure/applications/pycon_backend/worker.tf +++ b/infrastructure/applications/pycon_backend/worker.tf @@ -420,19 +420,31 @@ resource "aws_ecs_task_definition" "beat" { } resource "aws_ecs_service" "worker" { - name = "pythonit-${terraform.workspace}-worker" + name = "backend-worker" cluster = data.aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.worker.arn desired_count = 1 deployment_minimum_healthy_percent = 100 deployment_maximum_percent = 200 + + lifecycle { + ignore_changes = [ + capacity_provider_strategy + ] + } } resource "aws_ecs_service" "beat" { - name = "pythonit-${terraform.workspace}-beat" + name = "backend-beat" cluster = data.aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.beat.arn desired_count = 1 deployment_minimum_healthy_percent = 0 deployment_maximum_percent = 100 + + lifecycle { + ignore_changes = [ + capacity_provider_strategy + ] + } } diff --git a/infrastructure/applications/server/task_traefik.tf b/infrastructure/applications/server/task_traefik.tf index 2e46911752..dcda123bd5 100644 --- a/infrastructure/applications/server/task_traefik.tf +++ b/infrastructure/applications/server/task_traefik.tf @@ -74,10 +74,16 @@ resource "aws_ecs_task_definition" "traefik" { } resource "aws_ecs_service" "traefik" { - name = "pythonit-${terraform.workspace}-traefik" + name = "traefik" cluster = aws_ecs_cluster.server.id task_definition = aws_ecs_task_definition.traefik.arn desired_count = 1 deployment_minimum_healthy_percent = 100 deployment_maximum_percent = 200 + + lifecycle { + ignore_changes = [ + capacity_provider_strategy + ] + } } From bb394f45388ff66087f907e9c7b1f966965a010d Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 01:26:12 +0200 Subject: [PATCH 10/46] changes --- .../applications/pretix_arm/task_web.tf | 4 +- pretix/Dockerfile | 63 ++++++++++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 8368ddf047..5f43962f8f 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -101,7 +101,9 @@ resource "aws_ecs_task_definition" "pretix_web" { hostPort = 0 } ] - command = ["webworker"] + + entrypoint = ["/var/pretix/venv/bin/gunicorn"] + command = ["pretix.wsgi", "--name pretix", "--bind 0.0.0.0:8000", "--max-requests-jitter 50"] workingDirectory = "/var/pretix" diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 8e4e0a7b42..98c90329cf 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,14 +1,71 @@ -FROM pretix/standalone:2024.7.0 +FROM python:3.11-bookworm + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + gettext \ + git \ + libffi-dev \ + libjpeg-dev \ + libmemcached-dev \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + locales \ + nginx \ + python3-virtualenv \ + python3-dev \ + sudo \ + supervisor \ + libmaxminddb0 \ + libmaxminddb-dev \ + zlib1g-dev \ + nodejs \ + npm && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + dpkg-reconfigure locales && \ + locale-gen C.UTF-8 && \ + /usr/sbin/update-locale LANG=C.UTF-8 && \ + mkdir /etc/pretix && \ + mkdir /data && \ + useradd -ms /bin/bash -d /pretix -u 15371 pretixuser && \ + echo 'pretixuser ALL=(ALL) NOPASSWD:SETENV: /usr/bin/supervisord' >> /etc/sudoers && \ + mkdir /static && \ + mkdir /etc/supervisord + +ENV LC_ALL=C.UTF-8 \ + DJANGO_SETTINGS_MODULE=production_settings COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py +RUN pip3 install -U \ + pip \ + setuptools \ + wheel \ + pretix-plugin-extended-api==0.1.12 \ + pretix-plugin-attendance-certificate==0.1.11 \ + pretix-fattura-elettronica==0.2.15 && \ + cd /pretix && \ + PRETIX_DOCKER_BUILD=TRUE pip3 install \ + -e ".[memcached]" \ + gunicorn django-extensions ipython && \ + rm -rf ~/.cache/pip + +RUN chmod +x /usr/local/bin/pretix && \ + rm /etc/nginx/sites-enabled/default && \ + cd /pretix/src && \ + rm -f pretix.cfg && \ + mkdir -p data && \ + chown -R pretixuser:pretixuser /pretix /data data && \ + sudo -u pretixuser make production + USER root RUN chown -R pretixuser:pretixuser /pretix USER pretixuser -RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-certificate==0.1.11 pretix-fattura-elettronica==0.2.15 - RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles RUN cd /pretix/src && DATABASE_HOST=demo make compress From 51cda1c0c792db4b3cab3e1710d3db4470fb1de0 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 01:41:37 +0200 Subject: [PATCH 11/46] test --- .github/workflows/deploy.yml | 68 ++++++++++++++++++++++++++++++++++-- pretix/Dockerfile | 63 ++------------------------------- 2 files changed, 68 insertions(+), 63 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 360065d993..a30322960c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,6 +50,70 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-central-1 + build-pretix: + runs-on: [self-hosted] + steps: + - uses: actions/checkout@v4 + with: + repository: pretix/pretix + ref: v2024.8.0 + path: ./pretix-clone + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.aws_access_key_id }} + aws-secret-access-key: ${{ secrets.aws_secret_access_key }} + aws-region: eu-central-1 + - name: Get service githash + id: git + run: | + hash=$(git rev-list -1 HEAD -- pretix + echo "githash=$hash" >> $GITHUB_OUTPUT + - name: Check if commit is already on ECR + id: image + run: | + set +e + aws ecr describe-images --repository-name=pythonit/pretix --image-ids=imageTag=arm-${{ steps.git.outputs.githash }} + if [[ $? == 0 ]]; then + echo "image_exists=1" >> $GITHUB_OUTPUT + else + echo "image_exists=0" >> $GITHUB_OUTPUT + fi + - name: Login to Amazon ECR + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-buildx-action@v3 + - name: Build pretix base + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + with: + context: ./pretix-clone + file: ./pretix-clone/Dockerfile + builder: pretix-base + provenance: false + push: false + tags: pretix-base:${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 + - name: Build and push pretix + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + with: + context: ./pretix + file: ./pretix/Dockerfile + builder: pretix-base + provenance: false + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 + build-and-push-arm-service: runs-on: [self-hosted] permissions: @@ -61,8 +125,6 @@ jobs: service: - name: pycon-backend dir: backend - - name: pretix - dir: pretix steps: - uses: actions/checkout@v4 @@ -148,7 +210,7 @@ jobs: terraform: runs-on: ubuntu-latest - needs: [build-emails, build-and-push-arm-service, create-db] + needs: [build-emails, build-and-push-arm-service, build-pretix, create-db] environment: name: ${{ fromJSON('["pastaporto", "production"]')[github.ref == 'refs/heads/main'] }} defaults: diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 98c90329cf..fc3f10e088 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,71 +1,14 @@ -FROM python:3.11-bookworm - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - gettext \ - git \ - libffi-dev \ - libjpeg-dev \ - libmemcached-dev \ - libpq-dev \ - libssl-dev \ - libxml2-dev \ - libxslt1-dev \ - locales \ - nginx \ - python3-virtualenv \ - python3-dev \ - sudo \ - supervisor \ - libmaxminddb0 \ - libmaxminddb-dev \ - zlib1g-dev \ - nodejs \ - npm && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - dpkg-reconfigure locales && \ - locale-gen C.UTF-8 && \ - /usr/sbin/update-locale LANG=C.UTF-8 && \ - mkdir /etc/pretix && \ - mkdir /data && \ - useradd -ms /bin/bash -d /pretix -u 15371 pretixuser && \ - echo 'pretixuser ALL=(ALL) NOPASSWD:SETENV: /usr/bin/supervisord' >> /etc/sudoers && \ - mkdir /static && \ - mkdir /etc/supervisord - -ENV LC_ALL=C.UTF-8 \ - DJANGO_SETTINGS_MODULE=production_settings +FROM pretix-base:${PRETIX_BASE_TAG} COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py -RUN pip3 install -U \ - pip \ - setuptools \ - wheel \ - pretix-plugin-extended-api==0.1.12 \ - pretix-plugin-attendance-certificate==0.1.11 \ - pretix-fattura-elettronica==0.2.15 && \ - cd /pretix && \ - PRETIX_DOCKER_BUILD=TRUE pip3 install \ - -e ".[memcached]" \ - gunicorn django-extensions ipython && \ - rm -rf ~/.cache/pip - -RUN chmod +x /usr/local/bin/pretix && \ - rm /etc/nginx/sites-enabled/default && \ - cd /pretix/src && \ - rm -f pretix.cfg && \ - mkdir -p data && \ - chown -R pretixuser:pretixuser /pretix /data data && \ - sudo -u pretixuser make production - USER root RUN chown -R pretixuser:pretixuser /pretix USER pretixuser +RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-certificate==0.1.11 pretix-fattura-elettronica==0.2.15 + RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles RUN cd /pretix/src && DATABASE_HOST=demo make compress From 4bad797d402d4e00f9aac06c316346c05ec053c1 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 01:44:18 +0200 Subject: [PATCH 12/46] c --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a30322960c..d1dc764115 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,7 +67,7 @@ jobs: - name: Get service githash id: git run: | - hash=$(git rev-list -1 HEAD -- pretix + hash=$(git rev-list -1 HEAD -- ./pretix echo "githash=$hash" >> $GITHUB_OUTPUT - name: Check if commit is already on ECR id: image From 95b47b19607bbf096da902bd6d34dbdac473f551 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 01:45:33 +0200 Subject: [PATCH 13/46] aa --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d1dc764115..4256664f80 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,7 +67,7 @@ jobs: - name: Get service githash id: git run: | - hash=$(git rev-list -1 HEAD -- ./pretix + hash=$(git rev-list -1 HEAD -- ./pretix) echo "githash=$hash" >> $GITHUB_OUTPUT - name: Check if commit is already on ECR id: image From f0199f16c38bc9c04944ff8734a315a63e41ede4 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 01:47:40 +0200 Subject: [PATCH 14/46] c --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4256664f80..4159d5d4c0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -105,7 +105,7 @@ jobs: with: context: ./pretix file: ./pretix/Dockerfile - builder: pretix-base + builder: ${{ steps.buildx.outputs.name }} provenance: false push: true tags: | From f64a2ed81f368fc8fa9f5f5c3d67e564a02c22a8 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 01:49:53 +0200 Subject: [PATCH 15/46] fix --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4159d5d4c0..06e5078f90 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -92,7 +92,7 @@ jobs: with: context: ./pretix-clone file: ./pretix-clone/Dockerfile - builder: pretix-base + builder: ${{ steps.buildx.outputs.name }} provenance: false push: false tags: pretix-base:${{ steps.git.outputs.githash }} From e46b22d3233756ca2d2a43baaf555df880ff2439 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 02:04:38 +0200 Subject: [PATCH 16/46] change --- .github/workflows/deploy.yml | 4 +++- pretix/Dockerfile | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 06e5078f90..93f63bdffb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -95,7 +95,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false push: false - tags: pretix-base:${{ steps.git.outputs.githash }} + tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 @@ -113,6 +113,8 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 + build-args: | + PRETIX_BASE_TAG=${{ steps.git.outputs.githash }} build-and-push-arm-service: runs-on: [self-hosted] diff --git a/pretix/Dockerfile b/pretix/Dockerfile index fc3f10e088..3c106e34dd 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,4 +1,5 @@ -FROM pretix-base:${PRETIX_BASE_TAG} +ARG PRETIX_BASE_TAG +FROM pyconit-local-pretix-base:${PRETIX_BASE_TAG} COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py From b9e23cb2a65f381e8aa164ce1fffab05618e0c4d Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 02:23:20 +0200 Subject: [PATCH 17/46] c --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 93f63bdffb..6a32cd70e0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -94,7 +94,7 @@ jobs: file: ./pretix-clone/Dockerfile builder: ${{ steps.buildx.outputs.name }} provenance: false - push: false + load: true tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache From 1c19b83066649e4e1d626e76e363a3f1fa1e2695 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 21:00:06 +0200 Subject: [PATCH 18/46] test --- .github/workflows/deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6a32cd70e0..adae601fd2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -99,6 +99,12 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 + - name: Show Docker images + run: docker images + + - name: Inspect pretix-base image + run: docker inspect pretix-base:${{ steps.git.outputs.githash }} + - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 From e81e89191aaec4f3f22f7027b888099585dbb5ef Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 21:03:01 +0200 Subject: [PATCH 19/46] a --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index adae601fd2..992b716e0e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -103,7 +103,7 @@ jobs: run: docker images - name: Inspect pretix-base image - run: docker inspect pretix-base:${{ steps.git.outputs.githash }} + run: docker inspect pyconit-local-pretix-base:${{ steps.git.outputs.githash }} - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} From 9af588823c97c7daa17d9df4630de83d48cc4548 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 21:06:10 +0200 Subject: [PATCH 20/46] t --- .github/workflows/deploy.yml | 5 +++-- pretix/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 992b716e0e..f481a42932 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -95,7 +95,8 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false load: true - tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} + tags: | + ghcr.io/pythonitalia/pycon/pretix-base-image:arm-${{ steps.git.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 @@ -103,7 +104,7 @@ jobs: run: docker images - name: Inspect pretix-base image - run: docker inspect pyconit-local-pretix-base:${{ steps.git.outputs.githash }} + run: docker inspect ghcr.io/pythonitalia/pycon/pretix-base-image:arm-${{ steps.git.outputs.githash }} - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 3c106e34dd..8f324e7567 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,5 +1,5 @@ ARG PRETIX_BASE_TAG -FROM pyconit-local-pretix-base:${PRETIX_BASE_TAG} +FROM ghcr.io/pythonitalia/pycon/pretix-base-image:${PRETIX_BASE_TAG} COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py From ef6ee7fb0c472d478248379f7f260656d6ef81c0 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 21:15:48 +0200 Subject: [PATCH 21/46] change --- .github/workflows/deploy.yml | 12 ++++++------ pretix/Dockerfile | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f481a42932..165f21a714 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -89,22 +89,22 @@ jobs: - name: Build pretix base if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 + id: build-pretix-base with: context: ./pretix-clone file: ./pretix-clone/Dockerfile builder: ${{ steps.buildx.outputs.name }} provenance: false load: true - tags: | - ghcr.io/pythonitalia/pycon/pretix-base-image:arm-${{ steps.git.outputs.githash }} + tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 + - name: Show Docker images run: docker images - name: Inspect pretix-base image - run: docker inspect ghcr.io/pythonitalia/pycon/pretix-base-image:arm-${{ steps.git.outputs.githash }} + run: docker inspect pyconit-local-pretix-base:${{ steps.git.outputs.githash }} - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} @@ -115,13 +115,13 @@ jobs: builder: ${{ steps.buildx.outputs.name }} provenance: false push: true + pull: false tags: | ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ steps.git.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 build-args: | - PRETIX_BASE_TAG=${{ steps.git.outputs.githash }} + PRETIX_BASE_IMAGE_ID=${{ steps.build-pretix-base.outputs.imageid }} build-and-push-arm-service: runs-on: [self-hosted] diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 8f324e7567..9705a3751e 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,5 +1,5 @@ -ARG PRETIX_BASE_TAG -FROM ghcr.io/pythonitalia/pycon/pretix-base-image:${PRETIX_BASE_TAG} +ARG PRETIX_BASE_IMAGE_ID +FROM ${PRETIX_BASE_IMAGE_ID} COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py From 1a440cc4832978306912661fcc5b1c56089b97de Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 21:59:32 +0200 Subject: [PATCH 22/46] a --- .github/workflows/deploy.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 165f21a714..d774f1f260 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -82,10 +82,6 @@ jobs: - name: Login to Amazon ECR if: ${{ steps.image.outputs.image_exists == 0 }} uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-buildx-action@v3 - name: Build pretix base if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 @@ -93,7 +89,6 @@ jobs: with: context: ./pretix-clone file: ./pretix-clone/Dockerfile - builder: ${{ steps.buildx.outputs.name }} provenance: false load: true tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} @@ -112,7 +107,6 @@ jobs: with: context: ./pretix file: ./pretix/Dockerfile - builder: ${{ steps.buildx.outputs.name }} provenance: false push: true pull: false From 5c23660a5c6402c07f25856d90c4435d52e0b0d0 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 22:04:46 +0200 Subject: [PATCH 23/46] c --- .github/workflows/deploy.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d774f1f260..b01926ca47 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -89,11 +89,7 @@ jobs: with: context: ./pretix-clone file: ./pretix-clone/Dockerfile - provenance: false load: true - tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - name: Show Docker images run: docker images @@ -112,8 +108,6 @@ jobs: pull: false tags: | ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache build-args: | PRETIX_BASE_IMAGE_ID=${{ steps.build-pretix-base.outputs.imageid }} From 6303b591da4a0a627a642beffd714b3e81e45e48 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 22:19:55 +0200 Subject: [PATCH 24/46] ee --- .github/workflows/deploy.yml | 25 +++++++++++++++---------- pretix/Dockerfile | 5 +++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b01926ca47..6a32cd70e0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -82,34 +82,39 @@ jobs: - name: Login to Amazon ECR if: ${{ steps.image.outputs.image_exists == 0 }} uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + id: buildx + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/setup-buildx-action@v3 - name: Build pretix base if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 - id: build-pretix-base with: context: ./pretix-clone file: ./pretix-clone/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + provenance: false load: true - - - name: Show Docker images - run: docker images - - - name: Inspect pretix-base image - run: docker inspect pyconit-local-pretix-base:${{ steps.git.outputs.githash }} - + tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 with: context: ./pretix file: ./pretix/Dockerfile + builder: ${{ steps.buildx.outputs.name }} provenance: false push: true - pull: false tags: | ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:arm-${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 build-args: | - PRETIX_BASE_IMAGE_ID=${{ steps.build-pretix-base.outputs.imageid }} + PRETIX_BASE_TAG=${{ steps.git.outputs.githash }} build-and-push-arm-service: runs-on: [self-hosted] diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 9705a3751e..6a88bc5f09 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,5 +1,6 @@ -ARG PRETIX_BASE_IMAGE_ID -FROM ${PRETIX_BASE_IMAGE_ID} +# ARG PRETIX_BASE_TAG +# FROM pyconit-local-pretix-base:${PRETIX_BASE_TAG} +FROM --platform=arm64 pretix/standalone:2024.7.0 COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py From 15e0e7113dbde04c4064b01ed77bb1b7c08170a1 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 23:05:35 +0200 Subject: [PATCH 25/46] ee --- .github/workflows/deploy.yml | 26 +++++++++++++------------- pretix/Dockerfile | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6a32cd70e0..8b509ca551 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -86,19 +86,19 @@ jobs: id: buildx if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-buildx-action@v3 - - name: Build pretix base - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/build-push-action@v6 - with: - context: ./pretix-clone - file: ./pretix-clone/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - load: true - tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 + # - name: Build pretix base + # if: ${{ steps.image.outputs.image_exists == 0 }} + # uses: docker/build-push-action@v6 + # with: + # context: ./pretix-clone + # file: ./pretix-clone/Dockerfile + # builder: ${{ steps.buildx.outputs.name }} + # provenance: false + # load: true + # tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache + # platforms: linux/arm64 - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 6a88bc5f09..3b2e3c2975 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,6 +1,6 @@ # ARG PRETIX_BASE_TAG # FROM pyconit-local-pretix-base:${PRETIX_BASE_TAG} -FROM --platform=arm64 pretix/standalone:2024.7.0 +FROM --platform=linux/arm64 pretix/standalone:2024.7.0 COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py From ab7685d479503acf374cde3dae9f7bdf8167ca6a Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 23:18:27 +0200 Subject: [PATCH 26/46] ee --- .github/workflows/deploy.yml | 31 ++++++++++++++++--------------- pretix/Dockerfile | 5 ++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b509ca551..19ff44a7f7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -86,26 +86,27 @@ jobs: id: buildx if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/setup-buildx-action@v3 - # - name: Build pretix base - # if: ${{ steps.image.outputs.image_exists == 0 }} - # uses: docker/build-push-action@v6 - # with: - # context: ./pretix-clone - # file: ./pretix-clone/Dockerfile - # builder: ${{ steps.buildx.outputs.name }} - # provenance: false - # load: true - # tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} - # cache-from: type=local,src=/tmp/.buildx-cache - # cache-to: type=local,dest=/tmp/.buildx-cache - # platforms: linux/arm64 + - name: Build pretix base + if: ${{ steps.image.outputs.image_exists == 0 }} + uses: docker/build-push-action@v6 + id: build-pretix-base + with: + context: ./pretix-clone + file: ./pretix-clone/Dockerfile + builder: docker + provenance: false + load: true + tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + platforms: linux/arm64 - name: Build and push pretix if: ${{ steps.image.outputs.image_exists == 0 }} uses: docker/build-push-action@v6 with: context: ./pretix file: ./pretix/Dockerfile - builder: ${{ steps.buildx.outputs.name }} + builder: docker provenance: false push: true tags: | @@ -114,7 +115,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 build-args: | - PRETIX_BASE_TAG=${{ steps.git.outputs.githash }} + PRETIX_IMAGE=${{ steps.build-pretix-base.outputs.imageid }} build-and-push-arm-service: runs-on: [self-hosted] diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 3b2e3c2975..b2d5061d5f 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,6 +1,5 @@ -# ARG PRETIX_BASE_TAG -# FROM pyconit-local-pretix-base:${PRETIX_BASE_TAG} -FROM --platform=linux/arm64 pretix/standalone:2024.7.0 +ARG PRETIX_IMAGE +FROM ${PRETIX_IMAGE} COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py From ea6a51c3808d77c15c3117c2584cf4dc75a4c022 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 23:27:48 +0200 Subject: [PATCH 27/46] ee --- .github/workflows/deploy.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 19ff44a7f7..afad07cf8d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -95,8 +95,9 @@ jobs: file: ./pretix-clone/Dockerfile builder: docker provenance: false - load: true - tags: pyconit-local-pretix-base:${{ steps.git.outputs.githash }} + push: true + tags: | + ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ steps.git.outputs.githash }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 @@ -115,7 +116,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache platforms: linux/arm64 build-args: | - PRETIX_IMAGE=${{ steps.build-pretix-base.outputs.imageid }} + PRETIX_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/pretix:pretix-base-${{ steps.git.outputs.githash }} build-and-push-arm-service: runs-on: [self-hosted] From f7238d70cc7ae857102aaa620bdeaaffb01a9068 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 23:30:08 +0200 Subject: [PATCH 28/46] ee --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index afad07cf8d..c872fc094b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -93,7 +93,7 @@ jobs: with: context: ./pretix-clone file: ./pretix-clone/Dockerfile - builder: docker + builder: ${{ steps.buildx.outputs.name }} provenance: false push: true tags: | @@ -107,7 +107,7 @@ jobs: with: context: ./pretix file: ./pretix/Dockerfile - builder: docker + builder: ${{ steps.buildx.outputs.name }} provenance: false push: true tags: | From 00c8d2d3bd2c95f684e8fadb2273e30aade2bc1e Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sat, 7 Sep 2024 23:44:10 +0200 Subject: [PATCH 29/46] ee --- infrastructure/applications/pretix_arm/task_web.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 5f43962f8f..afdc61fb38 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -103,7 +103,7 @@ resource "aws_ecs_task_definition" "pretix_web" { ] entrypoint = ["/var/pretix/venv/bin/gunicorn"] - command = ["pretix.wsgi", "--name pretix", "--bind 0.0.0.0:8000", "--max-requests-jitter 50"] + command = ["pretix.wsgi", "--name pretix", "--bind 0.0.0.0:8000",] workingDirectory = "/var/pretix" From 32c6d285cad77f35d68d8577bca01b19b5863a39 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 00:35:20 +0200 Subject: [PATCH 30/46] ee --- .../applications/pretix_arm/task_web.tf | 41 +++++++++---------- infrastructure/applications/server/main.tf | 17 ++++++++ pretix/Dockerfile | 4 -- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index afdc61fb38..4cedaf348a 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -102,25 +102,24 @@ resource "aws_ecs_task_definition" "pretix_web" { } ] - entrypoint = ["/var/pretix/venv/bin/gunicorn"] - command = ["pretix.wsgi", "--name pretix", "--bind 0.0.0.0:8000",] - + entrypoint = ["gunicorn"] + command = ["pretix.wsgi", "--name pretix", "--bind 0.0.0.0:8000", "--max-requests 1200", "--max-requests-jitter 50"] workingDirectory = "/var/pretix" dockerLabels = { "traefik.enable" = "true" "traefik.http.routers.backend.rule" = "Host(`tickets.pycon.it`)" } - mountPoints = [ - { - sourceVolume = "media" - containerPath = "/data/media" - }, - { - sourceVolume = "data" - containerPath = "/var/pretix-data" - } - ] + # mountPoints = [ + # { + # sourceVolume = "media" + # containerPath = "/data/media" + # }, + # { + # sourceVolume = "data" + # containerPath = "/var/pretix-data" + # } + # ] systemControls = [ { "namespace" : "net.core.somaxconn", @@ -138,15 +137,15 @@ resource "aws_ecs_task_definition" "pretix_web" { }, ]) - volume { - name = "media" - host_path = "/var/pretix/data/media" - } + # volume { + # name = "media" + # host_path = "/var/pretix/data/media" + # } - volume { - name = "data" - host_path = "/var/pretix-data" - } + # volume { + # name = "data" + # host_path = "/var/pretix-data" + # } requires_compatibilities = [] tags = {} diff --git a/infrastructure/applications/server/main.tf b/infrastructure/applications/server/main.tf index 0ce8cf4d57..c57170be49 100644 --- a/infrastructure/applications/server/main.tf +++ b/infrastructure/applications/server/main.tf @@ -5,6 +5,15 @@ data "template_file" "server_user_data" { } } +data "aws_ami" "ecs" { + owners = ["self"] + + filter { + name = "image-id" + values = [var.ecs_arm_ami] + } +} + data "aws_security_group" "tempone" { name = "pythonit-${terraform.workspace}-worker-instance" } @@ -20,6 +29,14 @@ resource "aws_launch_template" "server" { name = aws_iam_instance_profile.server.name } + block_device_mappings { + device_name = data.aws_ami.ecs.root_device_name + + ebs { + volume_size = 20 + } + } + network_interfaces { associate_public_ip_address = true security_groups = [ diff --git a/pretix/Dockerfile b/pretix/Dockerfile index b2d5061d5f..589db5fed4 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -4,10 +4,6 @@ FROM ${PRETIX_IMAGE} COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py -USER root -RUN chown -R pretixuser:pretixuser /pretix -USER pretixuser - RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-certificate==0.1.11 pretix-fattura-elettronica==0.2.15 RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles From 95815a3320d74c1d1d067051de98e0f777e1f6c3 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 00:47:57 +0200 Subject: [PATCH 31/46] ee --- infrastructure/applications/pretix_arm/task_web.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 4cedaf348a..50f4e16dda 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -103,8 +103,9 @@ resource "aws_ecs_task_definition" "pretix_web" { ] entrypoint = ["gunicorn"] - command = ["pretix.wsgi", "--name pretix", "--bind 0.0.0.0:8000", "--max-requests 1200", "--max-requests-jitter 50"] + command = ["pretix.wsgi", "--name=pretix", "--bind=0.0.0.0:8000", "--max-requests=1200", "--max-requests-jitter=50"] workingDirectory = "/var/pretix" + user = "pretixuser" dockerLabels = { "traefik.enable" = "true" From 277ff08a10bd5fef0fde85bdb47686d538855897 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 01:08:54 +0200 Subject: [PATCH 32/46] ee --- .../applications/pretix_arm/task_web.tf | 50 ++++++++++++++++--- pretix/Dockerfile | 3 +- pretix/entrypoint.sh | 19 ------- pretix/settings.py | 31 +----------- 4 files changed, 45 insertions(+), 58 deletions(-) delete mode 100755 pretix/entrypoint.sh diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 50f4e16dda..a354a6aa4d 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -7,37 +7,73 @@ locals { { name = "PATH", value = "/var/pretix/venv/bin:/usr/local/bin:/usr/bin:/bin" + }, + { + name = "PRETIX_DATABASE_BACKEND", + value = "postgresql" }, { - name = "DATABASE_NAME" + name = "PRETIX_DATABASE_NAME" value = "pretix" }, { - name = "DATABASE_USERNAME" + name = "PRETIX_DATABASE_USER" value = data.aws_db_instance.database.master_username }, { - name = "DATABASE_PASSWORD" + name = "PRETIX_DATABASE_PASSWORD" value = module.common_secrets.value.database_password }, { - name = "DATABASE_HOST" + name = "PRETIX_DATABASE_HOST" value = data.aws_db_instance.database.address }, { - name = "MAIL_USER" + name = "PRETIX_DATABASE_PORT" + value = "5432" + }, + { + name = "PRETIX_MAIL_USER" value = module.secrets.value.mail_user }, { - name = "MAIL_PASSWORD" + name = "PRETIX_MAIL_PASSWORD" value = module.secrets.value.mail_password }, + { + name = "PRETIX_MAIL_HOST" + value = "email-smtp.eu-central-1.amazonaws.com" + }, + { + name = "PRETIX_MAIL_PORT" + value = "587" + }, + { + name = "PRETIX_MAIL_TLS" + value = true + }, + { + name = "PRETIX_MAIL_SSL" + value = false + }, + { + name = "PRETIX_MAIL_FROM" + value = "noreply@pycon.it" + }, + { + name = "PRETIX_PRETIX_TRUST_X_FORWARDED_HOST" + value = true + }, + { + name = "PRETIX_PRETIX_REGISTRATION" + value = true + }, { name = "PRETIX_SENTRY_DSN" value = module.secrets.value.sentry_dsn }, { - name = "SECRET_KEY" + name = "PRETIX_DJANGO_SECRET" value = module.secrets.value.secret_key }, { diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 589db5fed4..4811e944b5 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -1,7 +1,6 @@ ARG PRETIX_IMAGE FROM ${PRETIX_IMAGE} -COPY ./entrypoint.sh /entrypoint.sh COPY ./settings.py /pretix/src/production_settings.py RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-certificate==0.1.11 pretix-fattura-elettronica==0.2.15 @@ -9,5 +8,5 @@ RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-cert RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles RUN cd /pretix/src && DATABASE_HOST=demo make compress -ENTRYPOINT [ "/entrypoint.sh" ] +ENTRYPOINT ["pretix"] CMD [ "all" ] diff --git a/pretix/entrypoint.sh b/pretix/entrypoint.sh deleted file mode 100755 index 2ba0b96f75..0000000000 --- a/pretix/entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -l - -awk '{ - gsub("{{database_name}}", "'$DATABASE_NAME'" , $0); - gsub("{{database_username}}", "'$DATABASE_USERNAME'" , $0); - gsub("{{database_password}}", "'$DATABASE_PASSWORD'" , $0); - gsub("{{database_host}}", "'$DATABASE_HOST'" , $0); - - gsub("{{mail_user}}", "'$MAIL_USER'" , $0); - gsub("{{mail_password}}", "'$MAIL_PASSWORD'" , $0); - gsub("{{sentry_dsn}}", "'$SENTRY_DSN'" , $0); - - gsub("{{secret_key}}", "'$SECRET_KEY'" , $0); - - gsub("{{url}}", "'$URL'" , $0); - print $0 > "/pretix/src/production_settings.py"; -}' /pretix/src/production_settings.py - -pretix "$@" diff --git a/pretix/settings.py b/pretix/settings.py index 34d0624df3..c923e38787 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -1,48 +1,19 @@ from pretix.settings import * # noqa - -SECRET_KEY = "{{secret_key}}" +from pretix.settings import INSTALLED_APPS, ALL_LANGUAGES, LOGGING LOGGING["handlers"]["mail_admins"]["include_html"] = True # noqa -DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "{{database_name}}", - "USER": "{{database_username}}", - "PASSWORD": "{{database_password}}", - "HOST": "{{database_host}}", - "PORT": "5432", - } -} - # Allow all the languages # see: pretix/settings.py#L425-L435 LANGUAGES = [(k, v) for k, v in ALL_LANGUAGES] # noqa -USE_X_FORWARDED_HOST = True -SITE_URL = "https://tickets.pycon.it" - -MAIL_FROM = SERVER_EMAIL = DEFAULT_FROM_EMAIL = "noreply@pycon.it" -MAIL_FROM_NOTIFICATIONS = MAIL_FROM -MAIL_FROM_ORGANIZERS = MAIL_FROM -EMAIL_HOST = "email-smtp.eu-central-1.amazonaws.com" -EMAIL_PORT = 587 -EMAIL_HOST_USER = "{{mail_user}}" -EMAIL_HOST_PASSWORD = "{{mail_password}}" -EMAIL_USE_TLS = True -EMAIL_USE_SSL = False EMAIL_SUBJECT_PREFIX = "[PyCon Tickets] " - PRETIX_INSTANCE_NAME = "Python Italia" # this is is needed for our hack that updates the order view # without having to rewrite the whole template CSP_ADDITIONAL_HEADER = "script-src 'self' 'unsafe-inline'" -# Config -PRETIX_REGISTRATION = False - if "pretix_fattura_elettronica" in INSTALLED_APPS: # noqa INSTALLED_APPS.remove("pretix_fattura_elettronica") # noqa - INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa From 75d2b699ee48fda5868e1f364d50d5ba741845d1 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 02:47:34 +0200 Subject: [PATCH 33/46] ee --- infrastructure/applications/pretix_arm/s3.tf | 8 + .../applications/pretix_arm/task_web.tf | 248 +++++++++--------- .../pycon_backend/{server.tf => task_web.tf} | 2 +- pretix/settings.py | 10 +- 4 files changed, 142 insertions(+), 126 deletions(-) create mode 100644 infrastructure/applications/pretix_arm/s3.tf rename infrastructure/applications/pycon_backend/{server.tf => task_web.tf} (96%) diff --git a/infrastructure/applications/pretix_arm/s3.tf b/infrastructure/applications/pretix_arm/s3.tf new file mode 100644 index 0000000000..95aa90b9da --- /dev/null +++ b/infrastructure/applications/pretix_arm/s3.tf @@ -0,0 +1,8 @@ +locals { + is_prod = terraform.workspace == "production" +} + +resource "aws_s3_bucket" "media" { + bucket = "${terraform.workspace}-pretix-media" + force_destroy = !local.is_prod +} diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index a354a6aa4d..9afbe8a63c 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -1,106 +1,132 @@ locals { env_vars = [ { - name = "VIRTUAL_ENV", + name = "VIRTUAL_ENV", value = "/var/pretix/venv" }, { - name = "PATH", + name = "PATH", value = "/var/pretix/venv/bin:/usr/local/bin:/usr/bin:/bin" }, { - name = "PRETIX_DATABASE_BACKEND", + name = "PRETIX_DATABASE_BACKEND", value = "postgresql" }, - { - name = "PRETIX_DATABASE_NAME" - value = "pretix" - }, - { - name = "PRETIX_DATABASE_USER" - value = data.aws_db_instance.database.master_username - }, - { - name = "PRETIX_DATABASE_PASSWORD" - value = module.common_secrets.value.database_password - }, - { - name = "PRETIX_DATABASE_HOST" - value = data.aws_db_instance.database.address - }, - { - name = "PRETIX_DATABASE_PORT" - value = "5432" - }, - { - name = "PRETIX_MAIL_USER" - value = module.secrets.value.mail_user - }, - { - name = "PRETIX_MAIL_PASSWORD" - value = module.secrets.value.mail_password - }, - { - name = "PRETIX_MAIL_HOST" - value = "email-smtp.eu-central-1.amazonaws.com" - }, - { - name = "PRETIX_MAIL_PORT" - value = "587" - }, - { - name = "PRETIX_MAIL_TLS" - value = true - }, - { - name = "PRETIX_MAIL_SSL" - value = false - }, - { - name = "PRETIX_MAIL_FROM" - value = "noreply@pycon.it" - }, - { - name = "PRETIX_PRETIX_TRUST_X_FORWARDED_HOST" - value = true - }, - { - name = "PRETIX_PRETIX_REGISTRATION" - value = true - }, - { - name = "PRETIX_SENTRY_DSN" - value = module.secrets.value.sentry_dsn - }, - { - name = "PRETIX_DJANGO_SECRET" - value = module.secrets.value.secret_key - }, - { - name = "PRETIX_REDIS_LOCATION", - value = "redis://${data.aws_instance.redis.private_ip}/0" - }, - { - name = "PRETIX_REDIS_SESSIONS", - value = "false" - }, - { - name = "PRETIX_CELERY_BROKER", - value = "redis://${data.aws_instance.redis.private_ip}/1" - }, - { - name = "PRETIX_CELERY_BACKEND", - value = "redis://${data.aws_instance.redis.private_ip}/2" - }, - { - name = "PRETIX_PRETIX_URL", - value = "https://tickets.pycon.it/" - }, - { - name = "PRETIX_PRETIX_TRUST_X_FORWARDED_PROTO", - value = "true" - } - ] + { + name = "PRETIX_DATABASE_NAME" + value = "pretix" + }, + { + name = "PRETIX_DATABASE_USER" + value = data.aws_db_instance.database.master_username + }, + { + name = "PRETIX_DATABASE_PASSWORD" + value = module.common_secrets.value.database_password + }, + { + name = "PRETIX_DATABASE_HOST" + value = data.aws_db_instance.database.address + }, + { + name = "PRETIX_DATABASE_PORT" + value = "5432" + }, + { + name = "PRETIX_MAIL_USER" + value = module.secrets.value.mail_user + }, + { + name = "PRETIX_MAIL_PASSWORD" + value = module.secrets.value.mail_password + }, + { + name = "PRETIX_MAIL_HOST" + value = "email-smtp.eu-central-1.amazonaws.com" + }, + { + name = "PRETIX_MAIL_PORT" + value = "587" + }, + { + name = "PRETIX_MAIL_TLS" + value = "true" + }, + { + name = "PRETIX_MAIL_SSL" + value = "false" + }, + { + name = "PRETIX_MAIL_FROM" + value = "noreply@pycon.it" + }, + { + name = "PRETIX_PRETIX_TRUST_X_FORWARDED_HOST" + value = "true" + }, + { + name = "PRETIX_PRETIX_REGISTRATION" + value = "true" + }, + { + name = "PRETIX_SENTRY_DSN" + value = module.secrets.value.sentry_dsn + }, + { + name = "PRETIX_DJANGO_SECRET" + value = module.secrets.value.secret_key + }, + { + name = "PRETIX_REDIS_LOCATION", + value = "redis://${data.aws_instance.redis.private_ip}/0" + }, + { + name = "PRETIX_REDIS_SESSIONS", + value = "false" + }, + { + name = "PRETIX_CELERY_BROKER", + value = "redis://${data.aws_instance.redis.private_ip}/1" + }, + { + name = "PRETIX_CELERY_BACKEND", + value = "redis://${data.aws_instance.redis.private_ip}/2" + }, + { + name = "PRETIX_PRETIX_URL", + value = "https://tickets.pycon.it/" + }, + { + name = "PRETIX_PRETIX_TRUST_X_FORWARDED_PROTO", + value = "true" + }, + { + # this is is needed for our hack that updates the order view + # without having to rewrite the whole template + name = "PRETIX_PRETIX_CSP_ADDITIONAL_HEADER", + value = "script-src 'self' 'unsafe-inline'" + }, + { + name = "PRETIX_PRETIX_INSTANCE_NAME", + value = "Python Italia" + }, + { + name = "DJANGO_SETTINGS_MODULE", + value = "production_settings" + }, + { + name = "DATA_DIR", + value = "/data/" + }, + { + name = "HOME", + value = "/pretix" + }, + { + name = "PRETIX_PYCON_STORAGE_BUCKET_NAME", + value = aws_s3_bucket.media.bucket + } + ] } data "aws_ecs_cluster" "server" { cluster_name = "${terraform.workspace}-server" @@ -130,7 +156,7 @@ resource "aws_ecs_task_definition" "pretix_web" { image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" memoryReservation = 200 essential = true - environment = local.env_vars + environment = local.env_vars portMappings = [ { containerPort = 8000 @@ -138,25 +164,19 @@ resource "aws_ecs_task_definition" "pretix_web" { } ] - entrypoint = ["gunicorn"] - command = ["pretix.wsgi", "--name=pretix", "--bind=0.0.0.0:8000", "--max-requests=1200", "--max-requests-jitter=50"] - workingDirectory = "/var/pretix" - user = "pretixuser" + entrypoint = ["gunicorn"] + command = [ + "pretix.wsgi", "--name=pretix", "--bind=0.0.0.0:8000", "--max-requests=1200", "--max-requests-jitter=50", + "--workers=4" + ] + workingDirectory = "/pretix/src" + user = "pretixuser" dockerLabels = { - "traefik.enable" = "true" - "traefik.http.routers.backend.rule" = "Host(`tickets.pycon.it`)" + "traefik.enable" = "true" + "traefik.http.routers.pretix-web.rule" = "Host(`tickets.pycon.it`)" } - # mountPoints = [ - # { - # sourceVolume = "media" - # containerPath = "/data/media" - # }, - # { - # sourceVolume = "data" - # containerPath = "/var/pretix-data" - # } - # ] + systemControls = [ { "namespace" : "net.core.somaxconn", @@ -174,16 +194,6 @@ resource "aws_ecs_task_definition" "pretix_web" { }, ]) - # volume { - # name = "media" - # host_path = "/var/pretix/data/media" - # } - - # volume { - # name = "data" - # host_path = "/var/pretix-data" - # } - requires_compatibilities = [] tags = {} } diff --git a/infrastructure/applications/pycon_backend/server.tf b/infrastructure/applications/pycon_backend/task_web.tf similarity index 96% rename from infrastructure/applications/pycon_backend/server.tf rename to infrastructure/applications/pycon_backend/task_web.tf index 7098c74de1..f465d69ee5 100644 --- a/infrastructure/applications/pycon_backend/server.tf +++ b/infrastructure/applications/pycon_backend/task_web.tf @@ -26,7 +26,7 @@ resource "aws_ecs_task_definition" "backend" { dockerLabels = { "traefik.enable" = "true" - "traefik.http.routers.backend.rule" = "Host(`${local.admin_domain}`)" + "traefik.http.routers.backend-web.rule" = "Host(`${local.admin_domain}`)" } environment = local.env_vars diff --git a/pretix/settings.py b/pretix/settings.py index c923e38787..196e03eb8b 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -1,5 +1,5 @@ from pretix.settings import * # noqa -from pretix.settings import INSTALLED_APPS, ALL_LANGUAGES, LOGGING +from pretix.settings import INSTALLED_APPS, ALL_LANGUAGES, LOGGING, STORAGES, config LOGGING["handlers"]["mail_admins"]["include_html"] = True # noqa @@ -8,12 +8,10 @@ LANGUAGES = [(k, v) for k, v in ALL_LANGUAGES] # noqa EMAIL_SUBJECT_PREFIX = "[PyCon Tickets] " -PRETIX_INSTANCE_NAME = "Python Italia" - -# this is is needed for our hack that updates the order view -# without having to rewrite the whole template -CSP_ADDITIONAL_HEADER = "script-src 'self' 'unsafe-inline'" if "pretix_fattura_elettronica" in INSTALLED_APPS: # noqa INSTALLED_APPS.remove("pretix_fattura_elettronica") # noqa INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa + +STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" +AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name") From 6f051b8e4b23356cdbb489320473fcc78f196a23 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 12:42:52 +0200 Subject: [PATCH 34/46] ee --- pretix/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretix/settings.py b/pretix/settings.py index 196e03eb8b..eb954351fc 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -14,4 +14,4 @@ INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" -AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name") +AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", "") From 335ab9825332fc0682c4ff8fbfe1cc192396871b Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 12:48:52 +0200 Subject: [PATCH 35/46] ee --- pretix/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretix/settings.py b/pretix/settings.py index eb954351fc..d730c6aafa 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -14,4 +14,4 @@ INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" -AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", "") +AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", fallback="") From ec5e2528d38448bfbc74322bbf799ce134bb123c Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 16:16:48 +0200 Subject: [PATCH 36/46] ee --- infrastructure/applications/applications.tf | 5 ++ .../applications/pretix_arm/main.tf | 0 .../applications/pretix_arm/task_web.tf | 9 +- .../applications/pycon_backend/cdn.tf | 7 ++ .../applications/pycon_backend/main.tf | 84 +------------------ .../applications/pycon_backend/task_web.tf | 2 +- .../applications/server/cloudfront.tf | 77 +++++++++++++++++ infrastructure/applications/server/domains.tf | 27 ++++++ .../applications/server/providers.tf | 8 ++ pretix/settings.py | 1 + 10 files changed, 133 insertions(+), 87 deletions(-) delete mode 100644 infrastructure/applications/pretix_arm/main.tf create mode 100644 infrastructure/applications/server/cloudfront.tf create mode 100644 infrastructure/applications/server/domains.tf create mode 100644 infrastructure/applications/server/providers.tf diff --git a/infrastructure/applications/applications.tf b/infrastructure/applications/applications.tf index d28f390fb7..9a6e2f08f2 100644 --- a/infrastructure/applications/applications.tf +++ b/infrastructure/applications/applications.tf @@ -54,6 +54,11 @@ module "emails" { module "server" { source = "./server" ecs_arm_ami = local.ecs_arm_ami + + providers = { + aws = aws + aws.us = aws.us + } } module "pretix_arm" { diff --git a/infrastructure/applications/pretix_arm/main.tf b/infrastructure/applications/pretix_arm/main.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 9afbe8a63c..9dc79c571a 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -1,4 +1,6 @@ locals { + domain = local.is_prod ? "tickets.pycon.it" : "${terraform.workspace}-tickets.pycon.it" + env_vars = [ { name = "VIRTUAL_ENV", @@ -94,7 +96,7 @@ locals { }, { name = "PRETIX_PRETIX_URL", - value = "https://tickets.pycon.it/" + value = "https://${local.domain}/" }, { name = "PRETIX_PRETIX_TRUST_X_FORWARDED_PROTO", @@ -173,8 +175,9 @@ resource "aws_ecs_task_definition" "pretix_web" { user = "pretixuser" dockerLabels = { - "traefik.enable" = "true" - "traefik.http.routers.pretix-web.rule" = "Host(`tickets.pycon.it`)" + "traefik.enable" = "true" + "traefik.http.routers.pretix-web.rule" = "Host(`${local.domain}`)" + "traefik.http.routers.pretix-web.service" = "pretix-web" } systemControls = [ diff --git a/infrastructure/applications/pycon_backend/cdn.tf b/infrastructure/applications/pycon_backend/cdn.tf index 028cf1488c..c4341e2bd5 100644 --- a/infrastructure/applications/pycon_backend/cdn.tf +++ b/infrastructure/applications/pycon_backend/cdn.tf @@ -6,6 +6,13 @@ data "aws_cloudfront_cache_policy" "caching_optimized" { name = "Managed-CachingOptimized" } +data "aws_acm_certificate" "cert" { + domain = "*.pycon.it" + statuses = ["ISSUED"] + provider = aws.us +} + + resource "aws_cloudfront_distribution" "media_cdn" { enabled = true is_ipv6_enabled = true diff --git a/infrastructure/applications/pycon_backend/main.tf b/infrastructure/applications/pycon_backend/main.tf index cbc8f2e378..3f4c3a9567 100644 --- a/infrastructure/applications/pycon_backend/main.tf +++ b/infrastructure/applications/pycon_backend/main.tf @@ -1,8 +1,8 @@ locals { is_prod = terraform.workspace == "production" - admin_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" db_connection = var.enable_proxy ? "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_proxy.proxy[0].endpoint}:${data.aws_db_instance.database.port}/pycon" : "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" cdn_url = local.is_prod ? "cdn.pycon.it" : "${terraform.workspace}-cdn.pycon.it" + web_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" } data "aws_vpc" "default" { @@ -44,12 +44,6 @@ data "aws_db_proxy" "proxy" { name = "pythonit-${terraform.workspace}-database-proxy" } -data "aws_acm_certificate" "cert" { - domain = "*.pycon.it" - statuses = ["ISSUED"] - provider = aws.us -} - data "aws_lambda_function" "forward_host_header" { function_name = "forward_host_header" provider = aws.us @@ -124,79 +118,3 @@ module "lambda" { AWS_SES_CONFIGURATION_SET = data.aws_sesv2_configuration_set.main.configuration_set_name } } - -data "aws_instance" "server" { - instance_tags = { - Name = "pythonit-${terraform.workspace}-server" - } - - filter { - name = "instance-state-name" - values = ["running"] - } -} - -data "aws_cloudfront_origin_request_policy" "all_viewer" { - name = "Managed-AllViewer" -} - -data "aws_cloudfront_cache_policy" "caching_disabled" { - name = "Managed-CachingDisabled" -} - -resource "aws_cloudfront_distribution" "application" { - enabled = true - is_ipv6_enabled = true - comment = "${terraform.workspace}-${local.application}" - wait_for_deployment = false - aliases = [local.admin_domain] - - origin { - domain_name = data.aws_instance.server.public_dns - origin_id = "default" - - custom_origin_config { - origin_protocol_policy = "http-only" - http_port = "80" - https_port = "443" - origin_ssl_protocols = ["TLSv1"] - } - } - - viewer_certificate { - cloudfront_default_certificate = false - minimum_protocol_version = "TLSv1" - ssl_support_method = "sni-only" - acm_certificate_arn = data.aws_acm_certificate.cert.arn - } - - default_cache_behavior { - allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] - cached_methods = ["GET", "HEAD"] - target_origin_id = "default" - - cache_policy_id = data.aws_cloudfront_cache_policy.caching_disabled.id - origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id - - viewer_protocol_policy = "redirect-to-https" - compress = true - } - - restrictions { - geo_restriction { - restriction_type = "none" - } - } -} - -resource "aws_route53_record" "record" { - zone_id = data.aws_route53_zone.pycon_zone.zone_id - name = local.admin_domain - type = "A" - - alias { - name = aws_cloudfront_distribution.application.domain_name - zone_id = aws_cloudfront_distribution.application.hosted_zone_id - evaluate_target_health = false - } -} diff --git a/infrastructure/applications/pycon_backend/task_web.tf b/infrastructure/applications/pycon_backend/task_web.tf index f465d69ee5..258d3e2748 100644 --- a/infrastructure/applications/pycon_backend/task_web.tf +++ b/infrastructure/applications/pycon_backend/task_web.tf @@ -26,7 +26,7 @@ resource "aws_ecs_task_definition" "backend" { dockerLabels = { "traefik.enable" = "true" - "traefik.http.routers.backend-web.rule" = "Host(`${local.admin_domain}`)" + "traefik.http.routers.backend-web.rule" = "Host(`${local.web_domain}`)" } environment = local.env_vars diff --git a/infrastructure/applications/server/cloudfront.tf b/infrastructure/applications/server/cloudfront.tf new file mode 100644 index 0000000000..9e24120b03 --- /dev/null +++ b/infrastructure/applications/server/cloudfront.tf @@ -0,0 +1,77 @@ +locals { + pycon_web_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" + pretix_web_domain = local.is_prod ? "tickets.pycon.it" : "${terraform.workspace}-tickets.pycon.it" +} + +data "aws_instance" "server" { + instance_tags = { + Name = "pythonit-${terraform.workspace}-server" + } + + filter { + name = "instance-state-name" + values = ["running"] + } +} + +data "aws_cloudfront_origin_request_policy" "all_viewer" { + name = "Managed-AllViewer" +} + +data "aws_cloudfront_cache_policy" "caching_disabled" { + name = "Managed-CachingDisabled" +} + +data "aws_acm_certificate" "cert" { + domain = "*.pycon.it" + statuses = ["ISSUED"] + provider = aws.us +} + +resource "aws_cloudfront_distribution" "application" { + enabled = true + is_ipv6_enabled = true + comment = "${terraform.workspace} server" + wait_for_deployment = false + aliases = [ + local.pycon_web_domain, + local.pretix_web_domain + ] + + origin { + domain_name = data.aws_instance.server.public_dns + origin_id = "default" + + custom_origin_config { + origin_protocol_policy = "http-only" + http_port = "80" + https_port = "443" + origin_ssl_protocols = ["TLSv1"] + } + } + + viewer_certificate { + cloudfront_default_certificate = false + minimum_protocol_version = "TLSv1" + ssl_support_method = "sni-only" + acm_certificate_arn = data.aws_acm_certificate.cert.arn + } + + default_cache_behavior { + allowed_methods = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] + cached_methods = ["GET", "HEAD"] + target_origin_id = "default" + + cache_policy_id = data.aws_cloudfront_cache_policy.caching_disabled.id + origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id + + viewer_protocol_policy = "redirect-to-https" + compress = true + } + + restrictions { + geo_restriction { + restriction_type = "none" + } + } +} diff --git a/infrastructure/applications/server/domains.tf b/infrastructure/applications/server/domains.tf new file mode 100644 index 0000000000..f4d48a85ff --- /dev/null +++ b/infrastructure/applications/server/domains.tf @@ -0,0 +1,27 @@ +data "aws_route53_zone" "pyconit" { + name = "pycon.it" +} + +resource "aws_route53_record" "pycon_web" { + zone_id = data.aws_route53_zone.pyconit.zone_id + name = local.pycon_web_domain + type = "A" + + alias { + name = aws_cloudfront_distribution.application.domain_name + zone_id = aws_cloudfront_distribution.application.hosted_zone_id + evaluate_target_health = false + } +} + +resource "aws_route53_record" "pretix_web" { + zone_id = data.aws_route53_zone.pyconit.zone_id + name = local.pretix_web_domain + type = "A" + + alias { + name = aws_cloudfront_distribution.application.domain_name + zone_id = aws_cloudfront_distribution.application.hosted_zone_id + evaluate_target_health = false + } +} diff --git a/infrastructure/applications/server/providers.tf b/infrastructure/applications/server/providers.tf new file mode 100644 index 0000000000..2ec15ed060 --- /dev/null +++ b/infrastructure/applications/server/providers.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + configuration_aliases = [aws.us] + } + } +} diff --git a/pretix/settings.py b/pretix/settings.py index d730c6aafa..9bec5933df 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -14,4 +14,5 @@ INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" +STORAGES["staticfiles"]["BACKEND"] = "storages.backends.s3.S3Storage" AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", fallback="") From 5ac736ff31ac92cb0c4953164a4b306a1abe7e91 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 16:17:23 +0200 Subject: [PATCH 37/46] ee --- pretix/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 4811e944b5..d45e099c94 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -5,8 +5,8 @@ COPY ./settings.py /pretix/src/production_settings.py RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-certificate==0.1.11 pretix-fattura-elettronica==0.2.15 -RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles -RUN cd /pretix/src && DATABASE_HOST=demo make compress +# RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles +# RUN cd /pretix/src && DATABASE_HOST=demo make compress ENTRYPOINT ["pretix"] CMD [ "all" ] From 6734180b57e5ac134bad44238672035c973d3548 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 19:23:35 +0200 Subject: [PATCH 38/46] ee --- pretix/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pretix/Dockerfile b/pretix/Dockerfile index d45e099c94..f1fdbde0e6 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -3,7 +3,10 @@ FROM ${PRETIX_IMAGE} COPY ./settings.py /pretix/src/production_settings.py -RUN pip install pretix-plugin-extended-api==0.1.12 pretix-plugin-attendance-certificate==0.1.11 pretix-fattura-elettronica==0.2.15 +RUN pip install django-storages==1.14.4 \ + pretix-plugin-extended-api==0.1.12 \ + pretix-plugin-attendance-certificate==0.1.11 \ + pretix-fattura-elettronica==0.2.15 # RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles # RUN cd /pretix/src && DATABASE_HOST=demo make compress From e09e892225dc05999a345493620367545c6e49df Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 19:42:41 +0200 Subject: [PATCH 39/46] ee --- pretix/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pretix/Dockerfile b/pretix/Dockerfile index f1fdbde0e6..58649d72c7 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -3,7 +3,8 @@ FROM ${PRETIX_IMAGE} COPY ./settings.py /pretix/src/production_settings.py -RUN pip install django-storages==1.14.4 \ +RUN pip install boto3==1.35.14 \ + django-storages==1.14.4 \ pretix-plugin-extended-api==0.1.12 \ pretix-plugin-attendance-certificate==0.1.11 \ pretix-fattura-elettronica==0.2.15 From 111864326af003e8310cdec811ea601814507f56 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 20:26:52 +0200 Subject: [PATCH 40/46] ee --- infrastructure/applications/pretix_arm/task_web.tf | 1 - infrastructure/applications/server/role.tf | 2 ++ pretix/settings.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 9dc79c571a..7097240895 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -177,7 +177,6 @@ resource "aws_ecs_task_definition" "pretix_web" { dockerLabels = { "traefik.enable" = "true" "traefik.http.routers.pretix-web.rule" = "Host(`${local.domain}`)" - "traefik.http.routers.pretix-web.service" = "pretix-web" } systemControls = [ diff --git a/infrastructure/applications/server/role.tf b/infrastructure/applications/server/role.tf index 291d983f72..faf95b52cd 100644 --- a/infrastructure/applications/server/role.tf +++ b/infrastructure/applications/server/role.tf @@ -38,6 +38,8 @@ data "aws_iam_policy_document" "server_role_policy" { resources = [ "arn:aws:s3:::${terraform.workspace}-pycon-backend-media", "arn:aws:s3:::${terraform.workspace}-pycon-backend-media/*", + "arn:aws:s3:::${terraform.workspace}-pretix-media", + "arn:aws:s3:::${terraform.workspace}-pretix-media/*", ] } } diff --git a/pretix/settings.py b/pretix/settings.py index 9bec5933df..e2b7a6f885 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -16,3 +16,4 @@ STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" STORAGES["staticfiles"]["BACKEND"] = "storages.backends.s3.S3Storage" AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", fallback="") +COMPRESS_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.eu-central-1.amazonaws.com/" From 54e17f1597c41db513e6a443e9bd29ae8eea2b3f Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Sun, 8 Sep 2024 20:44:34 +0200 Subject: [PATCH 41/46] ee --- infrastructure/applications/pretix_arm/task_web.tf | 10 ++++------ pretix/Dockerfile | 4 ++-- pretix/settings.py | 2 -- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index 7097240895..a569703ffe 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -161,16 +161,14 @@ resource "aws_ecs_task_definition" "pretix_web" { environment = local.env_vars portMappings = [ { - containerPort = 8000 + containerPort = 80 hostPort = 0 } ] - entrypoint = ["gunicorn"] - command = [ - "pretix.wsgi", "--name=pretix", "--bind=0.0.0.0:8000", "--max-requests=1200", "--max-requests-jitter=50", - "--workers=4" - ] + entrypoint = ["pretix"] + command = ["web"] + workingDirectory = "/pretix/src" user = "pretixuser" diff --git a/pretix/Dockerfile b/pretix/Dockerfile index 58649d72c7..ab8a0f1c4b 100644 --- a/pretix/Dockerfile +++ b/pretix/Dockerfile @@ -9,8 +9,8 @@ RUN pip install boto3==1.35.14 \ pretix-plugin-attendance-certificate==0.1.11 \ pretix-fattura-elettronica==0.2.15 -# RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles -# RUN cd /pretix/src && DATABASE_HOST=demo make compress +RUN cd /pretix/src && DATABASE_HOST=demo make staticfiles +RUN cd /pretix/src && DATABASE_HOST=demo make compress ENTRYPOINT ["pretix"] CMD [ "all" ] diff --git a/pretix/settings.py b/pretix/settings.py index e2b7a6f885..d730c6aafa 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -14,6 +14,4 @@ INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" -STORAGES["staticfiles"]["BACKEND"] = "storages.backends.s3.S3Storage" AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", fallback="") -COMPRESS_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.s3.eu-central-1.amazonaws.com/" From 9ea7654b484873c7e0d90e3790b0f1ad1d305796 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 9 Sep 2024 00:13:22 +0200 Subject: [PATCH 42/46] ee --- backend/Dockerfile | 2 +- .../applications/pretix_arm/task_web.tf | 10 + .../applications/pretix_arm/task_worker.tf | 213 +++++++----------- .../applications/pycon_backend/main.tf | 14 +- .../applications/pycon_backend/providers.tf | 1 - .../applications/pycon_backend/task_web.tf | 4 +- .../{worker.tf => task_worker.tf} | 16 +- .../pycon_backend/worker_heavy_processing.tf | 12 +- .../applications/server/cloudfront.tf | 4 +- infrastructure/applications/server/db.tf | 2 +- infrastructure/applications/server/ecs.tf | 2 +- infrastructure/applications/server/main.tf | 36 +-- .../applications/server/security.tf | 9 - .../applications/server/task_traefik.tf | 8 +- pretix/settings.py | 10 +- 15 files changed, 152 insertions(+), 191 deletions(-) rename infrastructure/applications/pycon_backend/{worker.tf => task_worker.tf} (97%) diff --git a/backend/Dockerfile b/backend/Dockerfile index 8de8cb6d9d..fbb620be7e 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update -y && apt-get install -y \ libtiff5-dev libjpeg62 libopenjp2-7-dev zlib1g-dev \ libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \ libharfbuzz-dev libfribidi-dev libxcb1-dev libldap2-dev libldap-2.5-0 \ - ffmpeg libsm6 libxext6 libglib2.0-0 + ffmpeg libsm6 libxext6 libglib2.0-0 curl ENV LIBRARY_PATH=/lib:/usr/lib diff --git a/infrastructure/applications/pretix_arm/task_web.tf b/infrastructure/applications/pretix_arm/task_web.tf index a569703ffe..216dc3ac1f 100644 --- a/infrastructure/applications/pretix_arm/task_web.tf +++ b/infrastructure/applications/pretix_arm/task_web.tf @@ -177,6 +177,16 @@ resource "aws_ecs_task_definition" "pretix_web" { "traefik.http.routers.pretix-web.rule" = "Host(`${local.domain}`)" } + healthCheck = { + retries = 3 + command = [ + "CMD-SHELL", + "curl -f http://localhost/healthcheck/ || exit 1" + ] + timeout = 3 + interval = 10 + } + systemControls = [ { "namespace" : "net.core.somaxconn", diff --git a/infrastructure/applications/pretix_arm/task_worker.tf b/infrastructure/applications/pretix_arm/task_worker.tf index 1847079956..177c2af5c5 100644 --- a/infrastructure/applications/pretix_arm/task_worker.tf +++ b/infrastructure/applications/pretix_arm/task_worker.tf @@ -1,131 +1,92 @@ -# resource "aws_ecs_task_definition" "pretix_web" { -# family = "pythonit-${terraform.workspace}-pretix" -# container_definitions = jsonencode([ -# { -# name = "pretix" -# image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" -# memoryReservation = 200 -# essential = true -# environment = [ -# { -# name = "DATABASE_NAME" -# value = "pretix" -# }, -# { -# name = "DATABASE_USERNAME" -# value = data.aws_db_instance.database.master_username -# }, -# { -# name = "DATABASE_PASSWORD" -# value = module.common_secrets.value.database_password -# }, -# { -# name = "DATABASE_HOST" -# value = data.aws_db_instance.database.address -# }, -# { -# name = "MAIL_USER" -# value = module.secrets.value.mail_user -# }, -# { -# name = "MAIL_PASSWORD" -# value = module.secrets.value.mail_password -# }, -# { -# name = "PRETIX_SENTRY_DSN" -# value = module.secrets.value.sentry_dsn -# }, -# { -# name = "SECRET_KEY" -# value = module.secrets.value.secret_key -# }, -# { -# name = "PRETIX_REDIS_LOCATION", -# value = "redis://${data.aws_instance.redis.private_ip}/0" -# }, -# { -# name = "PRETIX_REDIS_SESSIONS", -# value = "false" -# }, -# { -# name = "PRETIX_CELERY_BROKER", -# value = "redis://${data.aws_instance.redis.private_ip}/1" -# }, -# { -# name = "PRETIX_CELERY_BACKEND", -# value = "redis://${data.aws_instance.redis.private_ip}/2" -# }, -# { -# name = "PRETIX_PRETIX_URL", -# value = "https://tickets.pycon.it/" -# }, -# { -# name = "PRETIX_PRETIX_TRUST_X_FORWARDED_PROTO", -# value = "true" -# } -# ] -# portMappings = [ -# { -# containerPort = 80 -# hostPort = 0 -# } -# ] -# dockerLabels = { -# "traefik.enable" = "true" -# "traefik.http.routers.backend.rule" = "Host(`tickets.pycon.it`)" -# } -# mountPoints = [ -# { -# sourceVolume = "media" -# containerPath = "/data/media" -# }, -# { -# sourceVolume = "data" -# containerPath = "/var/pretix-data" -# } -# ] -# systemControls = [ -# { -# "namespace" : "net.core.somaxconn", -# "value" : "4096" -# } -# ] -# logConfiguration = { -# logDriver = "awslogs" -# options = { -# "awslogs-group" = aws_cloudwatch_log_group.pretix.name -# "awslogs-region" = "eu-central-1" -# "awslogs-stream-prefix" = "ecs" -# } -# } -# }, -# ]) +resource "aws_cloudwatch_log_group" "pretix_worker" { + name = "/ecs/pythonit-${terraform.workspace}-pretix-worker" + retention_in_days = 7 +} -# volume { -# name = "media" -# host_path = "/var/pretix/data/media" -# } +resource "aws_ecs_task_definition" "pretix_worker" { + family = "pythonit-${terraform.workspace}-pretix-worker" + container_definitions = jsonencode([ + { + name = "worker" + image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" + memoryReservation = 200 + essential = true + environment = local.env_vars -# volume { -# name = "data" -# host_path = "/var/pretix-data" -# } + entrypoint = ["pretix"] + command = ["taskworker"] -# requires_compatibilities = [] -# tags = {} -# } + workingDirectory = "/pretix/src" + user = "pretixuser" -# resource "aws_ecs_service" "pretix_web" { -# name = "pretix-worker" -# cluster = data.aws_ecs_cluster.server.id -# task_definition = aws_ecs_task_definition.pretix_web.arn -# desired_count = 1 -# deployment_minimum_healthy_percent = 100 -# deployment_maximum_percent = 200 + healthCheck = { + retries = 3 + command = [ + "CMD-SHELL", + "celery -A pretix.celery_app inspect ping" + ] + timeout = 3 + interval = 10 + } -# lifecycle { -# ignore_changes = [ -# capacity_provider_strategy -# ] -# } -# } + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.pretix_worker.name + "awslogs-region" = "eu-central-1" + "awslogs-stream-prefix" = "ecs" + } + } + }, + { + name = "cron" + image = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}" + memoryReservation = 200 + essential = true + environment = local.env_vars + + entrypoint = ["bash", "-c"] + command = ["while true; do pretix cron; sleep 60; done"] + + healthCheck = { + retries = 3 + command = [ + "CMD-SHELL", + "echo 1" + ] + timeout = 3 + interval = 10 + } + + workingDirectory = "/pretix/src" + user = "pretixuser" + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.pretix_worker.name + "awslogs-region" = "eu-central-1" + "awslogs-stream-prefix" = "ecs" + } + } + }, + ]) + + requires_compatibilities = [] + tags = {} +} + +resource "aws_ecs_service" "pretix_worker" { + name = "pretix-worker" + cluster = data.aws_ecs_cluster.server.id + task_definition = aws_ecs_task_definition.pretix_worker.arn + desired_count = 1 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + + lifecycle { + ignore_changes = [ + capacity_provider_strategy + ] + } +} diff --git a/infrastructure/applications/pycon_backend/main.tf b/infrastructure/applications/pycon_backend/main.tf index 3f4c3a9567..5485efe214 100644 --- a/infrastructure/applications/pycon_backend/main.tf +++ b/infrastructure/applications/pycon_backend/main.tf @@ -1,8 +1,8 @@ locals { - is_prod = terraform.workspace == "production" - db_connection = var.enable_proxy ? "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_proxy.proxy[0].endpoint}:${data.aws_db_instance.database.port}/pycon" : "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" - cdn_url = local.is_prod ? "cdn.pycon.it" : "${terraform.workspace}-cdn.pycon.it" - web_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" + is_prod = terraform.workspace == "production" + db_connection = var.enable_proxy ? "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_proxy.proxy[0].endpoint}:${data.aws_db_instance.database.port}/pycon" : "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" + cdn_url = local.is_prod ? "cdn.pycon.it" : "${terraform.workspace}-cdn.pycon.it" + web_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" } data "aws_vpc" "default" { @@ -113,8 +113,8 @@ module "lambda" { CELERY_RESULT_BACKEND = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/6" : "redis://${data.aws_instance.redis.private_ip}/15" PLAIN_INTEGRATION_TOKEN = module.secrets.value.plain_integration_token HASHID_DEFAULT_SECRET_SALT = module.secrets.value.hashid_default_secret_salt - MEDIA_FILES_STORAGE_BACKEND = "pycon.storages.CustomS3Boto3Storage" - SNS_WEBHOOK_SECRET = module.common_secrets.value.sns_webhook_secret - AWS_SES_CONFIGURATION_SET = data.aws_sesv2_configuration_set.main.configuration_set_name + MEDIA_FILES_STORAGE_BACKEND = "pycon.storages.CustomS3Boto3Storage" + SNS_WEBHOOK_SECRET = module.common_secrets.value.sns_webhook_secret + AWS_SES_CONFIGURATION_SET = data.aws_sesv2_configuration_set.main.configuration_set_name } } diff --git a/infrastructure/applications/pycon_backend/providers.tf b/infrastructure/applications/pycon_backend/providers.tf index 53481f218f..2ec15ed060 100644 --- a/infrastructure/applications/pycon_backend/providers.tf +++ b/infrastructure/applications/pycon_backend/providers.tf @@ -2,7 +2,6 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.66.0" configuration_aliases = [aws.us] } } diff --git a/infrastructure/applications/pycon_backend/task_web.tf b/infrastructure/applications/pycon_backend/task_web.tf index 258d3e2748..3dbc15544f 100644 --- a/infrastructure/applications/pycon_backend/task_web.tf +++ b/infrastructure/applications/pycon_backend/task_web.tf @@ -25,7 +25,7 @@ resource "aws_ecs_task_definition" "backend" { ] dockerLabels = { - "traefik.enable" = "true" + "traefik.enable" = "true" "traefik.http.routers.backend-web.rule" = "Host(`${local.web_domain}`)" } @@ -58,7 +58,7 @@ resource "aws_ecs_task_definition" "backend" { retries = 3 command = [ "CMD-SHELL", - "echo 1" + "curl -f http://localhost:8000/health/ || exit 1" ] timeout = 3 interval = 10 diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/task_worker.tf similarity index 97% rename from infrastructure/applications/pycon_backend/worker.tf rename to infrastructure/applications/pycon_backend/task_worker.tf index b75e07e988..ebe1966df8 100644 --- a/infrastructure/applications/pycon_backend/worker.tf +++ b/infrastructure/applications/pycon_backend/task_worker.tf @@ -173,11 +173,11 @@ locals { value = module.secrets.value.hashid_default_secret_salt }, { - name = "MEDIA_FILES_STORAGE_BACKEND", + name = "MEDIA_FILES_STORAGE_BACKEND", value = "pycon.storages.CustomS3Boto3Storage" }, { - name = "CLAMAV_HOST", + name = "CLAMAV_HOST", value = module.secrets.value.clamav_host }, { @@ -192,15 +192,15 @@ locals { }) }, { - name = "ECS_SERVICE_ROLE", + name = "ECS_SERVICE_ROLE", value = aws_iam_role.ecs_service.arn }, { - name = "AWS_SES_CONFIGURATION_SET" + name = "AWS_SES_CONFIGURATION_SET" value = data.aws_sesv2_configuration_set.main.configuration_set_name }, { - name = "SNS_WEBHOOK_SECRET" + name = "SNS_WEBHOOK_SECRET" value = module.common_secrets.value.sns_webhook_secret } ] @@ -291,8 +291,8 @@ resource "aws_instance" "instance_1" { market_type = "spot" spot_options { - max_price = 0.0031 - spot_instance_type = "persistent" + max_price = 0.0031 + spot_instance_type = "persistent" instance_interruption_behavior = "stop" } } @@ -352,7 +352,7 @@ resource "aws_ecs_task_definition" "worker" { retries = 3 command = [ "CMD-SHELL", - "echo 1" + "celery -A pycon inspect ping" ] timeout = 3 interval = 10 diff --git a/infrastructure/applications/pycon_backend/worker_heavy_processing.tf b/infrastructure/applications/pycon_backend/worker_heavy_processing.tf index 5f1561219f..70f28bd250 100644 --- a/infrastructure/applications/pycon_backend/worker_heavy_processing.tf +++ b/infrastructure/applications/pycon_backend/worker_heavy_processing.tf @@ -13,20 +13,20 @@ resource "aws_cloudwatch_log_group" "heavy_processing_worker_logs" { } resource "aws_ecs_task_definition" "heavy_processing_worker" { - family = "pythonit-${terraform.workspace}-heavy-processing-worker" + family = "pythonit-${terraform.workspace}-heavy-processing-worker" requires_compatibilities = ["FARGATE"] cpu = 4096 memory = 16384 network_mode = "awsvpc" - execution_role_arn = aws_iam_role.worker.arn - task_role_arn = aws_iam_role.worker.arn + execution_role_arn = aws_iam_role.worker.arn + task_role_arn = aws_iam_role.worker.arn ephemeral_storage { size_in_gib = 21 } runtime_platform { operating_system_family = "LINUX" - cpu_architecture = "ARM64" + cpu_architecture = "ARM64" } container_definitions = jsonencode([ { @@ -83,9 +83,9 @@ resource "aws_ecs_task_definition" "heavy_processing_worker" { ]) volume { - name = "storage" + name = "storage" configure_at_launch = true } - tags = {} + tags = {} } diff --git a/infrastructure/applications/server/cloudfront.tf b/infrastructure/applications/server/cloudfront.tf index 9e24120b03..557ffb89fc 100644 --- a/infrastructure/applications/server/cloudfront.tf +++ b/infrastructure/applications/server/cloudfront.tf @@ -1,5 +1,5 @@ locals { - pycon_web_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" + pycon_web_domain = local.is_prod ? "admin.pycon.it" : "${terraform.workspace}-admin.pycon.it" pretix_web_domain = local.is_prod ? "tickets.pycon.it" : "${terraform.workspace}-tickets.pycon.it" } @@ -33,7 +33,7 @@ resource "aws_cloudfront_distribution" "application" { is_ipv6_enabled = true comment = "${terraform.workspace} server" wait_for_deployment = false - aliases = [ + aliases = [ local.pycon_web_domain, local.pretix_web_domain ] diff --git a/infrastructure/applications/server/db.tf b/infrastructure/applications/server/db.tf index 654129144f..b8086bf5e1 100644 --- a/infrastructure/applications/server/db.tf +++ b/infrastructure/applications/server/db.tf @@ -1,5 +1,5 @@ locals { - db_connection = "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" + db_connection = "postgres://${data.aws_db_instance.database.master_username}:${module.common_secrets.value.database_password}@${data.aws_db_instance.database.address}:${data.aws_db_instance.database.port}/pycon" } data "aws_db_instance" "database" { diff --git a/infrastructure/applications/server/ecs.tf b/infrastructure/applications/server/ecs.tf index 8ee260eaab..2cc3058d7b 100644 --- a/infrastructure/applications/server/ecs.tf +++ b/infrastructure/applications/server/ecs.tf @@ -14,7 +14,7 @@ resource "aws_ecs_capacity_provider" "server" { minimum_scaling_step_size = 1 status = "ENABLED" target_capacity = 1 - instance_warmup_period = 60 + instance_warmup_period = 60 } } } diff --git a/infrastructure/applications/server/main.tf b/infrastructure/applications/server/main.tf index c57170be49..7e0b2c013b 100644 --- a/infrastructure/applications/server/main.tf +++ b/infrastructure/applications/server/main.tf @@ -6,7 +6,7 @@ data "template_file" "server_user_data" { } data "aws_ami" "ecs" { - owners = ["self"] + owners = ["self"] filter { name = "image-id" @@ -15,15 +15,15 @@ data "aws_ami" "ecs" { } data "aws_security_group" "tempone" { - name = "pythonit-${terraform.workspace}-worker-instance" + name = "pythonit-${terraform.workspace}-worker-instance" } resource "aws_launch_template" "server" { - name = "pythonit-${terraform.workspace}-server" + name = "pythonit-${terraform.workspace}-server" image_id = var.ecs_arm_ami instance_type = "t4g.medium" - user_data = base64encode(data.template_file.server_user_data.rendered) - key_name = "pretix" + user_data = base64encode(data.template_file.server_user_data.rendered) + key_name = "pretix" iam_instance_profile { name = aws_iam_instance_profile.server.name @@ -41,30 +41,30 @@ resource "aws_launch_template" "server" { associate_public_ip_address = true security_groups = [ data.aws_security_group.rds.id, - data.aws_security_group.lambda.id, - data.aws_security_group.tempone.id, - aws_security_group.server.id, + data.aws_security_group.lambda.id, + data.aws_security_group.tempone.id, + aws_security_group.server.id, ] subnet_id = data.aws_subnet.public_1a.id } } resource "aws_autoscaling_group" "server" { - name = "pythonit-${terraform.workspace}-server" - vpc_zone_identifier = [data.aws_subnet.public_1a.id] - desired_capacity = 1 - max_size = 1 - min_size = 1 - termination_policies = ["OldestInstance"] + name = "pythonit-${terraform.workspace}-server" + vpc_zone_identifier = [data.aws_subnet.public_1a.id] + desired_capacity = 1 + max_size = 1 + min_size = 1 + termination_policies = ["OldestInstance"] protect_from_scale_in = true instance_refresh { strategy = "Rolling" preferences { - min_healthy_percentage = 100 - max_healthy_percentage = 110 + min_healthy_percentage = 100 + max_healthy_percentage = 110 scale_in_protected_instances = "Refresh" - instance_warmup = 30 + instance_warmup = 30 } } @@ -74,7 +74,7 @@ resource "aws_autoscaling_group" "server" { } tag { - key = "Name" + key = "Name" value = "pythonit-${terraform.workspace}-server" propagate_at_launch = true } diff --git a/infrastructure/applications/server/security.tf b/infrastructure/applications/server/security.tf index ae2fb1b588..8b8b953803 100644 --- a/infrastructure/applications/server/security.tf +++ b/infrastructure/applications/server/security.tf @@ -29,12 +29,3 @@ resource "aws_security_group_rule" "web_http" { cidr_blocks = ["0.0.0.0/0"] security_group_id = aws_security_group.server.id } - -resource "aws_security_group_rule" "web_dashboard" { - type = "ingress" - from_port = 8080 - to_port = 8080 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - security_group_id = aws_security_group.server.id -} diff --git a/infrastructure/applications/server/task_traefik.tf b/infrastructure/applications/server/task_traefik.tf index dcda123bd5..4ea05f61dc 100644 --- a/infrastructure/applications/server/task_traefik.tf +++ b/infrastructure/applications/server/task_traefik.tf @@ -14,19 +14,19 @@ resource "aws_ecs_task_definition" "traefik" { environment = [ { - name = "TRAEFIK_PROVIDERS_ECS_CLUSTERS" + name = "TRAEFIK_PROVIDERS_ECS_CLUSTERS" value = aws_ecs_cluster.server.name }, { - name = "TRAEFIK_PROVIDERS_ECS_AUTODISCOVERCLUSTERS" + name = "TRAEFIK_PROVIDERS_ECS_AUTODISCOVERCLUSTERS" value = "false", }, { - name = "TRAEFIK_PROVIDERS_ECS_EXPOSEDBYDEFAULT", + name = "TRAEFIK_PROVIDERS_ECS_EXPOSEDBYDEFAULT", value = "false", }, { - name = "TRAEFIK_ENTRYPOINTS_WEB_ADDRESS", + name = "TRAEFIK_ENTRYPOINTS_WEB_ADDRESS", value = ":80" }, ] diff --git a/pretix/settings.py b/pretix/settings.py index d730c6aafa..c3315213ff 100644 --- a/pretix/settings.py +++ b/pretix/settings.py @@ -1,17 +1,17 @@ from pretix.settings import * # noqa from pretix.settings import INSTALLED_APPS, ALL_LANGUAGES, LOGGING, STORAGES, config -LOGGING["handlers"]["mail_admins"]["include_html"] = True # noqa +LOGGING["handlers"]["mail_admins"]["include_html"] = True # Allow all the languages # see: pretix/settings.py#L425-L435 -LANGUAGES = [(k, v) for k, v in ALL_LANGUAGES] # noqa +LANGUAGES = [(k, v) for k, v in ALL_LANGUAGES] EMAIL_SUBJECT_PREFIX = "[PyCon Tickets] " -if "pretix_fattura_elettronica" in INSTALLED_APPS: # noqa - INSTALLED_APPS.remove("pretix_fattura_elettronica") # noqa - INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") # noqa +if "pretix_fattura_elettronica" in INSTALLED_APPS: + INSTALLED_APPS.remove("pretix_fattura_elettronica") + INSTALLED_APPS.insert(0, "pretix_fattura_elettronica") STORAGES["default"]["BACKEND"] = "storages.backends.s3.S3Storage" AWS_STORAGE_BUCKET_NAME = config.get("pycon", "storage_bucket_name", fallback="") From 6da5d2fa1697f3ca60d182ad89f3ee84d0a81745 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 9 Sep 2024 00:32:59 +0200 Subject: [PATCH 43/46] ee --- .github/workflows/build-images.yml | 156 ----------------------------- .github/workflows/deploy.yml | 2 +- 2 files changed, 1 insertion(+), 157 deletions(-) delete mode 100644 .github/workflows/build-images.yml diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml deleted file mode 100644 index 3de3d5bee6..0000000000 --- a/.github/workflows/build-images.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: Build images - -on: - workflow_call: - -jobs: - build-and-push-service: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - - strategy: - fail-fast: false - matrix: - service: - - name: pycon-backend - dir: backend - - name: pretix - dir: pretix - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-qemu-action@v3 - - name: Login to GitHub Packages - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-buildx-action@v3 - - name: Cache Docker layers - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.service.name }} - - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/build-push-action@v6 - with: - context: ./${{ matrix.service.dir }} - file: ./${{ matrix.service.dir }}/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} - ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/amd64 - - build-and-push-arm-service: - runs-on: [self-hosted] - permissions: - packages: write - contents: read - strategy: - fail-fast: false - matrix: - service: - - name: pycon-backend - dir: backend - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.aws_access_key_id }} - aws-secret-access-key: ${{ secrets.aws_secret_access_key }} - aws-region: eu-central-1 - - name: Get service githash - id: git - run: | - hash=$(git rev-list -1 HEAD -- ${{ matrix.service.dir }}) - echo "githash=$hash" >> $GITHUB_OUTPUT - - name: Check if commit is already on ECR - id: image - run: | - set +e - aws ecr describe-images --repository-name=pythonit/${{ matrix.service.name }} --image-ids=imageTag=arm-${{ steps.git.outputs.githash }} - if [[ $? == 0 ]]; then - echo "image_exists=1" >> $GITHUB_OUTPUT - else - echo "image_exists=0" >> $GITHUB_OUTPUT - fi - - name: Set up QEMU dependency - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-qemu-action@v3 - - name: Login to GitHub Packages - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Amazon ECR - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: aws-actions/amazon-ecr-login@v2 - - name: Set up Docker Buildx - id: buildx - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/setup-buildx-action@v3 - - name: Build and push - if: ${{ steps.image.outputs.image_exists == 0 }} - uses: docker/build-push-action@v6 - with: - context: ./${{ matrix.service.dir }} - file: ./${{ matrix.service.dir }}/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - provenance: false - push: true - tags: | - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.eu-central-1.amazonaws.com/pythonit/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} - ghcr.io/pythonitalia/pycon/${{ matrix.service.name }}:arm-${{ steps.git.outputs.githash }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - platforms: linux/arm64 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c872fc094b..dbd9bef036 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,7 +67,7 @@ jobs: - name: Get service githash id: git run: | - hash=$(git rev-list -1 HEAD -- ./pretix) + hash=$(git rev-list -1 HEAD -- pretix) echo "githash=$hash" >> $GITHUB_OUTPUT - name: Check if commit is already on ECR id: image From 62ff0441f06af3ee41f2d87c94686510ad6dc885 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 9 Sep 2024 00:34:08 +0200 Subject: [PATCH 44/46] ee --- .github/workflows/deploy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index dbd9bef036..da7d7dd677 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,6 +53,9 @@ jobs: build-pretix: runs-on: [self-hosted] steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} - uses: actions/checkout@v4 with: repository: pretix/pretix From 3f7ef70640fb6fd151c544a3f6a752f0dcbe7a63 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 9 Sep 2024 00:52:21 +0200 Subject: [PATCH 45/46] ee --- backend/Dockerfile | 4 +- .../applications/pycon_backend/main.tf | 106 +++++++++--------- .../applications/pycon_backend/queue.tf | 24 ++-- .../applications/pycon_backend/worker_repo.tf | 5 - 4 files changed, 68 insertions(+), 71 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index fbb620be7e..cf3ca721d0 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update -y && apt-get install -y \ libtiff5-dev libjpeg62 libopenjp2-7-dev zlib1g-dev \ libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \ libharfbuzz-dev libfribidi-dev libxcb1-dev libldap2-dev libldap-2.5-0 \ - ffmpeg libsm6 libxext6 libglib2.0-0 curl + ffmpeg libsm6 libxext6 libglib2.0-0 ENV LIBRARY_PATH=/lib:/usr/lib @@ -140,6 +140,8 @@ ARG FUNCTION_DIR WORKDIR ${FUNCTION_DIR} +RUN apt-get update -y && apt install -y curl + COPY --from=js-stage ${FUNCTION_DIR}/dist/*.html ${FUNCTION_DIR}/custom_admin/templates/astro/ COPY --from=js-stage ${FUNCTION_DIR}/dist/_astro ${FUNCTION_DIR}/custom_admin/static/_astro/ diff --git a/infrastructure/applications/pycon_backend/main.tf b/infrastructure/applications/pycon_backend/main.tf index 5485efe214..95472b7d92 100644 --- a/infrastructure/applications/pycon_backend/main.tf +++ b/infrastructure/applications/pycon_backend/main.tf @@ -64,57 +64,57 @@ data "aws_sesv2_configuration_set" "main" { configuration_set_name = "pythonit-${terraform.workspace}" } -module "lambda" { - source = "../../components/application_lambda" +# module "lambda" { +# source = "../../components/application_lambda" - application = local.application - local_path = local.local_path - role_arn = data.aws_iam_role.lambda.arn - subnet_ids = [for subnet in data.aws_subnets.private.ids : subnet] - security_group_ids = [data.aws_security_group.rds.id, data.aws_security_group.lambda.id] - env_vars = { - DATABASE_URL = local.db_connection - DEBUG = "False" - SECRET_KEY = module.secrets.value.secret_key - MAPBOX_PUBLIC_API_KEY = module.secrets.value.mapbox_public_api_key - SENTRY_DSN = module.secrets.value.sentry_dsn - VOLUNTEERS_PUSH_NOTIFICATIONS_IOS_ARN = module.secrets.value.volunteers_push_notifications_ios_arn - VOLUNTEERS_PUSH_NOTIFICATIONS_ANDROID_ARN = module.secrets.value.volunteers_push_notifications_android_arn - ALLOWED_HOSTS = ".pycon.it" - DJANGO_SETTINGS_MODULE = "pycon.settings.prod" - ASSOCIATION_FRONTEND_URL = "https://associazione.python.it" - AWS_MEDIA_BUCKET = aws_s3_bucket.backend_media.id - AWS_REGION_NAME = aws_s3_bucket.backend_media.region - SPEAKERS_EMAIL_ADDRESS = module.secrets.value.speakers_email_address - EMAIL_BACKEND = "django_ses.SESBackend" - PYTHONIT_EMAIL_BACKEND = "notifications.backends.ses.SESEmailBackend" - FRONTEND_URL = "https://pycon.it" - PRETIX_API = "https://tickets.pycon.it/api/v1/" - AWS_S3_CUSTOM_DOMAIN = local.cdn_url - PRETIX_API_TOKEN = module.common_secrets.value.pretix_api_token - PINPOINT_APPLICATION_ID = module.secrets.value.pinpoint_application_id - MAILCHIMP_SECRET_KEY = module.common_secrets.value.mailchimp_secret_key - MAILCHIMP_DC = module.common_secrets.value.mailchimp_dc - MAILCHIMP_LIST_ID = module.common_secrets.value.mailchimp_list_id - USER_ID_HASH_SALT = module.secrets.value.userid_hash_salt - AZURE_STORAGE_ACCOUNT_NAME = module.secrets.value.azure_storage_account_name - AZURE_STORAGE_ACCOUNT_KEY = module.secrets.value.azure_storage_account_key - PLAIN_API = "https://core-api.uk.plain.com/graphql/v1" - PLAIN_API_TOKEN = module.secrets.value.plain_api_token - CACHE_URL = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/8" : "redis://${data.aws_instance.redis.private_ip}/13" - STRIPE_WEBHOOK_SIGNATURE_SECRET = module.secrets.value.stripe_webhook_secret - STRIPE_SUBSCRIPTION_PRICE_ID = module.secrets.value.stripe_membership_price_id - STRIPE_SECRET_API_KEY = module.secrets.value.stripe_secret_api_key - PRETIX_WEBHOOK_SECRET = module.secrets.value.pretix_webhook_secret - DEEPL_AUTH_KEY = module.secrets.value.deepl_auth_key - FLODESK_API_KEY = module.secrets.value.flodesk_api_key - FLODESK_SEGMENT_ID = module.secrets.value.flodesk_segment_id - CELERY_BROKER_URL = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/5" : "redis://${data.aws_instance.redis.private_ip}/14" - CELERY_RESULT_BACKEND = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/6" : "redis://${data.aws_instance.redis.private_ip}/15" - PLAIN_INTEGRATION_TOKEN = module.secrets.value.plain_integration_token - HASHID_DEFAULT_SECRET_SALT = module.secrets.value.hashid_default_secret_salt - MEDIA_FILES_STORAGE_BACKEND = "pycon.storages.CustomS3Boto3Storage" - SNS_WEBHOOK_SECRET = module.common_secrets.value.sns_webhook_secret - AWS_SES_CONFIGURATION_SET = data.aws_sesv2_configuration_set.main.configuration_set_name - } -} +# application = local.application +# local_path = local.local_path +# role_arn = data.aws_iam_role.lambda.arn +# subnet_ids = [for subnet in data.aws_subnets.private.ids : subnet] +# security_group_ids = [data.aws_security_group.rds.id, data.aws_security_group.lambda.id] +# env_vars = { +# DATABASE_URL = local.db_connection +# DEBUG = "False" +# SECRET_KEY = module.secrets.value.secret_key +# MAPBOX_PUBLIC_API_KEY = module.secrets.value.mapbox_public_api_key +# SENTRY_DSN = module.secrets.value.sentry_dsn +# VOLUNTEERS_PUSH_NOTIFICATIONS_IOS_ARN = module.secrets.value.volunteers_push_notifications_ios_arn +# VOLUNTEERS_PUSH_NOTIFICATIONS_ANDROID_ARN = module.secrets.value.volunteers_push_notifications_android_arn +# ALLOWED_HOSTS = ".pycon.it" +# DJANGO_SETTINGS_MODULE = "pycon.settings.prod" +# ASSOCIATION_FRONTEND_URL = "https://associazione.python.it" +# AWS_MEDIA_BUCKET = aws_s3_bucket.backend_media.id +# AWS_REGION_NAME = aws_s3_bucket.backend_media.region +# SPEAKERS_EMAIL_ADDRESS = module.secrets.value.speakers_email_address +# EMAIL_BACKEND = "django_ses.SESBackend" +# PYTHONIT_EMAIL_BACKEND = "notifications.backends.ses.SESEmailBackend" +# FRONTEND_URL = "https://pycon.it" +# PRETIX_API = "https://tickets.pycon.it/api/v1/" +# AWS_S3_CUSTOM_DOMAIN = local.cdn_url +# PRETIX_API_TOKEN = module.common_secrets.value.pretix_api_token +# PINPOINT_APPLICATION_ID = module.secrets.value.pinpoint_application_id +# MAILCHIMP_SECRET_KEY = module.common_secrets.value.mailchimp_secret_key +# MAILCHIMP_DC = module.common_secrets.value.mailchimp_dc +# MAILCHIMP_LIST_ID = module.common_secrets.value.mailchimp_list_id +# USER_ID_HASH_SALT = module.secrets.value.userid_hash_salt +# AZURE_STORAGE_ACCOUNT_NAME = module.secrets.value.azure_storage_account_name +# AZURE_STORAGE_ACCOUNT_KEY = module.secrets.value.azure_storage_account_key +# PLAIN_API = "https://core-api.uk.plain.com/graphql/v1" +# PLAIN_API_TOKEN = module.secrets.value.plain_api_token +# CACHE_URL = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/8" : "redis://${data.aws_instance.redis.private_ip}/13" +# STRIPE_WEBHOOK_SIGNATURE_SECRET = module.secrets.value.stripe_webhook_secret +# STRIPE_SUBSCRIPTION_PRICE_ID = module.secrets.value.stripe_membership_price_id +# STRIPE_SECRET_API_KEY = module.secrets.value.stripe_secret_api_key +# PRETIX_WEBHOOK_SECRET = module.secrets.value.pretix_webhook_secret +# DEEPL_AUTH_KEY = module.secrets.value.deepl_auth_key +# FLODESK_API_KEY = module.secrets.value.flodesk_api_key +# FLODESK_SEGMENT_ID = module.secrets.value.flodesk_segment_id +# CELERY_BROKER_URL = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/5" : "redis://${data.aws_instance.redis.private_ip}/14" +# CELERY_RESULT_BACKEND = local.is_prod ? "redis://${data.aws_instance.redis.private_ip}/6" : "redis://${data.aws_instance.redis.private_ip}/15" +# PLAIN_INTEGRATION_TOKEN = module.secrets.value.plain_integration_token +# HASHID_DEFAULT_SECRET_SALT = module.secrets.value.hashid_default_secret_salt +# MEDIA_FILES_STORAGE_BACKEND = "pycon.storages.CustomS3Boto3Storage" +# SNS_WEBHOOK_SECRET = module.common_secrets.value.sns_webhook_secret +# AWS_SES_CONFIGURATION_SET = data.aws_sesv2_configuration_set.main.configuration_set_name +# } +# } diff --git a/infrastructure/applications/pycon_backend/queue.tf b/infrastructure/applications/pycon_backend/queue.tf index 0524f5d305..302d33243f 100644 --- a/infrastructure/applications/pycon_backend/queue.tf +++ b/infrastructure/applications/pycon_backend/queue.tf @@ -1,14 +1,14 @@ -resource "aws_sqs_queue" "queue" { - name = "${terraform.workspace}-pycon-backend.fifo" - fifo_queue = true - visibility_timeout_seconds = local.is_prod ? 60 * 30 : 60 * 5 +# resource "aws_sqs_queue" "queue" { +# name = "${terraform.workspace}-pycon-backend.fifo" +# fifo_queue = true +# visibility_timeout_seconds = local.is_prod ? 60 * 30 : 60 * 5 - tags = { - Env = terraform.workspace - } -} +# tags = { +# Env = terraform.workspace +# } +# } -resource "aws_lambda_event_source_mapping" "map_main_lambda_events" { - event_source_arn = aws_sqs_queue.queue.arn - function_name = module.lambda.arn -} +# resource "aws_lambda_event_source_mapping" "map_main_lambda_events" { +# event_source_arn = aws_sqs_queue.queue.arn +# function_name = module.lambda.arn +# } diff --git a/infrastructure/applications/pycon_backend/worker_repo.tf b/infrastructure/applications/pycon_backend/worker_repo.tf index 18c2a10c83..f9d040c759 100644 --- a/infrastructure/applications/pycon_backend/worker_repo.tf +++ b/infrastructure/applications/pycon_backend/worker_repo.tf @@ -2,11 +2,6 @@ data "aws_ecr_repository" "be_repo" { name = "pythonit/pycon-backend" } -data "aws_ecr_image" "be_image" { - repository_name = data.aws_ecr_repository.be_repo.name - image_tag = data.external.githash.result.githash -} - data "aws_ecr_image" "be_arm_image" { repository_name = data.aws_ecr_repository.be_repo.name image_tag = "arm-${data.external.githash.result.githash}" From 650f7495092d74653001a05190252d27458597a2 Mon Sep 17 00:00:00 2001 From: Marco Acierno Date: Mon, 9 Sep 2024 01:14:11 +0200 Subject: [PATCH 46/46] ee --- infrastructure/applications/pycon_backend/task_worker.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/applications/pycon_backend/task_worker.tf b/infrastructure/applications/pycon_backend/task_worker.tf index ebe1966df8..d0792eb852 100644 --- a/infrastructure/applications/pycon_backend/task_worker.tf +++ b/infrastructure/applications/pycon_backend/task_worker.tf @@ -326,7 +326,7 @@ resource "aws_ecs_task_definition" "worker" { ] command = [ - "-A", "pycon", "worker", "-l", "info", "-E" + "-A pycon", "worker", "-l info", "-E" ] environment = local.env_vars @@ -352,7 +352,7 @@ resource "aws_ecs_task_definition" "worker" { retries = 3 command = [ "CMD-SHELL", - "celery -A pycon inspect ping" + "/home/app/.venv/bin/celery -A pycon inspect ping" ] timeout = 3 interval = 10