Merge commit 'c51f5a0663fbbdacb8eea3bf163e6fc7b16fe1a3' into devsjc/t…

…est-new-consumer
openclimatefix · Jan 6, 2025 · bae8f4e · bae8f4e
2 parents a34a18e + c51f5a0
commit bae8f4e
Show file tree

Hide file tree

Showing 15 changed files with 35 additions and 307 deletions.
diff --git a/terraform/india/production/main.tf b/terraform/india/production/main.tf
@@ -414,7 +414,7 @@ module "forecast-ad" {
 
 # 5.0
 module "airflow" {
-  source                    = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=f0ecf51"
+  source                    = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=7e27e14"
   aws-environment           = local.environment
   aws-region                = local.region
   aws-domain                = local.domain

diff --git a/terraform/modules/networking/ec2_bastion/user_data.sh b/terraform/modules/networking/ec2_bastion/user_data.sh
@@ -18,4 +18,4 @@ echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8RIkirOd6pkHU6U0+8csUIoBGNuxSj5In63
 echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFrgNxs10xZEpzKSOAcG1cI6pQBhTlCcj41dB2jXLa3w [email protected]" >> /home/ec2-user/.ssh/authorized_keys
 echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOmGsEaesnmsM/np9W7G8vHuxIZBjum/NO3Zk0MT+soO [email protected]" >> /home/ec2-user/.ssh/authorized_keys
 echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0xop1hz1UsyqUUIf9416nGc2GCqm/exfnBGHDd+x8IOXJoGHc5fCM+pKpBaWjLFvOrXB/hx1qZTYFPCdygyzH7KPRIj22VBqgA26DIxd8/0aw4ookNCwOtWFJtm8AVibEffiMAl67zjMsUUZ0pGOM7w9/Zp2ZDr3TjBXKgDOkwLSu0y+66AagYl1dI7TFEAIKrAJQ6ZupYaTBAYxbekB97nYtrc1pUCR+707tm4U+eVjjni0MaQxiD83g35Ij1sOH+avijXTQQGSPJAMxnzEyUArdscg5Pc+qBQ/n5g9o9Sc2k8hwrxPDl4nzfHh9dUtqH7lMTTnjVc6SPdA6wPFY/rXJKk+9xPvphNo4wKZVdMCeVhHwx/9XN7JxKHSI4AQDCRUJ6WhoJFMaMdOziqnxc92sqHiUr1dwgFmKHsUmKm2pwU1JVmQCC+z0WjSbP5EVap8Vxu33t1XbpMowlLWe+Q1wxPR8n77bOqNMNY00bTKu0ki5eDgzHyvXfIQmFJk= preci@VINAYA" >> /home/ec2-user/.ssh/authorized_keys
-echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILYW49tR+g0gTMYhy4RHoY1SC94TSolqqjMnscaqDn3$ [email protected]" >> /home/ec2-user/.ssh/authorized_keys
+echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBPanFrUOmPJOK4ZMh5UQusuDRoCoDwDu1TjywNnOqCD" >> /home/ec2-user/.ssh/authorized_keys
diff --git a/terraform/modules/services/airflow/dags/india/forecast-site-dag.py b/terraform/modules/services/airflow/dags/india/forecast-site-dag.py
@@ -51,7 +51,7 @@
 
     latest_only >> [forecast]
 
-with DAG(f'{region}-ad-forecast', schedule_interval=f"0,30 * * * *", default_args=default_args, concurrency=10, max_active_tasks=10) as dag:
+with DAG(f'{region}-ad-forecast', schedule_interval=f"0,15,30,45 * * * *", default_args=default_args, concurrency=10, max_active_tasks=10) as dag:
     dag.doc_md = "Run the forecast for client AD"
 
     latest_only = LatestOnlyOperator(task_id="latest_only")

diff --git a/terraform/modules/services/airflow/dags/uk/dayafter-dag.py b/terraform/modules/services/airflow/dags/uk/dayafter-dag.py
@@ -38,7 +38,7 @@
 
     national_day_after = EcsRunTaskOperator(
         task_id=f'{region}-national-day-after',
-        task_definition='national-day-after',
+        task_definition='pvlive-national-day-after',
         cluster=cluster,
         overrides={},
         awslogs_region="eu-west-1",
@@ -66,7 +66,7 @@
 
     gsp_day_after = EcsRunTaskOperator(
         task_id=f'{region}-gsp-day-after',
-        task_definition='gsp-day-after',
+        task_definition='pvlive-gsp-day-after',
         cluster=cluster,
         overrides={},
         launch_type="FARGATE",

diff --git a/...dules/services/airflow/dags/uk/gsp-dag.py → ...es/services/airflow/dags/uk/pvlive-dag.py b/...dules/services/airflow/dags/uk/gsp-dag.py → ...es/services/airflow/dags/uk/pvlive-dag.py
@@ -39,7 +39,7 @@
 
     gsp_consumer = EcsRunTaskOperator(
         task_id=f'{region}-gsp-consumer',
-        task_definition='gsp',
+        task_definition='pvlive',
         cluster=cluster,
         overrides={},
         launch_type="FARGATE",

diff --git a/terraform/modules/services/eb_app/eb.tf b/terraform/modules/services/eb_app/eb.tf
@@ -87,7 +87,7 @@ resource "aws_elastic_beanstalk_environment" "eb-environment" {
   setting {
     namespace = "aws:autoscaling:asg"
     name      = "MaxSize"
-    value     = "1"
+    value     = var.max_ec2_count
     resource  = ""
   }
 

diff --git a/terraform/modules/services/eb_app/variables.tf b/terraform/modules/services/eb_app/variables.tf
@@ -97,3 +97,8 @@ variable "s3_bucket" {
   EOT
 }
 
+variable "max_ec2_count" {
+    type = number
+    description = "Maximum number of EC2 instances in the EB environment"
+    default = 1
+}
diff --git a/terraform/modules/services/forecast_generic/README.md b/terraform/modules/services/forecast_generic/README.md
diff --git a/terraform/modules/services/forecast_generic/cloudwatch.tf b/terraform/modules/services/forecast_generic/cloudwatch.tf
diff --git a/terraform/modules/services/forecast_generic/ecs.tf b/terraform/modules/services/forecast_generic/ecs.tf
diff --git a/terraform/modules/services/forecast_generic/iam.tf b/terraform/modules/services/forecast_generic/iam.tf
diff --git a/terraform/modules/services/forecast_generic/variables.tf b/terraform/modules/services/forecast_generic/variables.tf
diff --git a/terraform/modules/storage/database-pair/rds.tf b/terraform/modules/storage/database-pair/rds.tf
@@ -2,7 +2,7 @@
 
 resource "aws_db_instance" "db-forecast" {
   allocated_storage            = 150
-  max_allocated_storage        = 200
+  max_allocated_storage        = 300
   engine                       = "postgres"
   engine_version               = var.engine_version
   instance_class               = "db.t3.medium"