diff --git a/terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py b/terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py index e6b4a4f0..509b405e 100644 --- a/terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py +++ b/terraform/modules/services/airflow/dags/uk/forecast-gsp-dag.py @@ -49,6 +49,24 @@ on_failure_callback=on_failure_callback, ) + forecast_ecmwf = EcsRunTaskOperator( + task_id=f'{region}-gsp-forecast-pvnet-2-ecwmf', + task_definition='forecast_pvnet_ecmwf', + cluster=cluster, + overrides={}, + launch_type="FARGATE", + network_configuration={ + "awsvpcConfiguration": { + "subnets": [subnet], + "securityGroups": [security_group], + "assignPublicIp": "ENABLED", + }, + }, + task_concurrency=10, + on_failure_callback=on_failure_callback, + trigger_rule="all_failed" + ) + forecast_blend = EcsRunTaskOperator( task_id=f'{region}-forecast-blend-pvnet-2', task_definition='forecast_blend', @@ -64,9 +82,11 @@ }, task_concurrency=10, on_failure_callback=on_failure_callback, + trigger_rule="one_success" ) latest_only >> forecast >> forecast_blend + forecast >> forecast_ecmwf >> forecast_blend with DAG(f'{region}-gsp-forecast-pvnet-day-ahead', schedule_interval="15 * * * *", default_args=default_args, concurrency=10, max_active_tasks=10) as dag: diff --git a/terraform/nowcasting/development/main.tf b/terraform/nowcasting/development/main.tf index 7bc265ad..edd02096 100644 --- a/terraform/nowcasting/development/main.tf +++ b/terraform/nowcasting/development/main.tf @@ -25,8 +25,9 @@ The componentes ares: 4.2 - Forecast PVnet 1 4.3 - Forecast National XG 4.4 - Forecast PVnet 2 -4.5 - Forecast PVnet DA -4.6 - Forecast Blend +4.5 - Forecast PVnet ECMWF only +4.6 - Forecast PVnet DA +4.7 - Forecast Blend 5.1 - OCF Dashboard 5.2 - Airflow instance 6.1 - PVSite database @@ -544,6 +545,57 @@ module "forecast_pvnet" { } # 4.5 +module "forecast_pvnet_ecwmf" { +source = "../../modules/services/ecs_task" + + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [ + { + id : module.s3.s3-nwp-bucket.id, + access_policy_arn : module.s3.iam-policy-s3-nwp-read.arn + } + ] + + ecs-task_name = "forecast_pvnet_ecmwf" + ecs-task_type = "forecast" + ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn + ecs-task_size = { + memory = 8192 + cpu = 2048 + storage = 21 + } + + container-env_vars = [ + { "name" : "AWS_REGION", "value" : var.region }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "LOGLEVEL", "value" : "DEBUG" }, + { "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3.s3-nwp-bucket.id}/ecmwf/data/latest.zarr" }, + { "name" : "SENTRY_DSN", "value": var.sentry_dsn}, + {"name": "LOGLEVEL", "value" : "INFO"}, + {"name": "USE_ADJUSTER", "value": "false"}, + {"name": "SAVE_GSP_SUM", "value": "true"}, + {"name": "SENTRY_DSN", "value": var.sentry_dsn}, + {"name": "RUN_EXTRA_MODELS", "value": "false"}, + {"name": "DAY_AHEAD_MODEL", "value": "false"}, + {"name": "USE_ECMWF_ONLY", "value": "true"}, # THIS IS THE IMPORTANT one + ] + + container-secret_vars = [ + {secret_policy_arn: module.database.forecast-database-secret.arn, + values: ["DB_URL"] + } + ] + + container-tag = var.forecast_pvnet_version + container-name = "openclimatefix/pvnet_app" + container-registry = "docker.io" + container-command = [] + +} + +# 4.6 module "forecast_pvnet_day_ahead" { source = "../../modules/services/forecast_generic" @@ -612,7 +664,7 @@ module "analysis_dashboard" { ] } -# 4.6 +# 4.7 module "forecast_blend" { source = "../../modules/services/ecs_task"