Skip to content

Commit

Permalink
Merge branch 'main' into s3-site-forecast-ecs-task
Browse files Browse the repository at this point in the history
  • Loading branch information
@peterdudfield
peterdudfield committed Dec 2, 2024
2 parents bfbda6b + 29fe4b4 commit 188e0ca
Show file tree
Hide file tree
Showing 9 changed files with 261 additions and 74 deletions.
159 changes: 115 additions & 44 deletions terraform/india/development/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,17 @@
# 1.3 - ECS Cluster
# 2.0 - S3 bucket for NWP data
# 2.1 - S3 bucket for Satellite data
# 2.2 - S3 bucket for Forecast data
# 3.0 - Secret containing environment variables for the NWP consumer
# 3.1 - Secret containing environment variables for the Satellite consumer
# 3.2 - Secret containing HF read access
# 4.0 - ECS task definition for the NWP consumer
# 4.0 - ECS task definition for the ECMWF consumer
# 4.1 - ECS task definition for the GFS consumer
# 4.2 - ECS task definition for Collection RUVNL data
# 4.3 - Satellite Consumer
# 4.4 - ECS task definition for the Forecast - Client RU
# 4.5 - ECS task definition for the Forecast - Client AD
# 4.2 - ECS task definition for the MetOffice consumer
# 4.3 - ECS task definition for Collection RUVNL data
# 4.4 - Satellite Consumer
# 4.5 - ECS task definition for the Forecast - Client RU
# 4.6 - ECS task definition for the Forecast - Client AD
# 5.0 - Airflow EB Instance
# 5.1 - India API EB Instance
# 5.2 - India Analysis Dashboard
Expand Down Expand Up @@ -86,6 +88,16 @@ module "s3-satellite-bucket" {
lifecycled_prefixes = ["data"]
}

# 2.2
module "s3-forecast-bucket" {
source = "../../modules/storage/s3-private"
environment = local.environment
region = var.region
domain = local.domain
service_name = "forecast"
lifecycled_prefixes = [""]
}

# 3.0
resource "aws_secretsmanager_secret" "nwp_consumer_secret" {
name = "${local.environment}/data/nwp-consumer"
Expand Down Expand Up @@ -198,9 +210,49 @@ module "nwp_consumer_gfs_live_ecs_task" {
]
}

# 4.2
module "nwp-consumer-metoffice-live-ecs-task" {
source = "../../modules/services/ecs_task"

ecs-task_name = "nwp-consumer-metoffice-india"
ecs-task_type = "consumer"
ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
ecs-task_size = {
cpu = 512
memory = 1024
}

# 4.2
aws-region = var.region
aws-environment = local.environment

s3-buckets = [
{
id : module.s3-nwp-bucket.bucket_id
access_policy_arn : module.s3-nwp-bucket.write_policy_arn
}
]

container-env_vars = [
{ "name" : "LOGLEVEL", "value" : "INFO" },
{ "name" : "METOFFICE_ORDER_ID", "value" : "india-11params-54steps" },
{ "name" : "MODEL_REPOSITORY", "value" : "metoffice-datahub" },
{ "name" : "CONCURRENCY", "value" : "false" },
{ "name" : "ZARRDIR", "value" : format("s3://%s/metoffice/data", module.s3-nwp-bucket.bucket_id) },
{ "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
]
container-secret_vars = [
{
secret_policy_arn: aws_secretsmanager_secret.nwp_consumer_secret.arn,
values: ["METOFFICE_API_KEY"],
}
]
container-tag = "devsjc-major-refactor"
container-name = "openclimatefix/nwp-consumer"
container-command = ["consume"]
}


# 4.3
module "ruvnl_consumer_ecs" {
source = "../../modules/services/ecs_task"

Expand Down Expand Up @@ -234,7 +286,7 @@ module "ruvnl_consumer_ecs" {
]
}

# 4.3 - Satellite Consumer
# 4.4 - Satellite Consumer
module "satellite_consumer_ecs" {
source = "../../modules/services/ecs_task"

Expand Down Expand Up @@ -277,47 +329,58 @@ module "satellite_consumer_ecs" {



# 4.4 - Forecast - Client RU
# 4.5 - Forecast - Client RUVNL
module "forecast" {
source = "../../modules/services/forecast_generic"

region = var.region
environment = local.environment
app-name = "forecast"
ecs_config = {
docker_image = "openclimatefix/india_forecast_app"
docker_version = var.version-forecast
memory_mb = 3072
cpu = 1024
}
rds_config = {
database_secret_arn = module.postgres-rds.secret.arn
database_secret_read_policy_arn = module.postgres-rds.secret-policy.arn
}
s3_nwp_bucket = {
bucket_id = module.s3-nwp-bucket.bucket_id
bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn
datadir = "ecmwf/data"
}
s3_satellite_bucket = {
bucket_id = module.s3-satellite-bucket.bucket_id
bucket_read_policy_arn = module.s3-satellite-bucket.read_policy_arn
datadir = "data"
}
source = "../../modules/services/ecs_task"

// this isnt really needed
s3_ml_bucket = {
bucket_id = module.s3-nwp-bucket.bucket_id
bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn
}
aws-region = var.region
aws-environment = local.environment

s3-buckets = [
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.read_policy_arn
},
{
id : module.s3-forecast-bucket.bucket_id,
access_policy_arn : module.s3-forecast-bucket.write_policy_arn
}
]

loglevel = "INFO"
ecs-task_name = "forecast"
ecs-task_type = "forecast"
ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
sentry_dsn= var.sentry_dsn
ecs-task_size = {
memory = 3072
cpu = 1024
}

container-env_vars = [
{ "name" : "AWS_REGION", "value" : var.region },
{ "name" : "ENVIRONMENT", "value" : local.environment },
{ "name" : "LOGLEVEL", "value" : "INFO" },
{ "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/ecmwf/data/latest.zarr" },
{ "name" : "NWP_GFS_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data/latest.zarr" },
{ "name" : "NWP_MO_GLOBAL_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/metoffice/data/latest.zarr" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "False"},
{ "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/RUVNL"}
]

container-secret_vars = [
{secret_policy_arn: module.postgres-rds.secret.arn,
values: ["DB_URL"]
}
]

container-tag = var.version-forecast
container-name = "india_forecast_app"
container-registry = "openclimatefix"
container-command = []
}


# 4.5 - Forecast - Client AD
# 4.6 - Forecast - Client AD
module "forecast-ad" {
source = "../../modules/services/ecs_task"

Expand All @@ -327,11 +390,15 @@ module "forecast-ad" {
s3-buckets = [
{
id : module.s3-satellite-bucket.bucket_id,
access_policy_arn : module.s3-satellite-bucket.write_policy_arn
access_policy_arn : module.s3-satellite-bucket.read_policy_arn
},
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.write_policy_arn
access_policy_arn : module.s3-nwp-bucket.read_policy_arn
},
{
id : module.s3-forecast-bucket.bucket_id,
access_policy_arn : module.s3-forecast-bucket.write_policy_arn
}
]

Expand All @@ -349,10 +416,12 @@ module "forecast-ad" {
{ "name" : "LOGLEVEL", "value" : "DEBUG" },
{ "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/ecmwf/data/latest.zarr" },
{ "name" : "NWP_GFS_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data/latest.zarr" },
{ "name" : "NWP_MO_GLOBAL_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/metoffice/data/latest.zarr" },
{ "name" : "SATELLITE_ZARR_PATH", "value": "s3://${module.s3-satellite-bucket.bucket_id}/data/latest/iodc_latest.zarr.zip" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "True"},
{ "name" : "CLIENT_NAME", "value": "ad"}
{ "name" : "CLIENT_NAME", "value": "ad"},
{ "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/ad"},
]

container-secret_vars = [
Expand Down Expand Up @@ -432,6 +501,8 @@ module "analysis_dashboard" {
{ "name" : "ORIGINS", "value" : "*" },
{ "name" : "REGION", "value": local.domain},
{ "name" : "ENVIRONMENT", "value": local.environment},
{ "name" : "AUTH0_DOMAIN", "value" : var.auth_domain },
{ "name" : "AUTH0_CLIENT_ID", "value" : var.auth_dashboard_client_id },
]
container-name = "analysis-dashboard"
container-tag = var.analysis_dashboard_version
Expand Down
5 changes: 5 additions & 0 deletions terraform/india/development/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,9 @@ variable "auth_domain" {
variable "auth_api_audience" {
description = "The Auth API Audience that should be used"
default = "not-set"
}

variable "auth_dashboard_client_id" {
description = "The Auth client id for the dashboard that should be used"
default = "not-set"
}
66 changes: 55 additions & 11 deletions terraform/india/production/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@
# 3.2 - Secret containing HF read access
# 4.0 - ECS task definition for the NWP consumer
# 4.1 - ECS task definition for the GFS consumer
# 4.2 - ECS task definition for Collection RUVNL data
# 4.3 - Satellite Consumer
# 4.4 - ECS task definition for the Forecast - Client RU
# 4.5 - ECS task definition for the Forecast - Client AD
# 4.2 - ECS task definition for the MetOffice consumer
# 4.3 - ECS task definition for Collection RUVNL data
# 4.4 - Satellite Consumer
# 4.5 - ECS task definition for the Forecast - Client RU
# 4.6 - ECS task definition for the Forecast - Client AD
# 5.0 - Airflow EB Instance
# 5.1 - India API EB Instance
# 5.2 - India Analysis Dashboard
Expand Down Expand Up @@ -201,8 +202,49 @@ module "nwp_consumer_gfs_live_ecs_task" {
]
}


# 4.2
module "nwp-consumer-metoffice-live-ecs-task" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=f0ecf51"

ecs-task_name = "nwp-consumer-metoffice-india"
ecs-task_type = "consumer"
ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
ecs-task_size = {
cpu = 512
memory = 1024
}

aws-region = var.region
aws-environment = local.environment

s3-buckets = [
{
id : module.s3-nwp-bucket.bucket_id
access_policy_arn : module.s3-nwp-bucket.write_policy_arn
}
]

container-env_vars = [
{ "name" : "LOGLEVEL", "value" : "INFO" },
{ "name" : "METOFFICE_ORDER_ID", "value" : "india-11params-54steps" },
{ "name" : "MODEL_REPOSITORY", "value" : "metoffice-datahub" },
{ "name" : "CONCURRENCY", "value" : "false" },
{ "name" : "ZARRDIR", "value" : format("s3://%s/metoffice/data", module.s3-nwp-bucket.bucket_id) },
{ "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
]
container-secret_vars = [
{
secret_policy_arn: aws_secretsmanager_secret.nwp_consumer_secret.arn,
values: ["METOFFICE_API_KEY"],
}
]
container-tag = "devsjc-major-refactor"
container-name = "openclimatefix/nwp-consumer"
container-command = ["consume"]
}


# 4.3
module "ruvnl_consumer_ecs" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=205465e"

Expand Down Expand Up @@ -237,7 +279,7 @@ module "ruvnl_consumer_ecs" {
}


# 4.3 - Satellite Consumer
# 4.4 - Satellite Consumer
module "satellite_consumer_ecs" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=205465e"

Expand Down Expand Up @@ -278,17 +320,17 @@ module "satellite_consumer_ecs" {
}


# 4.4 - Forecast - Client RU
# 4.5 - Forecast - Client RU
module "forecast" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=42eba24"
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=f0ecf51"

region = var.region
environment = local.environment
app-name = "forecast"
ecs_config = {
docker_image = "openclimatefix/india_forecast_app"
docker_version = var.version-forecast
memory_mb = 2048
memory_mb = 4096
cpu = 1024
}
rds_config = {
Expand Down Expand Up @@ -316,7 +358,7 @@ module "forecast" {
sentry_dsn = var.sentry_dsn
}

# 4.5 - Forecast - Client AD
# 4.6 - Forecast - Client AD
module "forecast-ad" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=73255a4"

Expand Down Expand Up @@ -372,7 +414,7 @@ module "forecast-ad" {

# 5.0
module "airflow" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=c73cee9"
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=f0ecf51"
aws-environment = local.environment
aws-region = local.region
aws-domain = local.domain
Expand Down Expand Up @@ -426,6 +468,8 @@ module "analysis_dashboard" {
{ "name" : "ORIGINS", "value" : "*" },
{ "name" : "REGION", "value": local.domain},
{ "name" : "ENVIRONMENT", "value": local.environment},
{ "name" : "AUTH0_DOMAIN", "value" : var.auth_domain },
{ "name" : "AUTH0_CLIENT_ID", "value" : var.auth_dashboard_client_id },
]
container-name = "analysis-dashboard"
container-tag = var.analysis_dashboard_version
Expand Down
5 changes: 5 additions & 0 deletions terraform/india/production/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,8 @@ variable satellite-consumer {
default = "0.0.1"
description = "Container image tag of the satellite data consumer to use: openclimatefix/satip"
}

variable "auth_dashboard_client_id" {
description = "The Auth client id for the dashboard that should be used"
default = "not-set"
}
Loading

0 comments on commit 188e0ca

Please sign in to comment.