Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add s3 bucket, add to forecast, move foreacst ruvnl to ecs task #696

Merged
merged 4 commits into from
Nov 28, 2024
Merged

add s3 bucket, add to forecast, move foreacst ruvnl to ecs task #696

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
109aaa9
add s3 bucket, add to forecast, move foreacst ruvnl to ecs task
peterdudfield Nov 26, 2024
57a38a0
fix
peterdudfield Nov 26, 2024
8c9a9b3
tidy up
peterdudfield Nov 26, 2024
f5520e5
fix SAVE_BATCHES_DIR
peterdudfield Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 63 additions & 36 deletions terraform/india/development/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# 1.3 - ECS Cluster
# 2.0 - S3 bucket for NWP data
# 2.1 - S3 bucket for Satellite data
# 2.2 - S3 bucket for Forecast data
# 3.0 - Secret containing environment variables for the NWP consumer
# 3.1 - Secret containing environment variables for the Satellite consumer
# 3.2 - Secret containing HF read access
Expand Down Expand Up @@ -87,6 +88,16 @@ module "s3-satellite-bucket" {
lifecycled_prefixes = ["data"]
}

# 2.2
module "s3-forecast-bucket" {
source = "../../modules/storage/s3-private"
environment = local.environment
region = var.region
domain = local.domain
service_name = "forecast"
lifecycled_prefixes = [""]
}

# 3.0
resource "aws_secretsmanager_secret" "nwp_consumer_secret" {
name = "${local.environment}/data/nwp-consumer"
Expand Down Expand Up @@ -318,43 +329,54 @@ module "satellite_consumer_ecs" {



# 4.5 - Forecast - Client RU
# 4.5 - Forecast - Client RUVNL
module "forecast" {
source = "../../modules/services/forecast_generic"

region = var.region
environment = local.environment
app-name = "forecast"
ecs_config = {
docker_image = "openclimatefix/india_forecast_app"
docker_version = var.version-forecast
memory_mb = 3072
cpu = 1024
}
rds_config = {
database_secret_arn = module.postgres-rds.secret.arn
database_secret_read_policy_arn = module.postgres-rds.secret-policy.arn
}
s3_nwp_bucket = {
bucket_id = module.s3-nwp-bucket.bucket_id
bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn
datadir = "ecmwf/data"
}
s3_satellite_bucket = {
bucket_id = module.s3-satellite-bucket.bucket_id
bucket_read_policy_arn = module.s3-satellite-bucket.read_policy_arn
datadir = "data"
}
source = "../../modules/services/ecs_task"

// this isnt really needed
s3_ml_bucket = {
bucket_id = module.s3-nwp-bucket.bucket_id
bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn
}
aws-region = var.region
aws-environment = local.environment

loglevel = "INFO"
s3-buckets = [
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.read_policy_arn
},
{
id : module.s3-forecast-bucket.bucket_id,
access_policy_arn : module.s3-forecast-bucket.write_policy_arn
}
]

ecs-task_name = "forecast"
ecs-task_type = "forecast"
ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
sentry_dsn= var.sentry_dsn
ecs-task_size = {
memory = 3072
cpu = 1024
}

container-env_vars = [
{ "name" : "AWS_REGION", "value" : var.region },
{ "name" : "ENVIRONMENT", "value" : local.environment },
{ "name" : "LOGLEVEL", "value" : "INFO" },
{ "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/ecmwf/data/latest.zarr" },
{ "name" : "NWP_GFS_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data/latest.zarr" },
{ "name" : "NWP_MO_GLOBAL_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/metoffice/data/latest.zarr" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "False"},
{ "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/RUVNL"}
]

container-secret_vars = [
{secret_policy_arn: module.postgres-rds.secret.arn,
values: ["DB_URL"]
}
]

container-tag = var.version-forecast
container-name = "india_forecast_app"
container-registry = "openclimatefix"
container-command = []
}


Expand All @@ -368,11 +390,15 @@ module "forecast-ad" {
s3-buckets = [
{
id : module.s3-satellite-bucket.bucket_id,
access_policy_arn : module.s3-satellite-bucket.write_policy_arn
access_policy_arn : module.s3-satellite-bucket.read_policy_arn
},
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.write_policy_arn
access_policy_arn : module.s3-nwp-bucket.read_policy_arn
},
{
id : module.s3-forecast-bucket.bucket_id,
access_policy_arn : module.s3-forecast-bucket.write_policy_arn
}
]

Expand All @@ -394,7 +420,8 @@ module "forecast-ad" {
{ "name" : "SATELLITE_ZARR_PATH", "value": "s3://${module.s3-satellite-bucket.bucket_id}/data/latest/iodc_latest.zarr.zip" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "True"},
{ "name" : "CLIENT_NAME", "value": "ad"}
{ "name" : "CLIENT_NAME", "value": "ad"},
{ "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/ad"},
]

container-secret_vars = [
Expand Down