From 07498ad310e25f49aa2e54eec1cff433f82833c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Lindblad?= Date: Wed, 4 Dec 2024 09:59:02 +0100 Subject: [PATCH] Set pipeline state as terraform variables --- config/config.yaml.tftpl | 41 +++------- infrastructure/terraform/main.tf | 1 + infrastructure/terraform/variables.tf | 107 ++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 30 deletions(-) diff --git a/config/config.yaml.tftpl b/config/config.yaml.tftpl index 9c31bf3b..c58b620c 100644 --- a/config/config.yaml.tftpl +++ b/config/config.yaml.tftpl @@ -195,9 +195,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.feature-creation-auto-audience-segmentation.execution.schedule.state}' # The `pipeline_parameters` defines the parameters that are going to be used to compile the pipeline. # Those values may difer depending on the pipeline type and the pipeline steps being used. # Make sure you review the python function the defines the pipeline. @@ -279,9 +277,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.feature-creation-audience-segmentation.execution.schedule.state}' # The `pipeline_parameters` defines the parameters that are going to be used to compile the pipeline. # Those values may difer depending on the pipeline type and the pipeline steps being used. # Make sure you review the python function the defines the pipeline. @@ -344,9 +340,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.feature-creation-purchase-propensity.execution.schedule.state}' pipeline_parameters: project_id: "${project_id}" location: "${location}" @@ -407,9 +401,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.feature-creation-churn-propensity.execution.schedule.state}' pipeline_parameters: project_id: "${project_id}" location: "${location}" @@ -464,9 +456,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.feature-creation-customer-ltv.execution.schedule.state}' pipeline_parameters: project_id: "${project_id}" location: "${location}" @@ -527,9 +517,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.feature-creation-aggregated-value-based-bidding.execution.schedule.state}' pipeline_parameters: project_id: "${project_id}" location: "${location}" @@ -575,9 +563,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.value_based_bidding.training.schedule.state}' # These are pipeline parameters that will be passed to the pipeline to be recompiled pipeline_parameters: project: "${project_id}" @@ -658,9 +644,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.value_based_bidding.explanation.schedule.state}' pipeline_parameters: project: "${project_id}" location: "${cloud_region}" @@ -705,9 +689,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.purchase_propensity.training.schedule.state}' # These are pipeline parameters that will be passed to the pipeline to be recompiled pipeline_parameters: project: "${project_id}" @@ -808,9 +790,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false - # The `state` defines the state of the pipeline. - # In case you don't want to schedule the pipeline, set the state to `PAUSED`. - state: PAUSED # possible states ACTIVE or PAUSED + state: '${pipeline_configuration.purchase_propensity.prediction.schedule.state}' pipeline_parameters: project_id: "${project_id}" location: "${cloud_region}" @@ -871,6 +851,7 @@ vertex_ai: subnetwork: "default" # If you want to use the vpc network defined above, set the following flag to true use_private_service_access: false + state: '${pipeline_configuration.churn_propensity.training.schedule.state}' # The `state` defines the state of the pipeline. # In case you don't want to schedule the pipeline, set the state to `PAUSED`. state: PAUSED # possible states ACTIVE or PAUSED diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 4bf297a2..8ac99a5d 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -124,6 +124,7 @@ resource "local_file" "feature_store_configuration" { # TODO: this needs to be specific to environment. location = var.destination_data_location time_zone = var.time_zone + pipeline_configuration = var.pipeline_configuration }) } diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/variables.tf index 9819041a..8ce1197d 100644 --- a/infrastructure/terraform/variables.tf +++ b/infrastructure/terraform/variables.tf @@ -241,3 +241,110 @@ variable "time_zone" { type = string default = "America/New_York" } + +variable "pipeline_configuration" { + description = "Pipeline configuration that will alternate certain settings in the config.yaml.tftpl" + type = map( + map( + object({ + schedule = object({ + # The `state` defines the state of the pipeline. + # In case you don't want to schedule the pipeline, set the state to `PAUSED`. + state = string + }) + }) + ) + ) + + default = { + feature-creation-auto-audience-segmentation = { + execution = { + schedule = { + state = "PAUSED" + } + } + } + feature-creation-audience-segmentation = { + execution = { + schedule = { + state = "PAUSED" + } + } + } + feature-creation-purchase-propensity = { + execution = { + schedule = { + state = "PAUSED" + } + } + } + feature-creation-churn-propensity = { + execution = { + schedule = { + state = "PAUSED" + } + } + } + feature-creation-customer-ltv = { + execution = { + schedule = { + state = "PAUSED" + } + } + } + feature-creation-aggregated-value-based-bidding = { + execution = { + schedule = { + state = "PAUSED" + } + } + } + value_based_bidding = { + training = { + schedule = { + state = "PAUSED" + } + } + explanation = { + schedule = { + state = "PAUSED" + } + } + } + purchase_propensity = { + training = { + schedule = { + state = "PAUSED" + } + } + prediction = { + schedule = { + state = "PAUSED" + } + } + } + churn_propensity = { + training = { + schedule = { + state = "PAUSED" + } + } + prediction = { + schedule = { + state = "PAUSED" + } + } + } + } + validation { + condition = alltrue([ + for p in keys(var.pipeline_configuration) : alltrue([ + for c in keys(var.pipeline_configuration[p]) : ( + try(var.pipeline_configuration[p][c].schedule.state, "") == "ACTIVE" || + try(var.pipeline_configuration[p][c].schedule.state, "") == "PAUSED" + ) + ]) + ]) + error_message = "The 'state' field must be either 'PAUSED' or 'ACTIVE' for all pipeline configurations." + } +}