Skip to content

Commit

Permalink
Set pipeline state as terraform variables (#260)
Browse files Browse the repository at this point in the history
  • Loading branch information
@martenlindblad
martenlindblad authored Dec 9, 2024
1 parent fda8762 commit 5a7eb5a
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 30 deletions.
41 changes: 11 additions & 30 deletions config/config.yaml.tftpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-auto-audience-segmentation.execution.schedule.state}'
# The `pipeline_parameters` defines the parameters that are going to be used to compile the pipeline.
# Those values may difer depending on the pipeline type and the pipeline steps being used.
# Make sure you review the python function the defines the pipeline.
Expand Down Expand Up @@ -279,9 +277,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-audience-segmentation.execution.schedule.state}'
# The `pipeline_parameters` defines the parameters that are going to be used to compile the pipeline.
# Those values may difer depending on the pipeline type and the pipeline steps being used.
# Make sure you review the python function the defines the pipeline.
Expand Down Expand Up @@ -344,9 +340,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-purchase-propensity.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -407,9 +401,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-churn-propensity.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -464,9 +456,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-customer-ltv.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -527,9 +517,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-aggregated-value-based-bidding.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -575,9 +563,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.value_based_bidding.training.schedule.state}'
# These are pipeline parameters that will be passed to the pipeline to be recompiled
pipeline_parameters:
project: "${project_id}"
Expand Down Expand Up @@ -658,9 +644,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.value_based_bidding.explanation.schedule.state}'
pipeline_parameters:
project: "${project_id}"
location: "${cloud_region}"
Expand Down Expand Up @@ -705,9 +689,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.purchase_propensity.training.schedule.state}'
# These are pipeline parameters that will be passed to the pipeline to be recompiled
pipeline_parameters:
project: "${project_id}"
Expand Down Expand Up @@ -808,9 +790,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.purchase_propensity.prediction.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${cloud_region}"
Expand Down Expand Up @@ -871,6 +851,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
state: '${pipeline_configuration.churn_propensity.training.schedule.state}'
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
Expand Down
1 change: 1 addition & 0 deletions infrastructure/terraform/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ resource "local_file" "feature_store_configuration" {
# TODO: this needs to be specific to environment.
location = var.destination_data_location
time_zone = var.time_zone
pipeline_configuration = var.pipeline_configuration
})
}

Expand Down
107 changes: 107 additions & 0 deletions infrastructure/terraform/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,3 +241,110 @@ variable "time_zone" {
type = string
default = "America/New_York"
}

variable "pipeline_configuration" {
description = "Pipeline configuration that will alternate certain settings in the config.yaml.tftpl"
type = map(
map(
object({
schedule = object({
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state = string
})
})
)
)

default = {
feature-creation-auto-audience-segmentation = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-audience-segmentation = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-purchase-propensity = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-churn-propensity = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-customer-ltv = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-aggregated-value-based-bidding = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
value_based_bidding = {
training = {
schedule = {
state = "PAUSED"
}
}
explanation = {
schedule = {
state = "PAUSED"
}
}
}
purchase_propensity = {
training = {
schedule = {
state = "PAUSED"
}
}
prediction = {
schedule = {
state = "PAUSED"
}
}
}
churn_propensity = {
training = {
schedule = {
state = "PAUSED"
}
}
prediction = {
schedule = {
state = "PAUSED"
}
}
}
}
validation {
condition = alltrue([
for p in keys(var.pipeline_configuration) : alltrue([
for c in keys(var.pipeline_configuration[p]) : (
try(var.pipeline_configuration[p][c].schedule.state, "") == "ACTIVE" ||
try(var.pipeline_configuration[p][c].schedule.state, "") == "PAUSED"
)
])
])
error_message = "The 'state' field must be either 'PAUSED' or 'ACTIVE' for all pipeline configurations."
}
}

0 comments on commit 5a7eb5a

Please sign in to comment.