Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set pipeline state as terraform variables #260

Merged
merged 1 commit into from
Dec 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 11 additions & 30 deletions config/config.yaml.tftpl
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-auto-audience-segmentation.execution.schedule.state}'
# The `pipeline_parameters` defines the parameters that are going to be used to compile the pipeline.
# Those values may difer depending on the pipeline type and the pipeline steps being used.
# Make sure you review the python function the defines the pipeline.
Expand Down Expand Up @@ -279,9 +277,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-audience-segmentation.execution.schedule.state}'
# The `pipeline_parameters` defines the parameters that are going to be used to compile the pipeline.
# Those values may difer depending on the pipeline type and the pipeline steps being used.
# Make sure you review the python function the defines the pipeline.
Expand Down Expand Up @@ -344,9 +340,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-purchase-propensity.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -407,9 +401,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-churn-propensity.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -464,9 +456,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-customer-ltv.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -527,9 +517,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.feature-creation-aggregated-value-based-bidding.execution.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${location}"
Expand Down Expand Up @@ -575,9 +563,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.value_based_bidding.training.schedule.state}'
# These are pipeline parameters that will be passed to the pipeline to be recompiled
pipeline_parameters:
project: "${project_id}"
Expand Down Expand Up @@ -658,9 +644,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.value_based_bidding.explanation.schedule.state}'
pipeline_parameters:
project: "${project_id}"
location: "${cloud_region}"
Expand Down Expand Up @@ -705,9 +689,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.purchase_propensity.training.schedule.state}'
# These are pipeline parameters that will be passed to the pipeline to be recompiled
pipeline_parameters:
project: "${project_id}"
Expand Down Expand Up @@ -808,9 +790,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
state: '${pipeline_configuration.purchase_propensity.prediction.schedule.state}'
pipeline_parameters:
project_id: "${project_id}"
location: "${cloud_region}"
Expand Down Expand Up @@ -871,6 +851,7 @@ vertex_ai:
subnetwork: "default"
# If you want to use the vpc network defined above, set the following flag to true
use_private_service_access: false
state: '${pipeline_configuration.churn_propensity.training.schedule.state}'
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state: PAUSED # possible states ACTIVE or PAUSED
Expand Down
1 change: 1 addition & 0 deletions infrastructure/terraform/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ resource "local_file" "feature_store_configuration" {
# TODO: this needs to be specific to environment.
location = var.destination_data_location
time_zone = var.time_zone
pipeline_configuration = var.pipeline_configuration
})
}

Expand Down
107 changes: 107 additions & 0 deletions infrastructure/terraform/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -241,3 +241,110 @@ variable "time_zone" {
type = string
default = "America/New_York"
}

variable "pipeline_configuration" {
description = "Pipeline configuration that will alternate certain settings in the config.yaml.tftpl"
type = map(
map(
object({
schedule = object({
# The `state` defines the state of the pipeline.
# In case you don't want to schedule the pipeline, set the state to `PAUSED`.
state = string
})
})
)
)

default = {
feature-creation-auto-audience-segmentation = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-audience-segmentation = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-purchase-propensity = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-churn-propensity = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-customer-ltv = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
feature-creation-aggregated-value-based-bidding = {
execution = {
schedule = {
state = "PAUSED"
}
}
}
value_based_bidding = {
training = {
schedule = {
state = "PAUSED"
}
}
explanation = {
schedule = {
state = "PAUSED"
}
}
}
purchase_propensity = {
training = {
schedule = {
state = "PAUSED"
}
}
prediction = {
schedule = {
state = "PAUSED"
}
}
}
churn_propensity = {
training = {
schedule = {
state = "PAUSED"
}
}
prediction = {
schedule = {
state = "PAUSED"
}
}
}
}
validation {
condition = alltrue([
for p in keys(var.pipeline_configuration) : alltrue([
for c in keys(var.pipeline_configuration[p]) : (
try(var.pipeline_configuration[p][c].schedule.state, "") == "ACTIVE" ||
try(var.pipeline_configuration[p][c].schedule.state, "") == "PAUSED"
)
])
])
error_message = "The 'state' field must be either 'PAUSED' or 'ACTIVE' for all pipeline configurations."
}
}
Loading