fixing last issues

infrastructure/terraform/modules/feature-store/bigquery-datasets.tf

@@ -326,7 +326,8 @@ module "gemini_insights" {
   location     = local.config_bigquery.dataset.gemini_insights.location
   # The delete_contents_on_destroy attribute specifies whether the contents of the dataset should be deleted when the dataset is destroyed. 
   # In this case, the delete_contents_on_destroy attribute is set to false, which means that the contents of the dataset will not be deleted when the dataset is destroyed.
-  delete_contents_on_destroy = true
+  delete_contents_on_destroy = false
+  deletion_protection = true
 
   dataset_labels = {
     version    = "prod",
@@ -340,7 +341,7 @@ module "gemini_insights" {
       # The max_time_travel_hours attribute specifies the maximum number of hours that data in the dataset can be accessed using time travel queries.
       # In this case, the maximum time travel hours is set to the value of the local file config.yaml section bigquery.dataset.gemini_insights.max_time_travel_hours configuration.
       max_time_travel_hours = local.config_bigquery.dataset.gemini_insights.max_time_travel_hours
-      deletion_protection   = false
+      deletion_protection   = true
       time_partitioning     = null,
       range_partitioning    = null,
       expiration_time       = null,

infrastructure/terraform/modules/feature-store/bigquery-procedures.tf

@@ -1018,6 +1018,14 @@ resource "google_bigquery_routine" "user_behaviour_revenue_insights" {
   depends_on = [
     null_resource.check_gemini_model_exists
   ]
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 /*
@@ -1257,6 +1265,14 @@ resource "google_bigquery_routine" "invoke_backfill_user_behaviour_revenue_insig
     null_resource.check_gemini_model_exists,
     null_resource.create_gemini_model
   ]
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 /*
@@ -1700,8 +1716,9 @@ resource "null_resource" "create_gemini_model" {
   # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
   # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
   lifecycle {
-    #ignore_changes  = all
+    ignore_changes  = all
     #prevent_destroy = true
+    create_before_destroy = true
   }
 
   depends_on = [

infrastructure/terraform/modules/feature-store/main.tf

@@ -37,7 +37,7 @@ module "project_services" {
   source  = "terraform-google-modules/project-factory/google//modules/project_services"
   version = "17.0.0"
 
-  disable_dependent_services  = true
+  disable_dependent_services  = false
   disable_services_on_destroy = false
 
   project_id = local.feature_store_project_id
@@ -121,7 +121,8 @@ resource "google_bigquery_connection" "vertex_ai_connection" {
   # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
   lifecycle {
     ignore_changes  = all
-    prevent_destroy = true
+    #prevent_destroy = true
+    create_before_destroy = true
   }
 }
 
@@ -152,7 +153,8 @@ resource "google_project_iam_member" "vertex_ai_connection_sa_roles" {
   # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
   lifecycle {
     ignore_changes  = all
-    prevent_destroy = true
+    #prevent_destroy = true
+    create_before_destroy = true
   }
 }
 
@@ -165,6 +167,14 @@ resource "time_sleep" "wait_for_vertex_ai_connection_sa_role_propagation" {
   depends_on = [
     google_project_iam_member.vertex_ai_connection_sa_roles
   ]
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 

infrastructure/terraform/modules/pipelines/main.tf

@@ -37,7 +37,7 @@ module "project_services" {
   source  = "terraform-google-modules/project-factory/google//modules/project_services"
   version = "17.0.0"
 
-  disable_dependent_services  = true
+  disable_dependent_services  = false
   disable_services_on_destroy = false
 
   project_id = local.pipeline_vars.project_id

infrastructure/terraform/modules/pipelines/pipelines.tf

@@ -18,6 +18,14 @@ resource "google_service_account" "service_account" {
   account_id   = local.pipeline_vars.service_account_id
   display_name = local.pipeline_vars.service_account_id
   description  = "Service Account to run Vertex AI Pipelines"
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 # Wait for the pipelines service account to be created
@@ -72,6 +80,14 @@ resource "google_project_iam_member" "pipelines_sa_roles" {
     "roles/compute.networkUser"
   ])
   role = each.key
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 # This resource binds the service account to the required roles in the mds project
@@ -89,6 +105,14 @@ resource "google_project_iam_member" "pipelines_sa_mds_project_roles" {
     "roles/bigquery.dataViewer"
   ])
   role = each.key
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 # This resource creates a service account to run the dataflow jobs
@@ -97,6 +121,14 @@ resource "google_service_account" "dataflow_worker_service_account" {
   account_id   = local.dataflow_vars.worker_service_account_id
   display_name = local.dataflow_vars.worker_service_account_id
   description  = "Service Account to run Dataflow jobs"
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 # Wait for the dataflow worker service account to be created
@@ -143,6 +175,14 @@ resource "google_project_iam_member" "dataflow_worker_sa_roles" {
     "roles/storage.objectAdmin",
   ])
   role = each.key
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 # This resource binds the service account to the required roles
@@ -157,6 +197,14 @@ resource "google_service_account_iam_member" "dataflow_sa_iam" {
   service_account_id = "projects/${module.project_services.project_id}/serviceAccounts/${google_service_account.dataflow_worker_service_account.email}"
   role               = "roles/iam.serviceAccountUser"
   member             = "serviceAccount:${google_service_account.service_account.email}"
+
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
+  lifecycle {
+    ignore_changes  = all
+    #prevent_destroy = true
+    create_before_destroy = true
+  }
 }
 
 # This resource creates a Cloud Storage Bucket for the pipeline artifacts
@@ -170,12 +218,12 @@ resource "google_storage_bucket" "pipelines_bucket" {
   # even if it contains objects. In this case, it's set to false, which means that the bucket will not be destroyed if it contains objects.
   force_destroy = false
 
-  # The lifecycle block allows you to configure the lifecycle of the bucket. 
-  # In this case, the ignore_changes attribute is set to all, which means that Terraform 
-  # will ignore any changes to the bucket's lifecycle configuration. The prevent_destroy attribute is set to false, which means that the bucket can be destroyed.
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
   lifecycle {
     ignore_changes  = all
-    prevent_destroy = false ##true
+    #prevent_destroy = true
+    create_before_destroy = true
   }
 }
 
@@ -190,12 +238,12 @@ resource "google_storage_bucket" "custom_model_bucket" {
   # even if it contains objects. In this case, it's set to false, which means that the bucket will not be destroyed if it contains objects.
   force_destroy = false
 
-  # The lifecycle block allows you to configure the lifecycle of the bucket. 
-  # In this case, the ignore_changes attribute is set to all, which means that Terraform 
-  # will ignore any changes to the bucket's lifecycle configuration. The prevent_destroy attribute is set to false, which means that the bucket can be destroyed.
+  # The lifecycle block is used to configure the lifecycle of the table. In this case, the ignore_changes attribute is set to all, which means that Terraform will ignore 
+  # any changes to the table and will not attempt to update the table. The prevent_destroy attribute is set to true, which means that Terraform will prevent the table from being destroyed.
   lifecycle {
     ignore_changes  = all
-    prevent_destroy = false ##true
+    #prevent_destroy = true
+    create_before_destroy = true
   }
 }
 

python/pipelines/transformations-lead-score-propensity.json

@@ -331,7 +331,7 @@
     },
   {
     "numeric": {
-      "column_name": "recife_favorite_past_5_day",
+      "column_name": "recipe_favorite_past_5_day",
       "invalid_values_allowed": true
       }
     },

sql/query/create_gemini_model.sqlx

@@ -18,6 +18,6 @@
 -- Your supervised tuning computations also occur in the europe-west4 region, because that's where TPU resources are located.
 
 -- Create a {{endpoint_name}} model using a remote connection to {{region}}.{{connection_name}}
-CREATE OR REPLACE MODEL `{{project_id}}.{{dataset}}.{{model_name}}`
+CREATE MODEL IF NOT EXISTS `{{project_id}}.{{dataset}}.{{model_name}}`
 REMOTE WITH CONNECTION `{{project_id}}.{{region}}.{{connection_name}}`
 OPTIONS (ENDPOINT = '{{endpoint_name}}');