Fix Service Account Waiting TF logic (#141)

* predicting for only the users with traffic in the past 72h - purchase propensity * running inference only for users events in the past 72h * including 72h users for all models predictions * considering null values in TabWorkflow models * deleting unused pipfile * upgrading lib versions * implementing reporting preprocessing as a new pipeline * adding more code documentation * adding important information on the main README.md and DEVELOPMENT.md * adding schedule run name and more code documentation * implementing a new scheduler using the vertex ai sdk & adding user_id to procedures for consistency * adding more code documentation * adding code doc to the python custom component * adding more code documentation * fixing aggregated predictions query * removing unnecessary resources from deployment * Writing MDS guide * adding the MDS developer and troubleshooting documentation * fixing finding services accounts --------- Co-authored-by: Carlos Timoteo <[email protected]>
GoogleCloudPlatform · Jun 4, 2024 · 79ee516 · 79ee516
1 parent 2cb236e
commit 79ee516
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/infrastructure/terraform/modules/dataform-workflow/service-account.tf b/infrastructure/terraform/modules/dataform-workflow/service-account.tf
@@ -29,12 +29,13 @@ locals {
 }
 
 # Wait for the scheduler service account to be created
+#while ! gcloud asset search-all-iam-policies --scope=projects/${module.data_processing_project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.scheduler_sa}" && [ $COUNTER -lt $MAX_TRIES ]
 resource "null_resource" "wait_for_scheduler_sa_creation" {
   provisioner "local-exec" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.data_processing_project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.scheduler_sa}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.data_processing_project_services.project_id} --filter="EMAIL:${local.scheduler_sa} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."
@@ -83,7 +84,7 @@ resource "null_resource" "wait_for_workflows_sa_creation" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.data_processing_project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.workflows_sa}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.data_processing_project_services.project_id} --filter="EMAIL:${local.workflows_sa} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."

diff --git a/infrastructure/terraform/modules/pipelines/pipelines.tf b/infrastructure/terraform/modules/pipelines/pipelines.tf
@@ -26,7 +26,7 @@ resource "null_resource" "wait_for_vertex_pipelines_sa_creation" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.pipeline_vars.service_account}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.project_services.project_id} --filter="EMAIL:${local.pipeline_vars.service_account} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."
@@ -103,7 +103,7 @@ resource "null_resource" "wait_for_dataflow_worker_sa_creation" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.dataflow_vars.worker_service_account}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.project_services.project_id} --filter="EMAIL:${local.dataflow_vars.worker_service_account} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."