From 79ee516610674e464fe8f27d648219684a2371a8 Mon Sep 17 00:00:00 2001
From: Carlos Timoteo <chmstimoteo@gmail.com>
Date: Tue, 4 Jun 2024 16:58:40 -0400
Subject: [PATCH] Fix Service Account Waiting TF logic (#141)

* predicting for only the users with traffic in the past 72h - purchase propensity

* running inference only for users events in the past 72h

* including 72h users for all models predictions

* considering null values in TabWorkflow models

* deleting unused pipfile

* upgrading lib versions

* implementing reporting preprocessing as a new pipeline

* adding more code documentation

* adding important information on the main README.md and DEVELOPMENT.md

* adding schedule run name and more code documentation

* implementing a new scheduler using the vertex ai sdk & adding user_id to procedures for consistency

* adding more code documentation

* adding code doc to the python custom component

* adding more code documentation

* fixing aggregated predictions query

* removing unnecessary resources from deployment

* Writing MDS guide

* adding the MDS developer and troubleshooting documentation

* fixing finding services accounts

---------

Co-authored-by: Carlos Timoteo <ctimoteo@google.com>
---
 .../terraform/modules/dataform-workflow/service-account.tf   | 5 +++--
 infrastructure/terraform/modules/pipelines/pipelines.tf      | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/infrastructure/terraform/modules/dataform-workflow/service-account.tf b/infrastructure/terraform/modules/dataform-workflow/service-account.tf
index 16c3edee..39d31811 100644
--- a/infrastructure/terraform/modules/dataform-workflow/service-account.tf
+++ b/infrastructure/terraform/modules/dataform-workflow/service-account.tf
@@ -29,12 +29,13 @@ locals {
 }
 
 # Wait for the scheduler service account to be created
+#while ! gcloud asset search-all-iam-policies --scope=projects/${module.data_processing_project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.scheduler_sa}" && [ $COUNTER -lt $MAX_TRIES ]
 resource "null_resource" "wait_for_scheduler_sa_creation" {
   provisioner "local-exec" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.data_processing_project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.scheduler_sa}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.data_processing_project_services.project_id} --filter="EMAIL:${local.scheduler_sa} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."
@@ -83,7 +84,7 @@ resource "null_resource" "wait_for_workflows_sa_creation" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.data_processing_project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.workflows_sa}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.data_processing_project_services.project_id} --filter="EMAIL:${local.workflows_sa} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."
diff --git a/infrastructure/terraform/modules/pipelines/pipelines.tf b/infrastructure/terraform/modules/pipelines/pipelines.tf
index ae424e79..de93bef3 100644
--- a/infrastructure/terraform/modules/pipelines/pipelines.tf
+++ b/infrastructure/terraform/modules/pipelines/pipelines.tf
@@ -26,7 +26,7 @@ resource "null_resource" "wait_for_vertex_pipelines_sa_creation" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.pipeline_vars.service_account}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.project_services.project_id} --filter="EMAIL:${local.pipeline_vars.service_account} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."
@@ -103,7 +103,7 @@ resource "null_resource" "wait_for_dataflow_worker_sa_creation" {
     command = <<-EOT
     COUNTER=0
     MAX_TRIES=100
-    while ! gcloud asset search-all-iam-policies --scope=projects/${module.project_services.project_id} --flatten="policy.bindings[].members[]" --filter="policy.bindings.members~\"serviceAccount:\"" --format="value(policy.bindings.members.split(sep=\":\").slice(1))" | grep -i "${local.dataflow_vars.worker_service_account}" && [ $COUNTER -lt $MAX_TRIES ]
+    while ! gcloud iam service-accounts list --project=${module.project_services.project_id} --filter="EMAIL:${local.dataflow_vars.worker_service_account} AND DISABLED:False" --format="table(EMAIL, DISABLED)" && [ $COUNTER -lt $MAX_TRIES ]
     do
       sleep 3
       printf "."