Fix data leakage (#142)

* predicting for only the users with traffic in the past 72h - purchase propensity * running inference only for users events in the past 72h * including 72h users for all models predictions * considering null values in TabWorkflow models * deleting unused pipfile * upgrading lib versions * implementing reporting preprocessing as a new pipeline * adding more code documentation * adding important information on the main README.md and DEVELOPMENT.md * adding schedule run name and more code documentation * implementing a new scheduler using the vertex ai sdk & adding user_id to procedures for consistency * adding more code documentation * adding code doc to the python custom component * adding more code documentation * fixing aggregated predictions query * removing unnecessary resources from deployment * Writing MDS guide * adding the MDS developer and troubleshooting documentation * fix one deployment issue and removing bad predictors features * removing leakage feature device_os_version which is not a good predictor --------- Co-authored-by: Carlos Timoteo <[email protected]>
GoogleCloudPlatform · Jun 6, 2024 · b76ff34 · b76ff34
1 parent 79ee516
commit b76ff34
Show file tree

Hide file tree

Showing 10 changed files with 7 additions and 22 deletions.
diff --git a/config/config.yaml.tftpl b/config/config.yaml.tftpl
@@ -662,6 +662,8 @@ vertex_ai:
           #- feature_date
           - user_pseudo_id
           - user_id
+          - device_web_browser_version
+          - device_os_version
           - will_purchase
         pipeline_parameters_substitutions: null 
       prediction:
@@ -962,6 +964,8 @@ vertex_ai:
           #- feature_date
           - user_pseudo_id
           - user_id
+          - device_web_browser_version
+          - device_os_version
           - will_purchase
         pipeline_parameters_substitutions: null
 

diff --git a/docs/images/pipelines_model_performance_troubleshooting.png b/docs/images/pipelines_model_performance_troubleshooting.png
diff --git a/docs/images/pipelines_model_troubleshooting_devices_web_browser_values_label0.png b/docs/images/pipelines_model_troubleshooting_devices_web_browser_values_label0.png
diff --git a/docs/images/pipelines_model_troubleshooting_devices_web_browser_values_label1.png b/docs/images/pipelines_model_troubleshooting_devices_web_browser_values_label1.png
diff --git a/docs/images/pipelines_model_troubleshooting_predictions_probabilities.png b/docs/images/pipelines_model_troubleshooting_predictions_probabilities.png
diff --git a/docs/images/pipelines_model_troubleshooting_training_precision_label1.png b/docs/images/pipelines_model_troubleshooting_training_precision_label1.png
diff --git a/infrastructure/terraform/modules/activation/main.tf b/infrastructure/terraform/modules/activation/main.tf
@@ -416,7 +416,7 @@ module "pipeline_bucket" {
     condition = {
       age            = 365
       with_state     = "ANY"
-      matches_prefix = module.project_services.project_id
+      matches_prefix = var.project_id
     }
   }]
 
@@ -575,7 +575,7 @@ module "function_bucket" {
   source        = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
   version       = "~> 3.4.1"
   project_id    = null_resource.check_cloudfunctions_api.id != "" ? module.project_services.project_id : var.project_id
-  name          = "activation-trigger-${module.project_services.project_id}"
+  name          = "${local.app_prefix}-trigger-${module.project_services.project_id}"
   location      = var.location
   # When deleting a bucket, this boolean option will delete all contained objects. 
   # If false, Terraform will fail to delete buckets which contain objects.

diff --git a/infrastructure/terraform/modules/data-store/main.tf b/infrastructure/terraform/modules/data-store/main.tf
@@ -22,6 +22,7 @@ data "google_project" "data_processing" {
 
 data "google_secret_manager_secret" "github_secret_name" {
   secret_id = google_secret_manager_secret.github-secret.name
+  project   = var.data_processing_project_id
 }
 
 provider "google" {

diff --git a/python/pipelines/transformations-purchase-propensity-cltv.json b/python/pipelines/transformations-purchase-propensity-cltv.json
@@ -25,11 +25,6 @@
 		"column_name": "device_os"
 	  }
 	},
-	{
-	  "categorical": {
-		"column_name": "device_os_version"
-	  }
-	},
 	{
 	  "categorical": {
 		"column_name": "device_language"
@@ -40,11 +35,6 @@
 		"column_name": "device_web_browser"
 	  }
 	},
-	{
-	  "categorical": {
-		"column_name": "device_web_browser_version"
-	  }
-	},
 	{
 	  "categorical": {
 		"column_name": "geo_sub_continent"

diff --git a/python/pipelines/transformations-purchase-propensity.json b/python/pipelines/transformations-purchase-propensity.json
@@ -25,11 +25,6 @@
 		"column_name": "device_os"
 	  }
 	},
-	{
-	  "categorical": {
-		"column_name": "device_os_version"
-	  }
-	},
 	{
 	  "categorical": {
 		"column_name": "device_language"
@@ -40,11 +35,6 @@
 		"column_name": "device_web_browser"
 	  }
 	},
-	{
-	  "categorical": {
-		"column_name": "device_web_browser_version"
-	  }
-	},
 	{
 	  "categorical": {
 		"column_name": "geo_sub_continent"