fix minor error in regionalatlas step + moved scrape address step to …

…deprecated + added new bdc config Signed-off-by: Lucca Baumgärtner <[email protected]>
amosproj · Feb 5, 2024 · a20c71d · a20c71d
1 parent 524b72e
commit a20c71d
Show file tree

Hide file tree

Showing 12 changed files with 87 additions and 31 deletions.
diff --git a/src/bdc/steps/scrape_address.py → deprecated/steps/scrape_address.py b/src/bdc/steps/scrape_address.py → deprecated/steps/scrape_address.py
diff --git a/src/bdc/steps/__init__.py b/src/bdc/steps/__init__.py
@@ -9,6 +9,5 @@
 from .hash_generator import *
 from .preprocess_phonenumbers import *
 from .regionalatlas import *
-from .scrape_address import *
 from .search_offeneregister import *
 from .step import *
diff --git a/src/bdc/steps/regionalatlas.py b/src/bdc/steps/regionalatlas.py
@@ -6,7 +6,6 @@
 import geopandas as gpd
 import osmnx
 import pandas as pd
-from geopandas.tools import sjoin
 from pandas import DataFrame
 from tqdm import tqdm
 
@@ -118,13 +117,13 @@ def run(self) -> DataFrame:
 
         tqdm.pandas(desc="Computing Regional Score")
 
-        self.df[self.added_cols[:-1]] = self.df.progress_apply(
+        self.df[self.added_cols[-1:]] = self.df.progress_apply(
             lambda lead: pd.Series(
                 get_lead_hash_generator().hash_check(
                     lead,
                     self.calculate_regional_score,
                     self.name + "_Regional-Score",
-                    self.added_cols[:-1],
+                    self.added_cols[-1:],
                     lead,
                 )
             ),

diff --git a/src/database/leads/local_repository.py b/src/database/leads/local_repository.py
@@ -4,7 +4,6 @@
 import csv
 import json
 import os
-from datetime import datetime
 from pathlib import Path
 
 import joblib
@@ -20,7 +19,7 @@
 class LocalRepository(Repository):
     BASE_PATH = os.path.dirname(__file__)
     DF_INPUT = os.path.abspath(
-        os.path.join(BASE_PATH, "../../data/sumup_leads_email.csv")
+        os.path.join(BASE_PATH, "../../data/demo_leads_email.csv")
     )
     DF_OUTPUT = os.path.abspath(
         os.path.join(BASE_PATH, "../../data/leads_enriched.csv")

diff --git a/src/demo/pipeline_configs/config_sprint09_release.json b/src/demo/pipeline_configs/config_sprint09_release.json
@@ -6,10 +6,6 @@
         "name": "AnalyzeEmails",
         "force_refresh": true
       },
-      {
-        "name": "ScrapeAddress",
-        "force_refresh": true
-      },
       {
         "name": "PreprocessPhonenumbers",
         "force_refresh": true

diff --git a/src/demo/pipeline_configs/config_template b/src/demo/pipeline_configs/config_template
@@ -7,10 +7,6 @@
             "name": "AnalyzeEmails",
             "force_refresh": true
         },
-        {
-            "name": "ScrapeAddress",
-            "force_refresh": true
-        },
         {
             "name": "PreprocessPhonenumbers",
             "force_refresh": true

diff --git a/src/demo/pipeline_configs/force_refresh_all_steps.json b/src/demo/pipeline_configs/force_refresh_all_steps.json
@@ -0,0 +1,43 @@
+{
+  "description": "This config runs all steps with force_refresh set to true.",
+  "config": {
+    "steps": [
+      {
+        "name": "HashGenerator",
+        "force_refresh": true
+      },
+      {
+        "name": "AnalyzeEmails",
+        "force_refresh": true
+      },
+      {
+        "name": "PreprocessPhonenumbers",
+        "force_refresh": true
+      },
+      {
+        "name": "GooglePlaces",
+        "force_refresh": true
+      },
+      {
+        "name": "GooglePlacesDetailed",
+        "force_refresh": true
+      },
+      {
+        "name": "GPTReviewSentimentAnalyzer",
+        "force_refresh": true
+      },
+      {
+        "name": "GPTSummarizer",
+        "force_refresh": true
+      },
+      {
+        "name": "SmartReviewInsightsEnhancer",
+        "force_refresh": true
+      },
+      {
+        "name": "RegionalAtlas",
+        "force_refresh": true
+      }
+    ]
+  }
+}
diff --git a/src/demo/pipeline_configs/force_refresh_all_steps.json.license b/src/demo/pipeline_configs/force_refresh_all_steps.json.license
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: MIT
+SPDX-FileCopyrightText: 2023 Berkay Bozkurt <[email protected]>
diff --git a/src/demo/pipeline_configs/regionalatlas_only.json b/src/demo/pipeline_configs/regionalatlas_only.json
@@ -0,0 +1,27 @@
+{
+  "description": "This config runs all steps with force_refresh set to true.",
+  "config": {
+    "steps": [
+      {
+        "name": "HashGenerator",
+        "force_refresh": true
+      },
+      {
+        "name": "AnalyzeEmails",
+        "force_refresh": true
+      },
+      {
+        "name": "PreprocessPhonenumbers",
+        "force_refresh": true
+      },
+      {
+        "name": "GooglePlaces",
+        "force_refresh": true
+      },
+      {
+        "name": "RegionalAtlas",
+        "force_refresh": true
+      }
+    ]
+  }
+}
diff --git a/src/demo/pipeline_configs/regionalatlas_only.json.license b/src/demo/pipeline_configs/regionalatlas_only.json.license
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: MIT
+SPDX-FileCopyrightText: 2023 Lucca Baumgärtner <[email protected]>
diff --git a/src/demo/pipeline_configs/run_all_steps.json b/src/demo/pipeline_configs/run_all_steps.json
@@ -1,46 +1,42 @@
 {
-  "description": "This config runs all steps with force_refresh set to true.",
+  "description": "This config runs all steps with force_refresh set to false.",
   "config": {
     "steps": [
       {
         "name": "HashGenerator",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "AnalyzeEmails",
-        "force_refresh": true
-      },
-      {
-        "name": "ScrapeAddress",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "PreprocessPhonenumbers",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GooglePlaces",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GooglePlacesDetailed",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GPTReviewSentimentAnalyzer",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GPTSummarizer",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "SmartReviewInsightsEnhancer",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "RegionalAtlas",
-        "force_refresh": true
+        "force_refresh": false
       }
     ]
   }

diff --git a/src/demo/pipeline_utils.py b/src/demo/pipeline_utils.py
@@ -17,7 +17,6 @@
     HashGenerator,
     PreprocessPhonenumbers,
     RegionalAtlas,
-    ScrapeAddress,
     SearchOffeneRegister,
     SmartReviewInsightsEnhancer,
 )
@@ -33,14 +32,12 @@
     "GPTSummarizer": GPTSummarizer,
     "PreprocessPhonenumbers": PreprocessPhonenumbers,
     "RegionalAtlas": RegionalAtlas,
-    "ScrapeAddress": ScrapeAddress,
     "SearchOffeneRegister": SearchOffeneRegister,
     "SmartReviewInsightsEnhancer": SmartReviewInsightsEnhancer,
 }
 
 # Please do not write following lists! Use the functions below instead.
 _additional_pipeline_steps = [
-    (ScrapeAddress, "Scrape Address", "(will take a long time)"),
     (SearchOffeneRegister, "Search OffeneRegister", "(will take a long time)"),
     (PreprocessPhonenumbers, "Phone Number Validation", ""),
     (