chore: Migrate to poetry for package setup and dependencies management

data-preparation/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "sdp-data-preparation"
+version = "0.1.0"
+description = "All the steps of the data preparation for the Shift Data Portal: extraction, transformation and loading."
+authors = ["Sébastien Bourgeois <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+pandas = "^2.2.1"
+requests = "^2.31.0"
+
+[tool.poetry.group.dev.dependencies]
+jupyter = "^1.0.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

data-preparation/src/main_transformation.py

@@ -21,11 +21,13 @@
 import os
 import requests
 from pandas import json_normalize
+from utils import get_project_root_path
 
-RAW_DATA_DIR = "../../data/raw_data"  # contains the RAW data sources (have to be downloaded from Internet and placed in this folder)
-RESULTS_DIR = "../../data/new_prod_data"
-CURRENT_DATA_DIR = "../../data/current_data"
-CURRENT_PROD_DATA = "../../data/current_prod_data"  # contains TRANSFORMED old data generated by Dataiku.
+PROJECT_ROOT_DIR = get_project_root_path()
+RAW_DATA_DIR = os.path.join(PROJECT_ROOT_DIR, "/data/raw_data") # contains the RAW data sources (have to be downloaded from Internet and placed in this folder)
+RESULTS_DIR = os.path.join(PROJECT_ROOT_DIR, "/data/new_prod_data")
+CURRENT_DATA_DIR = os.path.join(PROJECT_ROOT_DIR, "/data/current_data")
+CURRENT_PROD_DATA = os.path.join(PROJECT_ROOT_DIR, "/data/current_prod_data") # contains TRANSFORMED old data generated by Dataiku.
 
 
 class TransformationPipeline:

data-preparation/src/transformation/fossil_import_export.py

@@ -1,10 +1,9 @@
 import pandas as pd
-import numpy as np
 import sys
 
 sys.path.insert(0, r'C:\Users\HP\Desktop\shiftdataportal_data')
-from src.sdp_data.utils.translation import CountryTranslatorFrenchToEnglish
-from src.sdp_data.utils.utils import diff_evaluation
+from utils.translation import CountryTranslatorFrenchToEnglish
+from utils.utils import diff_evaluation
 
 
 # Class to update recent data

data-preparation/src/transformation/fossil_reserves.py

@@ -2,7 +2,7 @@
 import numpy as np
 import sys
 sys.path.insert(0, r'C:\Users\HP\Desktop\shiftdataportal_data')
-from src.sdp_data.utils.translation import CountryTranslatorFrenchToEnglish
+from utils.translation import CountryTranslatorFrenchToEnglish
 
 
 class CoalReservesConsolidatedProdGenerator:

data-preparation/src/transformation/historical_co2.py

@@ -1,6 +1,6 @@
 import pandas as pd
 from utils.translation import CountryTranslatorFrenchToEnglish
-from sdp_data.transformation.demographic.countries import StatisticsPerCountriesAndZonesJoiner
+from demographic.countries import StatisticsPerCountriesAndZonesJoiner
 
 
 class HistoricalCo2PerZoneAndCountryProcessor:

data-preparation/src/transformation/iea.py

@@ -1,6 +1,6 @@
 from utils.iso3166 import countries
 from utils.translation import CountryTranslatorFrenchToEnglish
-from sdp_data.transformation.demographic.countries import StatisticsPerCountriesAndZonesJoiner
+from demographic.countries import StatisticsPerCountriesAndZonesJoiner
 from utils.format import StatisticsDataframeFormatter
 import requests
 import json

data-preparation/src/utils/__init__.py

@@ -0,0 +1 @@
+from .utils import get_project_root_path

data-preparation/src/utils/utils.py

@@ -1,3 +1,5 @@
+import os
+
 from utils.translation import CountryTranslatorFrenchToEnglish
 import pandas as pd
 
@@ -118,3 +120,15 @@ def check_columns_diff(res_old, res_new, country_col):
     return country_old, country_new
 
 
+def get_project_root_path() -> str:
+    """Returns the path of the project root"""
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    dir_name = os.path.basename(dir_path)
+
+    # We go up in the directory level until
+    # we are at the root of the project
+    while dir_name != "shiftdataportal":
+        dir_path = os.path.dirname(dir_path)
+        dir_name = os.path.basename(dir_path)
+
+    return dir_path