chores: update dep + add sentry (#107)

.github/workflows/deploy-main.yml

@@ -13,7 +13,7 @@ on:
 
 env:
   PYTHON_VERSION: '3.11'
-  POETRY_VERSION: '1.6.1'
+  POETRY_VERSION: '1.7.1'
 
 jobs:
   build:

.github/workflows/test.yml

@@ -5,7 +5,7 @@ on:
 
 env:
   PYTHON_VERSION: '3.11'
-  POETRY_VERSION: '1.6.1'
+  POETRY_VERSION: '1.7.1'
 
 jobs:
   # Label of the runner job

Dockerfile

@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.6.1
+RUN pip install poetry==1.7.1
 
 RUN poetry install
 

Dockerfile_api_import

@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.6.1
+RUN pip install poetry==1.7.1
 
 RUN poetry install
 

Dockerfile_ingest

@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.6.1
+RUN pip install poetry==1.7.1
 
 RUN poetry install
 

Dockerfile_streamlit

@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.6.1
+RUN pip install poetry==1.7.1
 
 RUN poetry install
 

README.md

@@ -180,6 +180,10 @@ When you need to install a new dependency (use a new package, e.g. nltk), run
 ```bash
 poetry add ntlk
 ```
+Update dependencies
+```
+poetry self update
+```
 
 After commiting to the repo, other team members will be able to use the exact same environment you are using. 
 
@@ -264,6 +268,11 @@ Contact QuotaClimat team to 2 files with the API's username and password inside
 docker compose up mediatree
 ```
 
+## Monitoring
+With Sentry, with env variable `SENTRY_DSN`.
+
+Learn more here : https://docs.sentry.io/platforms/python/configuration/options/
+
 ### Batch import based on time
 Use env variable `START_DATE` like in docker compose (epoch second format : 1705409797).
 

pyproject.toml

@@ -12,8 +12,8 @@ priority = "primary"
 
 [tool.poetry.dependencies]
 python = ">=3.11.0,<3.13.0"
-pandas = "^1.5.3"
-advertools = "^0.13.2"
+pandas = "^2.2.0"
+advertools = "^0.14.1"
 xmltodict = "^0.13.0"
 sqlalchemy = "^2.0.21"
 psycopg2-binary = "^2.9.5"
@@ -33,9 +33,11 @@ asyncio = "^3.4.3"
 tomli = "^2.0.1"
 pandera = "^0.17.2"
 aiohttp = "^3.8.6"
-pytest-asyncio = "^0.21.1"
+pytest-asyncio = "^0.23.5"
 swifter = "^1.4.0"
 tenacity = "^8.2.3"
+sentry-sdk = "^1.40.5"
+coverage = "^7.4.2"
 
 [build-system]
 requires = ["poetry-core>=1.1"]

quotaclimat/data_processing/mediatree/api_import.py

@@ -21,6 +21,18 @@
 from quotaclimat.data_processing.mediatree.keyword.keyword import THEME_KEYWORDS
 from typing import List, Optional
 from tenacity import *
+import sentry_sdk
+
+# read SENTRY_DSN from env
+sentry_sdk.init(
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    traces_sample_rate=0.7,
+    # Set profiles_sample_rate to 1.0 to profile 100%
+    # of sampled transactions.
+    # We recommend adjusting this value in production.
+    profiles_sample_rate=0.7,
+)
 
 #read whole file to a string
 password = get_password()
@@ -204,7 +216,7 @@ def parse_reponse_subtitle(response_sub, channel = None) -> Optional[pd.DataFram
         logging.info(f"{total_results} 'total_results' field")
 
         new_df = json_normalize(response_sub.get('data'))
-        logging.info("Schema from API before formatting :\n%s", new_df.dtypes)
+        logging.debug("Schema from API before formatting :\n%s", new_df.dtypes)
         new_df.drop('channel.title', axis=1, inplace=True) # keep only channel.name
 
         new_df['timestamp'] = (pd.to_datetime(new_df['start'], unit='s').dt.tz_localize('utc').dt.tz_convert('Europe/Paris'))