chores: update keywords list + channel france24/sudradio (#129)

* chores * chores: sud-radio * test: keyword updated * wip * dependencies updates * poetry update 1.8.1
dataforgoodfr · Mar 1, 2024 · 6c80490 · 6c80490 · github-actions · Mar 1, 2024
1 parent 48e9742
commit 6c80490
Show file tree

Hide file tree

Showing 13 changed files with 448 additions and 401 deletions.
diff --git a/.github/workflows/deploy-main.yml b/.github/workflows/deploy-main.yml
@@ -13,7 +13,7 @@ on:
 
 env:
   PYTHON_VERSION: '3.11'
-  POETRY_VERSION: '1.7.1'
+  POETRY_VERSION: '1.8.1'
 
 jobs:
   build:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,7 +5,7 @@ on:
 
 env:
   PYTHON_VERSION: '3.11'
-  POETRY_VERSION: '1.7.1'
+  POETRY_VERSION: '1.8.1'
 
 jobs:
   # Label of the runner job

diff --git a/Dockerfile b/Dockerfile
@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.7.1
+RUN pip install poetry==1.8.1
 
 RUN poetry install
 

diff --git a/Dockerfile_api_import b/Dockerfile_api_import
@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.7.1
+RUN pip install poetry==1.8.1
 
 RUN poetry install
 

diff --git a/Dockerfile_ingest b/Dockerfile_ingest
@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.7.1
+RUN pip install poetry==1.8.1
 
 RUN poetry install
 

diff --git a/Dockerfile_streamlit b/Dockerfile_streamlit
@@ -12,7 +12,7 @@ WORKDIR /app
 
 COPY pyproject.toml poetry.lock ./
 
-RUN pip install poetry==1.7.1
+RUN pip install poetry==1.8.1
 
 RUN poetry install
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -61,7 +61,7 @@ def get_channels():
     else: #prod  - all channels
         logging.warning("All channels are used")
         return ["tf1", "france2", "fr3-idf", "france5", "m6", "arte", "d8", "tmc", "bfmtv", "lci", "franceinfotv", "itele",
-        "europe1", "france-culture", "france-inter", "nrj", "rmc", "rtl", "rtl2"]
+        "europe1", "france-culture", "france-inter", "sud-radio", "rmc", "rtl", "france24"]
 
     return channels
 
@@ -99,7 +99,7 @@ async def get_and_save_api_data(exit_event):
 # @see https://github.com/jd/tenacity/tree/main
 @retry(wait=wait_random_exponential(multiplier=1, max=60),stop=stop_after_attempt(7))
 def get_auth_token(password=password, user_name=USER):
-    logger.info(f"Getting a token for user {user_name}")
+    logging.info(f"Getting a token for user {user_name}")
     try:
         post_arguments = {
             'grant_type': 'password'
@@ -201,7 +201,7 @@ def extract_api_sub(
 
 def parse_raw_json(response):
     if response.status_code == 504:
-        logger.error(f"Mediatree API server error 504 (retry enabled)\n {response.content}")
+        logging.error(f"Mediatree API server error 504 (retry enabled)\n {response.content}")
         raise Exception
     else:
         return json.loads(response.content.decode('utf_8'))
@@ -215,7 +215,7 @@ def parse_number_pages(response_sub) -> int :
 def parse_reponse_subtitle(response_sub, channel = None) -> Optional[pd.DataFrame]:
     with sentry_sdk.start_transaction(op="task", name="parse_reponse_subtitle"):
         total_results = parse_total_results(response_sub)
-        logging.getLogger("modin.logger.default").setLevel(logging.WARNING)
+        logging.getLogger("modin.logging.default").setLevel(logging.WARNING)
         if(total_results > 0):
             logging.info(f"{total_results} 'total_results' field")
 
@@ -244,7 +244,7 @@ def log_dataframe_size(df, channel):
 
 async def main():
     with monitor(monitor_slug='mediatree'): #https://docs.sentry.io/platforms/python/crons/
-        logger.info("Start api mediatree import")
+        logging.info("Start api mediatree import")
         create_tables()
 
         event_finish = asyncio.Event()

diff --git a/quotaclimat/data_processing/mediatree/detect_keywords.py b/quotaclimat/data_processing/mediatree/detect_keywords.py
@@ -42,6 +42,8 @@ def format_word_regex(word: str) -> str:
         return word + "s?"
     elif word.endswith('s'):
         return word + '?'
+    elif word.endswith('x'):
+        return word + '?'
     else:
         return word
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,7 +13,7 @@ on: @@
     env:
       PYTHON_VERSION: '3.11'
-      POETRY_VERSION: '1.7.1'
+      POETRY_VERSION: '1.8.1'
     jobs:
       build:
@@ Expand Down @@
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	44	7	84%	37–39, 58–60, 65
insert_existing_data_example.py	20	3	85%	25–27
postgres/schemas
models.py	72	15	79%	74–81, 91–92, 101–111
quotaclimat/data_analytics
analytics_signataire_charte.py	29	29	0%	1–67
bilan.py	108	108	0%	2–372
data_coverage.py	34	34	0%	1–94
exploration.py	125	125	0%	1–440
sitemap_analytics.py	118	118	0%	1–343
quotaclimat/data_ingestion
categorization_program_type.py	1	1	0%	1
config_youtube.py	1	1	0%	1
scaleway_db_backups.py	34	34	0%	1–74
scrap_chartejournalismeecologie_signataires.py	50	50	0%	1–169
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
scrap_tv_program.py	62	62	0%	1–149
scrap_youtube.py	114	114	0%	1–238
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	59	41	31%	21–42, 45–65, 69–80
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	177	103	42%	38–42, 47–53, 57–60, 66, 69–96, 102–117, 122–124, 149–161, 165–168, 172–178, 189–200, 203–207, 213, 237–238, 242, 246–265, 268–270
config.py	15	2	87%	7, 16
detect_keywords.py	145	6	96%	77–79, 167–169
utils.py	66	22	67%	19, 30–54, 57, 76–77
quotaclimat/data_processing/sitemap
sitemap_processing.py	41	27	34%	15–19, 23–25, 29–47, 51–58, 66–96, 101–103
quotaclimat/utils
channels.py	6	6	0%	1–95
climate_keywords.py	2	2	0%	3–35
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
plotly_theme.py	17	17	0%	1–56
sentry.py	10	2	80%	21–22
TOTAL	1625	974	40%