chores: update stop words + log

dataforgoodfr · Jul 5, 2024 · 64693aa · 64693aa · github-actions · Jul 5, 2024
1 parent c917ba7
commit 64693aa
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/quotaclimat/data_processing/mediatree/detect_keywords.py b/quotaclimat/data_processing/mediatree/detect_keywords.py
@@ -102,7 +102,7 @@ def filter_keyword_with_same_timestamp(keywords_with_timestamp: List[dict])-> Li
     final_result = len(keywords_with_timestamp)
 
     if final_result < number_of_keywords:
-        logging.info(f"Filtering keywords {final_result} out of {number_of_keywords} | {keywords_with_timestamp} with final result")
+        logging.debug(f"Filtering keywords {final_result} out of {number_of_keywords} | {keywords_with_timestamp} with final result")
 
     return keywords_with_timestamp
 

diff --git a/quotaclimat/data_processing/mediatree/keyword/stop_words.py b/quotaclimat/data_processing/mediatree/keyword/stop_words.py
@@ -20,6 +20,7 @@
     ,"installateur de panneaux solaires"
     ,"installateurs de panneaux solaires"
     ,"panneaux photovoltaïques garanti à vie"
+    ,"on installe des panneaux photovoltaïques borne de recharge"
     ,"avec le leader du photovoltaïque"
     ,"en train d"
     ,"consigne de vote"
@@ -56,4 +57,7 @@
     ,"l' entreprise de julie fabrique des meubles en matériaux recyclés"
     ,"jardin miami fabriqué avec quatre-vingts pour cent de matériaux recyclés"
     ,"est à fond sur le tri sélectif"
+    ,"transition énergétique baisse de lumière"
+    ,"transition énergétique co"
+    ,"transition énergétique nos lumières"
 ]
diff --git a/quotaclimat/data_processing/mediatree/update_pg_keywords.py b/quotaclimat/data_processing/mediatree/update_pg_keywords.py
@@ -58,7 +58,7 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
                     keywords_with_timestamp != new_keywords_with_timestamp or
                     theme != matching_themes
                     ):
-                    logging.debug(f"Difference detected for themes for ID {keyword_id} -  {theme} - {matching_themes} \
+                    logging.info(f"Difference detected for themes for ID {keyword_id} -  {theme} - {matching_themes} \
                                 \nnumber_of_keywords {number_of_keywords} - {new_number_of_keywords}\
                                 \nkeywords_with_timestamp : {keywords_with_timestamp}\
                                 \n new_nkeywords_with_timestamp : {new_keywords_with_timestamp}"
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	43	7	84%	36–38, 56–58, 63
insert_existing_data_example.py	19	3	84%	25–27
postgres/schemas
models.py	147	10	93%	121–128, 140–141, 199–200, 214–215
quotaclimat/data_ingestion
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	55	37	33%	21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	205	128	38%	43–47, 52–68, 72–75, 81, 84–122, 128–143, 147–148, 161–173, 177–183, 196–207, 210–214, 220, 255–256, 260, 264–298, 301–303
channel_program.py	136	51	62%	30–32, 43–45, 59, 95, 104, 142–183
config.py	15	2	87%	7, 16
detect_keywords.py	213	8	96%	169–172, 216, 271–273
update_pg_keywords.py	54	39	28%	14–100, 125–129, 152–178, 184
utils.py	69	22	68%	27–51, 54, 63, 84–85
quotaclimat/utils
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
sentry.py	10	2	80%	21–22
TOTAL	1215	354	71%