fix: timezone when updating

dataforgoodfr · Sep 17, 2024 · ac926b9 · ac926b9 · github-actions · Sep 17, 2024
1 parent 5aec98e
commit ac926b9
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 3 deletions.
diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -67,7 +67,9 @@ async def update_pg_data(exit_event):
         update_keywords(session, batch_size=batch_size, start_date=start_date, program_only=program_only, end_date=end_date, channel=channel)
         exit_event.set()
     except Exception as err:
-        logging.error("Could update_pg_data %s:(%s)" % (type(err).__name__, err))
+        logging.fatal("Could not update_pg_data %s:(%s)" % (type(err).__name__, err))
+        ray.shutdown()
+        sys.exit(1)
 
 def get_channels():
     if(os.environ.get("ENV") == "docker" or os.environ.get("CHANNEL") is not None):

diff --git a/quotaclimat/data_processing/mediatree/channel_program.py b/quotaclimat/data_processing/mediatree/channel_program.py
@@ -116,7 +116,7 @@ def get_a_program_with_start_timestamp(df_program: pd.DataFrame, start_time: pd.
         logging.debug(f"matching_rows {matching_rows}")
         return matching_rows.iloc[0]['program_name'], matching_rows.iloc[0]['program_type']
     else:
-        logging.info(f"no programs found for {channel_name} - {start_time}")
+        logging.warning(f"no programs found for {channel_name} - {start_time}")
         return "", ""
 
 def process_subtitle(row, df_program):

diff --git a/quotaclimat/data_processing/mediatree/update_pg_keywords.py b/quotaclimat/data_processing/mediatree/update_pg_keywords.py
@@ -88,7 +88,13 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
                 ,number_of_keywords_ressources=new_number_of_keywords_ressources
                 )
             else:
-                program_name, program_name_type = get_a_program_with_start_timestamp(df_programs, pd.Timestamp(start).tz_convert('Europe/Paris'), channel_name)
+                logging.info(f"Updating program for keyword {keyword_id} - {channel_name} - original tz : {start}")
+                if(os.environ.get("ENV") == "prod"): # weird bug i don't want to know about
+                    start_tz = pd.Timestamp(start).tz_localize("UTC").tz_convert("Europe/Paris")
+                else:
+                    start_tz = pd.Timestamp(start).tz_convert("Europe/Paris")
+                logging.info(f"Updating program for keyword {keyword_id} - {channel_name} - converted tz : {start_tz}")
+                program_name, program_name_type = get_a_program_with_start_timestamp(df_programs, start_tz, channel_name)
                 update_keyword_row_program(session
                 ,keyword_id
                 ,channel_program=program_name
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	43	7	84%	36–38, 56–58, 63
insert_existing_data_example.py	19	3	84%	25–27
postgres/schemas
models.py	150	10	93%	124–131, 143–144, 202–203, 217–218
quotaclimat/data_ingestion
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	55	37	33%	21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	215	136	37%	44–48, 53–72, 76–79, 85, 88–130, 136–151, 155–156, 169–181, 185–191, 204–216, 219–223, 229, 265–266, 269–305, 308–310
channel_program.py	157	56	64%	28–30, 41–43, 60–61, 64–66, 93, 105, 114, 154–195
config.py	15	2	87%	7, 16
detect_keywords.py	209	8	96%	220, 278–285
update_pg_keywords.py	62	45	27%	14–107, 130, 137–141, 164–190, 197
utils.py	69	22	68%	27–51, 54, 63, 84–85
quotaclimat/utils
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
sentry.py	11	2	82%	22–23
TOTAL	1254	373	70%