fix: slacking for really close subtitle near the 2 minute limit

dataforgoodfr · Feb 26, 2024 · 611ee9d · 611ee9d · github-actions · Feb 26, 2024
1 parent 51adc52
commit 611ee9d
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 6 deletions.
diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -45,9 +45,12 @@ def refresh_token(token, date):
 # use when word detection is changed
 async def update_pg_data(exit_event):
     logging.info("Updating already saved data from Postgresql")
-    session = get_db_session()
-    update_keywords(session)
-    exit_event.set()
+    try:
+        session = get_db_session()
+        update_keywords(session)
+        exit_event.set()
+    except Exception as err:
+        logging.error("Could update_pg_data %s:(%s) %s" % (type(err).__name__, err))
 
 def get_channels():
     if(os.environ.get("ENV") == "docker" or os.environ.get("CHANNEL") is not None):

diff --git a/quotaclimat/data_processing/mediatree/detect_keywords.py b/quotaclimat/data_processing/mediatree/detect_keywords.py
@@ -145,7 +145,12 @@ def get_keyword_by_fifteen_second_window(filtered_themes: List[dict], start: dat
         window_number = int( (keyword_info['timestamp'] - start.timestamp() * 1000) // (window_size_seconds) )
         logging.debug(f"Window number {window_number} - kwtimestamp {keyword_info['timestamp']} - start {start.timestamp() * 1000}")
         if window_number >= number_of_windows and window_number >= 0:
-            logging.error(f"Window number {window_number} is out of range - kwtimestamp {keyword_info['timestamp']} - start {start.timestamp() * 1000}")
+            if(window_number == number_of_windows): # give some slack to mediatree subtitle edge case
+                logging.warning(f"Edge cases around 2 minutes - still counting for one - kwtimestamp {keyword_info['timestamp']} - start {start.timestamp() * 1000}")
+                window_number = number_of_windows - 1
+                fifteen_second_window[window_number] = 1
+            else:
+                logging.error(f"Window number {window_number} is out of range - kwtimestamp {keyword_info['timestamp']} - start {start.timestamp() * 1000}")
         else:
             fifteen_second_window[window_number] = 1
 

diff --git a/test/sitemap/test_detect_keywords.py b/test/sitemap/test_detect_keywords.py
@@ -793,17 +793,29 @@ def test_simple_get_keyword_by_fifteen_second_window():
 
     assert get_keyword_by_fifteen_second_window(keywords_with_timestamp, start) == [1, 0, 0, 0, 0, 0, 0, 0]
 
-def test_out_of_bound_get_keyword_by_fifteen_second_window():
+def test_edge_out_of_bound_get_keyword_by_fifteen_second_window():
     keywords_with_timestamp = [
             {
                 "keyword" : 'conditions de vie sur terre',
-                "timestamp": original_timestamp + get_keyword_time_separation_ms() * 8,
+                "timestamp": original_timestamp + get_keyword_time_separation_ms() * 8 + 10, # edge case - still counting for one
+                "theme":"changement_climatique_constat",
+            }
+    ]
+
+    assert get_keyword_by_fifteen_second_window(keywords_with_timestamp, start) == [0, 0, 0, 0, 0, 0, 0, 1]
+
+def test_really_out_of_bound_get_keyword_by_fifteen_second_window():
+    keywords_with_timestamp = [
+            {
+                "keyword" : 'conditions de vie sur terre',
+                "timestamp": original_timestamp + get_keyword_time_separation_ms() * 15 + 10, # edge case - still counting for one
                 "theme":"changement_climatique_constat",
             }
     ]
 
     assert get_keyword_by_fifteen_second_window(keywords_with_timestamp, start) == [0, 0, 0, 0, 0, 0, 0, 0]
 
+
 def test_almost_out_of_bound_get_keyword_by_fifteen_second_window():
     keywords_with_timestamp = [
             {
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	46	7	85%	38–40, 59–61, 66
insert_existing_data_example.py	20	3	85%	25–27
postgres/schemas
models.py	71	15	79%	74–81, 91–92, 101–111
quotaclimat/data_analytics
analytics_signataire_charte.py	29	29	0%	1–67
bilan.py	108	108	0%	2–372
data_coverage.py	34	34	0%	1–94
exploration.py	125	125	0%	1–440
sitemap_analytics.py	118	118	0%	1–343
quotaclimat/data_ingestion
categorization_program_type.py	1	1	0%	1
config_youtube.py	1	1	0%	1
scaleway_db_backups.py	34	34	0%	1–74
scrap_chartejournalismeecologie_signataires.py	50	50	0%	1–169
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
scrap_tv_program.py	62	62	0%	1–149
scrap_youtube.py	114	114	0%	1–238
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	59	41	31%	21–42, 45–65, 69–80
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	180	105	42%	38–42, 47–53, 57–60, 66, 69–96, 102–117, 122–124, 149–156, 160–163, 167–173, 184–195, 198–202, 208, 234–235, 239, 243–262, 266–277
config.py	15	2	87%	7, 16
detect_keywords.py	114	3	97%	131–133
utils.py	66	22	67%	19, 30–54, 57, 76–77
quotaclimat/data_processing/sitemap
sitemap_processing.py	41	27	34%	15–19, 23–25, 29–47, 51–58, 66–96, 101–103
quotaclimat/utils
channels.py	6	6	0%	1–95
climate_keywords.py	2	2	0%	3–35
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	14	3	79%	22–24
plotly_theme.py	17	17	0%	1–56
sentry.py	10	2	80%	21–22
TOTAL	1588	965	39%