feat: remove indirect from number_of_keywords (#108)

* feat: remove indirect from number_of_keywords

* log: to debug

quotaclimat/data_processing/mediatree/detect_keywords.py

@@ -70,7 +70,7 @@ def get_themes_keywords_duration(plaintext: str, subtitle_duration: List[str]) -
             keywords_with_timestamp.extend(keywords_to_add)
 
     if len(matching_themes) > 0:
-        return [matching_themes, keywords_with_timestamp, count_keywords_duration_overlap(keywords_with_timestamp)]
+        return [matching_themes, keywords_with_timestamp, count_keywords_duration_overlap_without_indirect(keywords_with_timestamp)]
     else:
         return [None, None, None]
 
@@ -107,25 +107,35 @@ def add_primary_key(df):
         logging.error(error)
         return get_consistent_hash("empty") #  TODO improve - should be a None ?
 
-def count_keywords_duration_overlap(keywords_with_timestamp: List[dict]) -> int:
-    if(len(keywords_with_timestamp)) <= 1:
-        return len(keywords_with_timestamp)
+def filter_indirect_words(keywords_with_timestamp: List[dict]) -> List[dict]:
+    return list(filter(lambda kw: 'indirectes' not in kw['theme'], keywords_with_timestamp))
+
+def count_keywords_duration_overlap_without_indirect(keywords_with_timestamp: List[dict]) -> int:
+    total_keywords = len(keywords_with_timestamp)
+    if(total_keywords) == 0:
+        return 0
     else:
         # in case keywords are not in the right order
-        sorted_keywords = iter(sorted(keywords_with_timestamp, key=lambda x: x['timestamp']))
-
-        count = 1
-        previous_timestamp = next(sorted_keywords)['timestamp']
-
-        for keyword_info in sorted_keywords:
-            current_timestamp = keyword_info['timestamp']
-            overlap_time = current_timestamp - previous_timestamp
-
-            if is_time_distance_between_keyword_enough(overlap_time):
-                logging.debug(f"No overlapping keyword {count} + 1 : {overlap_time}")
-                count += 1
-                previous_timestamp = current_timestamp
-            else:
-                logging.debug(f"Keyword timestamp overlap : {overlap_time} - current count is {count}")
-
-        return count
+        sorted_keywords = sorted(keywords_with_timestamp, key=lambda x: x['timestamp'])
+        filtered_themes = filter_indirect_words(sorted_keywords)
+        length_filtered_items = len(filtered_themes)
+        logging.debug(f"Before filtering {total_keywords} - After filtering indirect kw {length_filtered_items}")
+        if length_filtered_items > 0:
+            iter_filtered_themes = iter(filtered_themes)
+            count = 1
+            previous_timestamp = next(iter_filtered_themes)['timestamp']
+
+            for keyword_info in filtered_themes:
+                current_timestamp = keyword_info['timestamp']
+                overlap_time = current_timestamp - previous_timestamp
+
+                if is_time_distance_between_keyword_enough(overlap_time):
+                    logging.debug(f"No overlapping keyword {count} + 1 : {overlap_time}")
+                    count += 1
+                    previous_timestamp = current_timestamp
+                else:
+                    logging.debug(f"Keyword timestamp overlap : {overlap_time} - current count is {count}")
+
+            return count
+        else:
+            return 0

quotaclimat/data_processing/mediatree/keyword/keyword.py

@@ -1,5 +1,4 @@
 # Based on experts' document
-
 THEME_KEYWORDS = {
  "changement_climatique_constat" : [ # 1.1.2
             "climatique"

quotaclimat/data_processing/mediatree/update_pg_keywords.py

@@ -8,13 +8,21 @@
 from postgres.schemas.models import Keywords
 from quotaclimat.data_processing.mediatree.detect_keywords import *
 
-def update_keywords(session: Session) -> list:
+def update_keywords(session: Session, batch_size: int = 50000) -> list:
     saved_keywords = get_keywords_columns(session)
-    logging.info(f"Updating {len(saved_keywords)} saved keywords")
-    for keyword_id, plaintext, keywords_with_timestamp, number_of_keywords in saved_keywords:
-        new_number_of_keywords = count_keywords_duration_overlap(keywords_with_timestamp)
-        logging.debug(f"{keyword_id} new value {new_number_of_keywords}")
-        update_number_of_keywords(session, keyword_id, new_number_of_keywords)
+    total_updates = len(saved_keywords)
+    logging.info(f"Updating {total_updates} saved keywords")
+    for i in range(0, total_updates, batch_size):
+        batch_updates = saved_keywords[i:i+batch_size]
+        for keyword_id, plaintext, keywords_with_timestamp, number_of_keywords in batch_updates:
+            logging
+            new_number_of_keywords = count_keywords_duration_overlap_without_indirect(keywords_with_timestamp)
+            logging.debug(f"{keyword_id} new value {new_number_of_keywords}")
+            update_number_of_keywords(session, keyword_id, new_number_of_keywords)
+
+        logging.info(f"bulk update done {i} out of {total_updates}")
+        session.commit()
+
     logging.info("updated all keywords")
 
 
@@ -33,5 +41,4 @@ def update_number_of_keywords(session: Session, keyword_id: int, new_number_of_k
     session.query(Keywords).filter(Keywords.id == keyword_id).update(
         {Keywords.number_of_keywords: new_number_of_keywords},
         synchronize_session=False
-    )
-    session.commit()
+    )