feat: import only one channel based on var env (#105)

* feat: import only one channel based on var env

* fix: ci

README.md

@@ -269,6 +269,11 @@ Use env variable `START_DATE` like in docker compose (epoch second format : 1705
 
 Otherwise, default is yesterday midnight date.
 
+### Batch import based on channel
+Use env variable `CHANNEL` like in docker compose (string: tf1)
+
+Otherwise, default is all channels
+
 ### Batch update
 In case we have a new word detection logic, we must re apply it to all saved keywords inside our database.
 

docker-compose.yml

@@ -8,6 +8,7 @@ services:
     entrypoint: ["poetry", "run", "pytest", "--cov-report", "term:skip-covered", "--cov=quotaclimat", "--cov=postgres", "test/"]
     environment:
       ENV: docker
+      # CHANNEL: "fr3-idf"
       LOGLEVEL: DEBUG
       PYTHONPATH: /app
       POSTGRES_USER: user
@@ -133,8 +134,9 @@ services:
       POSTGRES_PORT: 5432
       PORT: 5050 # healthcheck
       HEALTHCHECK_SERVER: "0.0.0.0"
-     # START_DATE: 1704576615 # to test batch import 
+      # START_DATE: 1704576615 # to test batch import 
       # UPDATE: "true" # to batch update PG 
+      # CHANNEL : fr3-idf # to reimport only one channel
       MEDIATREE_USER : /run/secrets/username_api
       MEDIATREE_PASSWORD:  /run/secrets/pwd_api
       MEDIATREE_AUTH_URL: https://keywords.mediatree.fr/api/auth/token/

quotaclimat/data_processing/mediatree/api_import.py

@@ -43,20 +43,28 @@ async def update_pg_data(exit_event):
     update_keywords(session)
     exit_event.set()
 
+def get_channels():
+    if(os.environ.get("ENV") == "docker" or os.environ.get("CHANNEL") is not None):
+        default_channel = os.environ.get("CHANNEL") or "france2"
+        logging.warning(f"Only one channel of env var CHANNEL {default_channel} (default to france2) is used")
+
+        channels = [default_channel]
+    else: #prod  - all channels
+        logging.warning("All channels are used")
+        return ["tf1", "france2", "fr3-idf", "m6", "arte", "d8", "tmc", "bfmtv", "lci", "franceinfotv", "itele",
+        "europe1", "france-culture", "france-inter", "nrj", "rmc", "rtl", "rtl2"]
+
+    return channels
+
 async def get_and_save_api_data(exit_event):
     conn = connect_to_db()
     token=get_auth_token(password=password, user_name=USER)
     type_sub = 's2t'
 
     (start_date_to_query, end_epoch) = get_start_end_date_env_variable_with_default()
 
-    if(os.environ.get("ENV") == "docker"):
-        logging.warning("Docker cases - only some channels are used")
-        channels = ["france2"]
-    else: #prod    
-        channels = ["tf1", "france2", "m6", "arte", "d8", "tmc", "bfmtv", "lci", "franceinfotv", "itele",
-        "europe1", "france-culture", "france-inter", "nrj", "rmc", "rtl", "rtl2"]
-
+    channels = get_channels()
+
     range = get_date_range(start_date_to_query, end_epoch)
     logging.info(f"Number of date to query : {len(range)}")
     for date in range:

test/sitemap/test_mediatree.py

@@ -129,6 +129,12 @@ def test_transform_theme_query_includes():
 
     assert output == expected
 
+def test_get_channels():
+    if(os.environ.get("ENV") == "docker"):
+        assert get_channels() == ["france2"] # default for docker compose config
+    else:
+        assert get_channels() == ["tf1", "france2", "fr3-idf", "m6", "arte", "d8", "tmc", "bfmtv", "lci", "franceinfotv", "itele",
+        "europe1", "france-culture", "france-inter", "nrj", "rmc", "rtl", "rtl2"]
 
 def test_get_themes_keywords_duration():
     subtitles = [{