gzip content

dataforgoodfr · Dec 12, 2024 · 8f9a7bb · 8f9a7bb · github-actions · Dec 12, 2024
1 parent 709cdae
commit 8f9a7bb
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 14 deletions.
diff --git a/quotaclimat/data_processing/mediatree/s3/api_to_s3.py b/quotaclimat/data_processing/mediatree/s3/api_to_s3.py
@@ -7,7 +7,7 @@
 from time import sleep
 import sys
 import os
-import gc
+import gzip
 from quotaclimat.utils.healthcheck_config import run_health_check_server
 from quotaclimat.utils.logger import getLogger
 from quotaclimat.data_processing.mediatree.utils import *
@@ -30,7 +30,7 @@
 import ray
 import s3fs
 import boto3
-from io import StringIO
+from io import BytesIO
 
 from quotaclimat.utils.sentry import sentry_init
 logging.getLogger('modin.logger.default').setLevel(logging.ERROR)
@@ -62,7 +62,7 @@
 
 def get_bucket_key(date, channel):
     (year, month, day) = (date.year, date.month, date.day)
-    return f'year={year}/month={month:02}/day={day:02}/channel={channel}/data.json'
+    return f'year={year}/month={month:02}/day={day:02}/channel={channel}/data.json.gz'
 
 def save_to_s3(df: pd.DataFrame, channel: str, date: pd.Timestamp):
     logging.info(f"Saving DF with {len(df)} elements to S3 for {date} and channel {channel}")
@@ -74,14 +74,11 @@ def save_to_s3(df: pd.DataFrame, channel: str, date: pd.Timestamp):
     logging.info(f"s3://{BUCKET_NAME}/{object_key}")
 
     try:
-        json_buffer = StringIO()
-        json_buffer = df.to_json(
-            None,
-            index=False,
-            orient='records',
-            lines=False
-        )
-        s3_client.put_object(Bucket=BUCKET_NAME, Key=object_key, Body=json_buffer)
+        json_buffer = BytesIO()
+        with gzip.GzipFile(fileobj=json_buffer, mode='w') as gz:
+            df.to_json(gz, orient='records', lines=False)
+
+        s3_client.put_object(Bucket=BUCKET_NAME, Key=object_key, Body=json_buffer.getvalue())
 
         logging.info(f"Uploaded partition: {object_key}")
     except Exception as e:
@@ -117,7 +114,7 @@ async def get_and_save_api_data(exit_event):
                             channel_program_type = str(program.program_type)
                             logging.info(f"Querying API for {channel} - {channel_program} - {channel_program_type} - {start_epoch} - {end_epoch}")
                             df = get_df_api(token, type_sub, start_epoch, channel, end_epoch, channel_program, channel_program_type)
-                            df.drop(columns=['srt'], axis=1, inplace=True)
+
                             if(df is not None):
                                 df_res = pd.concat([df_res, df ], ignore_index=True)
                             else:

diff --git a/test/s3/test_s3.py b/test/s3/test_s3.py
@@ -7,11 +7,11 @@ def test_get_bucket_key():
     friday_6h26 = 1726719981
     date = pd.to_datetime(friday_6h26, unit='s', utc=True)
     channel = "tf1"
-    assert get_bucket_key(date, channel) == "year=2024/month=09/day=19/channel=tf1/data.json"
+    assert get_bucket_key(date, channel) == "year=2024/month=09/day=19/channel=tf1/data.json.gz"
 
 
 def test_get_bucket_key_first_of_the_month():
     first_december = 1733040125
     date = pd.to_datetime(first_december, unit='s', utc=True)
     channel = "tf1"
-    assert get_bucket_key(date, channel) == "year=2024/month=12/day=01/channel=tf1/data.json"
+    assert get_bucket_key(date, channel) == "year=2024/month=12/day=01/channel=tf1/data.json.gz"
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	43	7	84%	36–38, 56–58, 63
insert_existing_data_example.py	19	3	84%	25–27
postgres/schemas
models.py	168	11	93%	137–144, 157, 159–160, 225–226, 240–241
quotaclimat/data_ingestion
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	55	37	33%	21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	213	133	38%	44–48, 53–74, 78–81, 87, 90–132, 138–153, 158, 171–183, 187–193, 206–218, 221–225, 231, 269–270, 273–304, 307–309
channel_program.py	162	57	65%	21–23, 34–36, 53–54, 57–59, 98–99, 108, 124, 175–216
config.py	15	2	87%	7, 16
detect_keywords.py	252	16	94%	111–118, 126–127, 271, 341–348, 390
update_pg_keywords.py	67	49	27%	15–130, 154, 157, 164–179, 213–250, 257
utils.py	79	25	68%	29–53, 56, 65, 86–87, 117–120
quotaclimat/data_processing/mediatree/s3
api_to_s3.py	121	68	44%	68–86, 89–133, 136–162, 165–167
quotaclimat/utils
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
sentry.py	11	2	82%	22–23
TOTAL	1455	455	69%