Data quality: normalize channel title (#271)

* wip * test: add real tests
dataforgoodfr · Oct 15, 2024 · 6ac72c8 · 6ac72c8 · github-actions · Oct 15, 2024
1 parent 7604fb5
commit 6ac72c8
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -399,3 +399,4 @@ There is a debt regarding the cleanest of the code right now. Let's just not mak
 
 ## Thanks
 * [Eleven-Strategy](https://www.welcometothejungle.com/fr/companies/eleven-strategy)
+* [Kevin Tessier](https://kevintessier.fr)
diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -255,6 +255,9 @@ def parse_reponse_subtitle(response_sub, channel = None, channel_program = "", c
                         inplace=True
             )
 
+            logging.debug("setting channel_title")
+            new_df['channel_title'] = new_df.apply(lambda x: get_channel_title_for_name(x['channel_name']), axis=1)
+
             logging.debug(f"setting program {channel_program}")
             # weird error if not using this way: (ValueError) format number 1 of "20h30 le samedi" is not recognized
             new_df['channel_program'] = new_df.apply(lambda x: channel_program, axis=1)

diff --git a/test/sitemap/test_mediatree.py b/test/sitemap/test_mediatree.py
@@ -41,7 +41,7 @@
                 "text": "france"
                 }
             ],
-            "channel":{"name":"m6","title":"M6","radio":false},"start":1704798000,
+            "channel":{"name":"m6","title":"fake m6","radio":false},"start":1704798000,
             "plaintext":"test1"
         },
         {
@@ -51,7 +51,7 @@
                 "text": "adaptation"
                 }
             ],
-            "channel":{"name":"tf1","title":"TF1","radio":false},"start":1704798120,
+            "channel":{"name":"tf1","title":"fake TF1","radio":false},"start":1704798120,
             "plaintext":"test2"}
     ],
     "elapsed_time_ms":335}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -399,3 +399,4 @@ There is a debt regarding the cleanest of the code right now. Let's just not mak

		## Thanks
		* [Eleven-Strategy](https://www.welcometothejungle.com/fr/companies/eleven-strategy)
		* [Kevin Tessier](https://kevintessier.fr)
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	43	7	84%	36–38, 56–58, 63
insert_existing_data_example.py	19	3	84%	25–27
postgres/schemas
models.py	157	11	93%	126–133, 146, 148–149, 214–215, 229–230
quotaclimat/data_ingestion
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	55	37	33%	21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	213	133	38%	44–48, 53–74, 78–81, 87, 90–132, 138–153, 158, 171–183, 187–193, 206–218, 221–225, 231, 269–270, 273–304, 307–309
channel_program.py	162	53	67%	21–23, 34–36, 53–54, 57–59, 98–99, 108, 124, 177, 180–216
config.py	15	2	87%	7, 16
detect_keywords.py	223	9	96%	222, 280–287, 323
update_pg_keywords.py	67	49	27%	15–108, 132, 135, 142–157, 180–206, 213
utils.py	79	25	68%	29–53, 56, 65, 86–87, 117–120
quotaclimat/utils
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	24	11	54%	22–24, 28–37
sentry.py	11	2	82%	22–23
TOTAL	1294	376	71%