Monitoring/cron duration (#117)

* monitor: cron duration * fix: monitor cron duration
dataforgoodfr · Feb 25, 2024 · e9ffc16 · e9ffc16 · github-actions · Feb 25, 2024
1 parent 90ab6cc
commit e9ffc16
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 12 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -134,6 +134,7 @@ services:
       POSTGRES_PORT: 5432
       PORT: 5050 # healthcheck
       HEALTHCHECK_SERVER: "0.0.0.0"
+      SENTRY_DSN: prod_only
       # START_DATE: 1704576615 # to test batch import 
       # UPDATE: "true" # to batch update PG 
       # CHANNEL : fr3-idf # to reimport only one channel

diff --git a/quotaclimat/data_ingestion/ingest_db/ingest_sitemap_in_db.py b/quotaclimat/data_ingestion/ingest_db/ingest_sitemap_in_db.py
@@ -70,8 +70,8 @@ async def main():
             time.sleep(60 * minutes)
 
         res=health_check_task.cancel()
-        logging.info("Exiting with success")
-        sys.exit(0)
+    logging.info("Exiting with success")
+    sys.exit(0)
 
 if __name__ == "__main__":
     # create logger with 'spam_application'

diff --git a/quotaclimat/data_processing/mediatree/api_import.py b/quotaclimat/data_processing/mediatree/api_import.py
@@ -262,17 +262,9 @@ async def main():
         # Wait for both tasks to complete
         await event_finish.wait()
 
-        # only for scaleway - delay for serverless container
-        # Without this we have a CrashLoopBackOff (Kubernetes health error)
-        if (os.environ.get("ENV") != "dev" and os.environ.get("ENV") != "docker"):
-            minutes = 15
-            seconds_to_minute = 60
-            logging.warning(f"Sleeping {minutes} before safely exiting scaleway container")
-            sleep(seconds_to_minute * minutes)
-
         res=health_check_task.cancel()
-        logging.info("Exiting with success")
-        sys.exit(0)
+    logging.info("Exiting with success")
+    sys.exit(0)
 
 if __name__ == "__main__":
     # create logger with 'spam_application'
File	Stmts	Miss	Cover	Missing
postgres
insert_data.py	46	7	85%	38–40, 59–61, 66
insert_existing_data_example.py	20	3	85%	25–27
postgres/schemas
models.py	71	15	79%	74–81, 91–92, 101–111
quotaclimat/data_analytics
analytics_signataire_charte.py	29	29	0%	1–67
bilan.py	108	108	0%	2–372
data_coverage.py	34	34	0%	1–94
exploration.py	125	125	0%	1–440
sitemap_analytics.py	118	118	0%	1–343
quotaclimat/data_ingestion
categorization_program_type.py	1	1	0%	1
config_youtube.py	1	1	0%	1
scaleway_db_backups.py	34	34	0%	1–74
scrap_chartejournalismeecologie_signataires.py	50	50	0%	1–169
scrap_sitemap.py	134	17	87%	27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
scrap_tv_program.py	62	62	0%	1–149
scrap_youtube.py	114	114	0%	1–238
quotaclimat/data_ingestion/ingest_db
ingest_sitemap_in_db.py	58	41	29%	30–51, 54–74, 78–89
quotaclimat/data_ingestion/scrap_html
scrap_description_article.py	36	3	92%	19–20, 32
quotaclimat/data_processing/mediatree
api_import.py	178	102	43%	45–49, 54–57, 61–64, 70, 73–98, 104–119, 124–126, 151–158, 162–165, 169–175, 186–197, 200–204, 210, 235–236, 242, 244, 248–267, 271–282
config.py	15	2	87%	7, 16
detect_keywords.py	103	6	94%	115–122
utils.py	64	21	67%	27–51, 54, 73–74
quotaclimat/data_processing/sitemap
sitemap_processing.py	41	27	34%	15–19, 23–25, 29–47, 51–58, 66–96, 101–103
quotaclimat/utils
channels.py	6	6	0%	1–95
climate_keywords.py	2	2	0%	3–35
healthcheck_config.py	29	14	52%	22–24, 27–38
logger.py	14	3	79%	22–24
plotly_theme.py	17	17	0%	1–56
TOTAL	1559	962	38%