fix(ena-submission): fix ena request error handling (#2900)

* Handle requests.exceptions whenever using requests.get or requests.post * Add more logging * Update ena-submission/Snakefile * also remove config reloader --------- Co-authored-by: Cornelius Roemer <[email protected]>
loculus-project · Oct 4, 2024 · 495f050 · 495f050
1 parent 35bb851
commit 495f050
Show file tree

Hide file tree

Showing 9 changed files with 68 additions and 35 deletions.
diff --git a/ena-submission/scripts/call_loculus.py b/ena-submission/scripts/call_loculus.py
@@ -256,13 +256,21 @@ def record_factory(*args, **kwargs):
 
     if mode == "submit-external-metadata":
         logging.info("Submitting external metadata")
-        response = submit_external_metadata(metadata, config=config, organism=organism)
-        logging.info(f"Completed {mode}")
-        Path(output_file).write_text("", encoding="utf-8")
+        try:
+            response = submit_external_metadata(metadata, config=config, organism=organism)
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error submitting external metadata: {e}")
+        else:
+            logging.info(f"Completed {mode}")
+            Path(output_file).write_text("", encoding="utf-8")
 
     if mode == "get-released-data":
         logger.info("Getting released sequences")
-        response = fetch_released_entries(config, organism, remove_if_has_metadata)
+        response = None
+        try:
+            response = fetch_released_entries(config, organism, remove_if_has_metadata)
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error fetching released sequences: {e}")
         if response:
             Path(output_file).write_text(json.dumps(response), encoding="utf-8")
         else:

diff --git a/ena-submission/scripts/create_assembly.py b/ena-submission/scripts/create_assembly.py
@@ -439,7 +439,7 @@ def assembly_table_update(
     conditions = {"status": Status.WAITING}
     waiting = find_conditions_in_db(db_config, table_name="assembly_table", conditions=conditions)
     if len(waiting) > 0:
-        logger.debug(f"Found {len(waiting)} entries in assembly_table in status READY")
+        logger.debug(f"Found {len(waiting)} entries in assembly_table in status WAITING")
     # Check if ENA has assigned an accession, don't do this too frequently
     time = datetime.now(tz=pytz.utc)
     if not _last_ena_check or time - timedelta(minutes=time_threshold) > _last_ena_check:
@@ -609,6 +609,7 @@ def create_assembly(
     )
 
     while True:
+        logger.debug("Checking for assemblies to create")
         submission_table_start(db_config)
         submission_table_update(db_config)
 

diff --git a/ena-submission/scripts/create_project.py b/ena-submission/scripts/create_project.py
@@ -183,10 +183,8 @@ def submission_table_update(db_config: SimpleConnectionPool):
         db_config, table_name="submission_table", conditions=conditions
     )
     logger.debug(
-        (
-            f"Found {len(submitting_project)} entries in submission_table in",
-            " status SUBMITTING_PROJECT",
-        )
+        f"Found {len(submitting_project)} entries in submission_table in"
+        " status SUBMITTING_PROJECT"
     )
     for row in submitting_project:
         group_key = {"group_id": row["group_id"], "organism": row["organism"]}
@@ -397,6 +395,7 @@ def create_project(log_level, config_file, test=False, time_between_iterations=1
     )
 
     while True:
+        logger.debug("Checking for projects to create")
         submission_table_start(db_config)
         submission_table_update(db_config)
 

diff --git a/ena-submission/scripts/create_sample.py b/ena-submission/scripts/create_sample.py
@@ -439,6 +439,7 @@ def create_sample(log_level, config_file, test=False, time_between_iterations=10
     )
 
     while True:
+        logger.debug("Checking for samples to create")
         submission_table_start(db_config)
         submission_table_update(db_config)
 

diff --git a/ena-submission/scripts/ena_submission_helper.py b/ena-submission/scripts/ena_submission_helper.py
@@ -135,8 +135,14 @@ def get_project_xml(project_set):
             "PROJECT": dataclass_to_xml(project_set, root_name="PROJECT_SET"),
         }
 
-    xml = get_project_xml(project_set)
-    response = post_webin(config, xml)
+    try:
+        xml = get_project_xml(project_set)
+        response = post_webin(config, xml)
+    except requests.exceptions.RequestException as e:
+        error_message = f"Request failed with exception: {e}."
+        logger.error(error_message)
+        errors.append(error_message)
+        return CreationResult(results=None, errors=errors, warnings=warnings)
     if not response.ok:
         error_message = (
             f"Request failed with status:{response.status_code}. " f"Response: {response.text}."
@@ -184,9 +190,14 @@ def get_sample_xml(sample_set):
             "SAMPLE": dataclass_to_xml(sample_set, root_name="SAMPLE_SET"),
         }
         return files
-
-    xml = get_sample_xml(sample_set)
-    response = post_webin(config, xml)
+    try:
+        xml = get_sample_xml(sample_set)
+        response = post_webin(config, xml)
+    except requests.exceptions.RequestException as e:
+        error_message = f"Request failed with exception: {e}."
+        logger.error(error_message)
+        errors.append(error_message)
+        return CreationResult(results=None, errors=errors, warnings=warnings)
     if not response.ok:
         error_message = (
             f"Request failed with status:{response.status_code}. "
@@ -386,13 +397,17 @@ def get_ena_analysis_process(
     errors = []
     warnings = []
     assembly_results = {"segment_order": segment_order, "erz_accession": erz_accession}
-
-    response = requests.get(
-        url,
-        auth=HTTPBasicAuth(config.ena_submission_username, config.ena_submission_password),
-        timeout=10,  # wait a full 10 seconds for a response incase slow
-    )
-    response.raise_for_status()
+    try:
+        response = requests.get(
+            url,
+            auth=HTTPBasicAuth(config.ena_submission_username, config.ena_submission_password),
+            timeout=10,  # wait a full 10 seconds for a response incase slow
+        )
+    except requests.exceptions.RequestException as e:
+        error_message = f"Request failed with exception: {e}."
+        logger.error(error_message)
+        errors.append(error_message)
+        return CreationResult(results=None, errors=errors, warnings=warnings)
     if not response.ok:
         error_message = (
             f"ENA check failed with status:{response.status_code}. "

diff --git a/ena-submission/scripts/notifications.py b/ena-submission/scripts/notifications.py
@@ -85,5 +85,8 @@ def send_slack_notification(
         or time - timedelta(hours=time_threshold) > slack_config.last_notification_sent
     ):
         logger.warning(comment)
-        notify(slack_config, comment)
-        slack_config.last_notification_sent = time
+        try:
+            notify(slack_config, comment)
+            slack_config.last_notification_sent = time
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Error sending slack notification: {e}")
diff --git a/ena-submission/scripts/trigger_submission_to_ena.py b/ena-submission/scripts/trigger_submission_to_ena.py
@@ -103,19 +103,25 @@ def trigger_submission_to_ena(
             return
 
     while True:
+        logger.debug("Checking for new sequences to upload to submission_table")
         # In a loop get approved sequences uploaded to Github and upload to submission_table
-        response = requests.get(
-            config.github_url,
-            timeout=10,
-        )
-
-        if response.ok:
+        try:
+            response = requests.get(
+                config.github_url,
+                timeout=10,
+            )
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to retrieve file due to requests exception: {e}")
+            time.sleep(min_between_github_requests * 60)
+            continue
+        try:
             sequences_to_upload = response.json()
-        else:
-            error_msg = f"Failed to retrieve file: {response.status_code}"
-            logger.error(error_msg)
-        upload_sequences(db_config, sequences_to_upload)
-        time.sleep(min_between_github_requests * 60)  # Sleep for x min to not overwhelm github
+            upload_sequences(db_config, sequences_to_upload)
+        except Exception as upload_error:
+            logger.error(f"Failed to upload sequences: {upload_error}")
+        finally:
+            time.sleep(min_between_github_requests * 60)  # Sleep for x min to not overwhelm github
 
 
 if __name__ == "__main__":

diff --git a/ena-submission/scripts/upload_external_metadata_to_loculus.py b/ena-submission/scripts/upload_external_metadata_to_loculus.py
@@ -230,6 +230,7 @@ def upload_external_metadata(log_level, config_file, time_between_iterations=10)
     )
 
     while True:
+        logger.debug("Checking for external metadata to upload to Loculus")
         get_external_metadata_and_send_to_loculus(db_config, config)
         upload_handle_errors(
             db_config,

diff --git a/kubernetes/loculus/templates/ena-submission-deployment.yaml b/kubernetes/loculus/templates/ena-submission-deployment.yaml
@@ -133,7 +133,6 @@ spec:
             component: loculus-get-ena-submission-list-cronjob
           annotations:
             argocd.argoproj.io/sync-options: Replace=true
-            reloader.stakater.com/auto: "true"
         spec:
           restartPolicy: Never
           containers: