diff --git a/ena-submission/scripts/call_loculus.py b/ena-submission/scripts/call_loculus.py index df0dda936..0e69477ed 100644 --- a/ena-submission/scripts/call_loculus.py +++ b/ena-submission/scripts/call_loculus.py @@ -256,13 +256,21 @@ def record_factory(*args, **kwargs): if mode == "submit-external-metadata": logging.info("Submitting external metadata") - response = submit_external_metadata(metadata, config=config, organism=organism) - logging.info(f"Completed {mode}") - Path(output_file).write_text("", encoding="utf-8") + try: + response = submit_external_metadata(metadata, config=config, organism=organism) + except requests.exceptions.RequestException as e: + logger.error(f"Error submitting external metadata: {e}") + else: + logging.info(f"Completed {mode}") + Path(output_file).write_text("", encoding="utf-8") if mode == "get-released-data": logger.info("Getting released sequences") - response = fetch_released_entries(config, organism, remove_if_has_metadata) + response = None + try: + response = fetch_released_entries(config, organism, remove_if_has_metadata) + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching released sequences: {e}") if response: Path(output_file).write_text(json.dumps(response), encoding="utf-8") else: diff --git a/ena-submission/scripts/create_assembly.py b/ena-submission/scripts/create_assembly.py index eb31f786d..955043e46 100644 --- a/ena-submission/scripts/create_assembly.py +++ b/ena-submission/scripts/create_assembly.py @@ -439,7 +439,7 @@ def assembly_table_update( conditions = {"status": Status.WAITING} waiting = find_conditions_in_db(db_config, table_name="assembly_table", conditions=conditions) if len(waiting) > 0: - logger.debug(f"Found {len(waiting)} entries in assembly_table in status READY") + logger.debug(f"Found {len(waiting)} entries in assembly_table in status WAITING") # Check if ENA has assigned an accession, don't do this too frequently time = datetime.now(tz=pytz.utc) if not _last_ena_check or time - timedelta(minutes=time_threshold) > _last_ena_check: @@ -609,6 +609,7 @@ def create_assembly( ) while True: + logger.debug("Checking for assemblies to create") submission_table_start(db_config) submission_table_update(db_config) diff --git a/ena-submission/scripts/create_project.py b/ena-submission/scripts/create_project.py index 3a6bec28d..437d037cd 100644 --- a/ena-submission/scripts/create_project.py +++ b/ena-submission/scripts/create_project.py @@ -183,10 +183,8 @@ def submission_table_update(db_config: SimpleConnectionPool): db_config, table_name="submission_table", conditions=conditions ) logger.debug( - ( - f"Found {len(submitting_project)} entries in submission_table in", - " status SUBMITTING_PROJECT", - ) + f"Found {len(submitting_project)} entries in submission_table in" + " status SUBMITTING_PROJECT" ) for row in submitting_project: group_key = {"group_id": row["group_id"], "organism": row["organism"]} @@ -397,6 +395,7 @@ def create_project(log_level, config_file, test=False, time_between_iterations=1 ) while True: + logger.debug("Checking for projects to create") submission_table_start(db_config) submission_table_update(db_config) diff --git a/ena-submission/scripts/create_sample.py b/ena-submission/scripts/create_sample.py index ece6f9a55..c0fec4844 100644 --- a/ena-submission/scripts/create_sample.py +++ b/ena-submission/scripts/create_sample.py @@ -439,6 +439,7 @@ def create_sample(log_level, config_file, test=False, time_between_iterations=10 ) while True: + logger.debug("Checking for samples to create") submission_table_start(db_config) submission_table_update(db_config) diff --git a/ena-submission/scripts/ena_submission_helper.py b/ena-submission/scripts/ena_submission_helper.py index 418a5e1c7..04649ee96 100644 --- a/ena-submission/scripts/ena_submission_helper.py +++ b/ena-submission/scripts/ena_submission_helper.py @@ -135,8 +135,14 @@ def get_project_xml(project_set): "PROJECT": dataclass_to_xml(project_set, root_name="PROJECT_SET"), } - xml = get_project_xml(project_set) - response = post_webin(config, xml) + try: + xml = get_project_xml(project_set) + response = post_webin(config, xml) + except requests.exceptions.RequestException as e: + error_message = f"Request failed with exception: {e}." + logger.error(error_message) + errors.append(error_message) + return CreationResult(results=None, errors=errors, warnings=warnings) if not response.ok: error_message = ( f"Request failed with status:{response.status_code}. " f"Response: {response.text}." @@ -184,9 +190,14 @@ def get_sample_xml(sample_set): "SAMPLE": dataclass_to_xml(sample_set, root_name="SAMPLE_SET"), } return files - - xml = get_sample_xml(sample_set) - response = post_webin(config, xml) + try: + xml = get_sample_xml(sample_set) + response = post_webin(config, xml) + except requests.exceptions.RequestException as e: + error_message = f"Request failed with exception: {e}." + logger.error(error_message) + errors.append(error_message) + return CreationResult(results=None, errors=errors, warnings=warnings) if not response.ok: error_message = ( f"Request failed with status:{response.status_code}. " @@ -386,13 +397,17 @@ def get_ena_analysis_process( errors = [] warnings = [] assembly_results = {"segment_order": segment_order, "erz_accession": erz_accession} - - response = requests.get( - url, - auth=HTTPBasicAuth(config.ena_submission_username, config.ena_submission_password), - timeout=10, # wait a full 10 seconds for a response incase slow - ) - response.raise_for_status() + try: + response = requests.get( + url, + auth=HTTPBasicAuth(config.ena_submission_username, config.ena_submission_password), + timeout=10, # wait a full 10 seconds for a response incase slow + ) + except requests.exceptions.RequestException as e: + error_message = f"Request failed with exception: {e}." + logger.error(error_message) + errors.append(error_message) + return CreationResult(results=None, errors=errors, warnings=warnings) if not response.ok: error_message = ( f"ENA check failed with status:{response.status_code}. " diff --git a/ena-submission/scripts/notifications.py b/ena-submission/scripts/notifications.py index ecc5be8bc..eb9265291 100644 --- a/ena-submission/scripts/notifications.py +++ b/ena-submission/scripts/notifications.py @@ -85,5 +85,8 @@ def send_slack_notification( or time - timedelta(hours=time_threshold) > slack_config.last_notification_sent ): logger.warning(comment) - notify(slack_config, comment) - slack_config.last_notification_sent = time + try: + notify(slack_config, comment) + slack_config.last_notification_sent = time + except requests.exceptions.RequestException as e: + logger.error(f"Error sending slack notification: {e}") diff --git a/ena-submission/scripts/trigger_submission_to_ena.py b/ena-submission/scripts/trigger_submission_to_ena.py index 6d00edd96..8681888df 100644 --- a/ena-submission/scripts/trigger_submission_to_ena.py +++ b/ena-submission/scripts/trigger_submission_to_ena.py @@ -103,19 +103,25 @@ def trigger_submission_to_ena( return while True: + logger.debug("Checking for new sequences to upload to submission_table") # In a loop get approved sequences uploaded to Github and upload to submission_table - response = requests.get( - config.github_url, - timeout=10, - ) - - if response.ok: + try: + response = requests.get( + config.github_url, + timeout=10, + ) + response.raise_for_status() + except requests.exceptions.RequestException as e: + logger.error(f"Failed to retrieve file due to requests exception: {e}") + time.sleep(min_between_github_requests * 60) + continue + try: sequences_to_upload = response.json() - else: - error_msg = f"Failed to retrieve file: {response.status_code}" - logger.error(error_msg) - upload_sequences(db_config, sequences_to_upload) - time.sleep(min_between_github_requests * 60) # Sleep for x min to not overwhelm github + upload_sequences(db_config, sequences_to_upload) + except Exception as upload_error: + logger.error(f"Failed to upload sequences: {upload_error}") + finally: + time.sleep(min_between_github_requests * 60) # Sleep for x min to not overwhelm github if __name__ == "__main__": diff --git a/ena-submission/scripts/upload_external_metadata_to_loculus.py b/ena-submission/scripts/upload_external_metadata_to_loculus.py index f91f89ec5..b66f05142 100644 --- a/ena-submission/scripts/upload_external_metadata_to_loculus.py +++ b/ena-submission/scripts/upload_external_metadata_to_loculus.py @@ -230,6 +230,7 @@ def upload_external_metadata(log_level, config_file, time_between_iterations=10) ) while True: + logger.debug("Checking for external metadata to upload to Loculus") get_external_metadata_and_send_to_loculus(db_config, config) upload_handle_errors( db_config, diff --git a/kubernetes/loculus/templates/ena-submission-deployment.yaml b/kubernetes/loculus/templates/ena-submission-deployment.yaml index 2e64633eb..f52c53cde 100644 --- a/kubernetes/loculus/templates/ena-submission-deployment.yaml +++ b/kubernetes/loculus/templates/ena-submission-deployment.yaml @@ -133,7 +133,6 @@ spec: component: loculus-get-ena-submission-list-cronjob annotations: argocd.argoproj.io/sync-options: Replace=true - reloader.stakater.com/auto: "true" spec: restartPolicy: Never containers: