Merge branch 'main' into feat/2258-edit-group-info

backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt

@@ -749,7 +749,8 @@ open class SubmissionDatabaseService(
                 SequenceEntriesTable.versionColumn,
                 SequenceEntriesTable.versionCommentColumn,
                 SequenceEntriesTable.submissionIdColumn,
-                SequenceEntriesTable.submitterColumn, SequenceEntriesTable.groupIdColumn,
+                SequenceEntriesTable.submitterColumn,
+                SequenceEntriesTable.groupIdColumn,
                 SequenceEntriesTable.submittedAtTimestampColumn,
                 SequenceEntriesTable.isRevocationColumn,
                 SequenceEntriesTable.organismColumn,

ena-submission/config/config.yaml

@@ -2,7 +2,7 @@ backend_url: http://localhost:8079/
 keycloak_token_url: http://localhost:8083/realms/loculus/protocol/openid-connect/token
 db_username: postgres
 db_password: unsecure
-db_url: "jdbc:postgresql://127.0.0.1:5432/loculus?options=-c%20search_path%3Dena_deposition_schema"
+db_url: "jdbc:postgresql://127.0.0.1:5432/loculus"
 organisms:
   cchf:
     ingest:

ena-submission/scripts/create_assembly.py

@@ -8,12 +8,12 @@
 import pytz
 import yaml
 from ena_submission_helper import (
-    CreationResults,
-    check_ena,
+    CreationResult,
     create_chromosome_list,
     create_ena_assembly,
     create_fasta,
     create_manifest,
+    get_ena_analysis_process,
     get_ena_config,
 )
 from ena_types import (
@@ -100,14 +100,12 @@ def create_chromosome_list_object(
     return AssemblyChromosomeListFile(chromosomes=entries)
 
 
-def get_segment_order(unaligned_sequences) -> list[str]:
+def get_segment_order(unaligned_sequences: dict[str, str]) -> list[str]:
+    """Order in which we put the segments in the chromosome list file"""
     segment_order = []
-    if len(unaligned_sequences.keys()) > 1:
-        for segment_name, item in unaligned_sequences.items():
-            if item:  # Only list sequenced segments
-                segment_order.append(segment_name)
-    else:
-        segment_order.append("main")
+    for segment_name, item in unaligned_sequences.items():
+        if item:  # Only list sequenced segments
+            segment_order.append(segment_name)
     return sorted(segment_order)
 
 
@@ -371,14 +369,14 @@ def assembly_table_create(
         segment_order = get_segment_order(
             sample_data_in_submission_table[0]["unaligned_nucleotide_sequences"]
         )
-        assembly_creation_results: CreationResults = create_ena_assembly(
+        assembly_creation_results: CreationResult = create_ena_assembly(
             ena_config, manifest_file, center_name=center_name, test=test
         )
-        if assembly_creation_results.results:
-            assembly_creation_results.results["segment_order"] = segment_order
+        if assembly_creation_results.result:
+            assembly_creation_results.result["segment_order"] = segment_order
             update_values = {
                 "status": Status.WAITING,
-                "result": json.dumps(assembly_creation_results.results),
+                "result": json.dumps(assembly_creation_results.result),
             }
             number_rows_updated = 0
             tries = 0
@@ -448,17 +446,52 @@ def assembly_table_update(
         logger.debug("Checking state in ENA")
         for row in waiting:
             seq_key = {"accession": row["accession"], "version": row["version"]}
-            segment_order = row["result"]["segment_order"]
-            check_results: CreationResults = check_ena(
-                ena_config, row["result"]["erz_accession"], segment_order
+            # Previous means from the last time the entry was checked, from db
+            previous_result = row["result"]
+            segment_order = previous_result["segment_order"]
+            new_result: CreationResult = get_ena_analysis_process(
+                ena_config, previous_result["erz_accession"], segment_order
             )
             _last_ena_check = time
-            if not check_results.results:
+
+            if not new_result.result:
                 continue
 
+            result_contains_gca_accession = "gca_accession" in new_result.result
+            result_contains_insdc_accession = any(
+                key.startswith("insdc_accession_full") for key in new_result.result
+            )
+
+            if not (result_contains_gca_accession and result_contains_insdc_accession):
+                if previous_result == new_result.result:
+                    continue
+                update_values = {
+                    "status": Status.WAITING,
+                    "result": json.dumps(new_result.result),
+                    "finished_at": datetime.now(tz=pytz.utc),
+                }
+                number_rows_updated = 0
+                tries = 0
+                while number_rows_updated != 1 and tries < retry_number:
+                    if tries > 0:
+                        logger.warning(
+                            f"Assembly partially in ENA but DB update failed - reentry DB update #{tries}."
+                        )
+                    number_rows_updated = update_db_where_conditions(
+                        db_config,
+                        table_name="assembly_table",
+                        conditions=seq_key,
+                        update_values=update_values,
+                    )
+                    tries += 1
+                if number_rows_updated == 1:
+                    logger.info(
+                        f"Partial results of assembly submission for accession {row["accession"]} returned!"
+                    )
+                continue
             update_values = {
                 "status": Status.SUBMITTED,
-                "result": json.dumps(check_results.results),
+                "result": json.dumps(new_result.result),
                 "finished_at": datetime.now(tz=pytz.utc),
             }
             number_rows_updated = 0
@@ -522,7 +555,10 @@ def assembly_table_handle_errors(
             f" status WAITING for over {time_threshold_waiting}h"
         )
         send_slack_notification(
-            config, error_msg, time=datetime.now(tz=pytz.utc), time_threshold=slack_time_threshold
+            error_msg,
+            slack_config,
+            time=datetime.now(tz=pytz.utc),
+            time_threshold=slack_time_threshold,
         )
 
 

ena-submission/scripts/create_project.py

@@ -8,7 +8,7 @@
 import pytz
 import yaml
 from call_loculus import get_group_info
-from ena_submission_helper import CreationResults, create_ena_project, get_ena_config
+from ena_submission_helper import CreationResult, create_ena_project, get_ena_config
 from ena_types import (
     OrganismType,
     ProjectLink,
@@ -275,11 +275,11 @@ def project_table_create(
         logger.info(
             f"Starting Project creation for group_id {row["group_id"]} organism {row["organism"]}"
         )
-        project_creation_results: CreationResults = create_ena_project(ena_config, project_set)
-        if project_creation_results.results:
+        project_creation_results: CreationResult = create_ena_project(ena_config, project_set)
+        if project_creation_results.result:
             update_values = {
                 "status": Status.SUBMITTED,
-                "result": json.dumps(project_creation_results.results),
+                "result": json.dumps(project_creation_results.result),
                 "finished_at": datetime.now(tz=pytz.utc),
             }
             number_rows_updated = 0

ena-submission/scripts/create_sample.py

@@ -8,7 +8,7 @@
 import click
 import pytz
 import yaml
-from ena_submission_helper import CreationResults, create_ena_sample, get_ena_config
+from ena_submission_helper import CreationResult, create_ena_sample, get_ena_config
 from ena_types import (
     ProjectLink,
     SampleAttribute,
@@ -320,11 +320,11 @@ def sample_table_create(
             )
             continue
         logger.info(f"Starting sample creation for accession {row["accession"]}")
-        sample_creation_results: CreationResults = create_ena_sample(ena_config, sample_set)
-        if sample_creation_results.results:
+        sample_creation_results: CreationResult = create_ena_sample(ena_config, sample_set)
+        if sample_creation_results.result:
             update_values = {
                 "status": Status.SUBMITTED,
-                "result": json.dumps(sample_creation_results.results),
+                "result": json.dumps(sample_creation_results.result),
                 "finished_at": datetime.now(tz=pytz.utc),
             }
             number_rows_updated = 0