Skip to content

Commit

Permalink
Add assembly enums and fix enum handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@anna-parker
anna-parker committed Jul 24, 2024
1 parent 2c63c7a commit 05cd091
Show file tree
Hide file tree
Showing 4 changed files with 205 additions and 104 deletions.
56 changes: 30 additions & 26 deletions ena-submission/scripts/create_project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,14 @@ def create_project(log_level, config_file):

while True:
# Check submission_table for newly added sequences
conditions = {"status_all": StatusAll.READY_TO_SUBMIT.name}
conditions = {"status_all": StatusAll.READY_TO_SUBMIT}
ready_to_submit = find_conditions_in_db(
db_config, table_name="submission_table", conditions=conditions
)
logger.debug(
f"Found {len(ready_to_submit)} entries in submission_table in status READY_TO_SUBMIT"
)
if len(ready_to_submit) > 0:
logger.debug(
f"Found {len(ready_to_submit)} entries in submission_table in status READY_TO_SUBMIT"
)
for row in ready_to_submit:
group_key = {"group_id": row["group_id"], "organism": row["organism"]}
seq_key = {"accession": row["accession"], "version": row["version"]}
Expand All @@ -125,9 +126,9 @@ def create_project(log_level, config_file):
db_config, table_name="project_table", conditions=group_key
)
if len(corresponding_project) == 1:
if corresponding_project[0]["status"] == Status.SUBMITTED.name:
if corresponding_project[0]["status"] == Status.SUBMITTED:
update_values = {
"status_all": StatusAll.SUBMITTED_PROJECT.name,
"status_all": StatusAll.SUBMITTED_PROJECT,
"center_name": corresponding_project[0]["center_name"],
}
number_rows_updated = update_db_where_conditions(
Expand All @@ -137,7 +138,7 @@ def create_project(log_level, config_file):
update_values=update_values,
)
else:
update_values = {"status_all": StatusAll.SUBMITTING_PROJECT.name}
update_values = {"status_all": StatusAll.SUBMITTING_PROJECT}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="submission_table",
Expand All @@ -152,21 +153,22 @@ def create_project(log_level, config_file):
}
project_table_entry = ProjectTableEntry(**entry)
add_to_project_table(db_config, project_table_entry)
update_values = {"status_all": StatusAll.SUBMITTING_PROJECT.name}
update_values = {"status_all": StatusAll.SUBMITTING_PROJECT}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="submission_table",
conditions=seq_key,
update_values=update_values,
)
conditions = {"status_all": StatusAll.SUBMITTING_PROJECT.name}
conditions = {"status_all": StatusAll.SUBMITTING_PROJECT}
submitting_project = find_conditions_in_db(
db_config, table_name="submission_table", conditions=conditions
)
logger.debug(
f"Found {len(submitting_project)} entries in submission_table in"
" status SUBMITTING_PROJECT"
)
if len(submitting_project) > 0:
logger.debug(
f"Found {len(submitting_project)} entries in submission_table in"
" status SUBMITTING_PROJECT"
)
for row in submitting_project:
group_key = {"group_id": row["group_id"], "organism": row["organism"]}
seq_key = {"accession": row["accession"], "version": row["version"]}
Expand All @@ -177,10 +179,10 @@ def create_project(log_level, config_file):
)
if (
len(corresponding_project) == 1
and corresponding_project[0]["status"] == Status.SUBMITTED.name
and corresponding_project[0]["status"] == Status.SUBMITTED
):
update_values = {
"status_all": StatusAll.SUBMITTED_PROJECT.name,
"status_all": StatusAll.SUBMITTED_PROJECT,
"center_name": corresponding_project[0]["center_name"],
}
number_rows_updated = update_db_where_conditions(
Expand All @@ -196,13 +198,14 @@ def create_project(log_level, config_file):
)
raise RuntimeError(error_msg)
# Check project_table for newly added sequences
conditions = {"status": Status.READY.name}
conditions = {"status": Status.READY}
ready_to_submit_project = find_conditions_in_db(
db_config, table_name="project_table", conditions=conditions
)
logger.debug(
f"Found {len(ready_to_submit_project)} entries in project_table in status READY"
)
if len(ready_to_submit_project) > 0:
logger.debug(
f"Found {len(ready_to_submit_project)} entries in project_table in status READY"
)
for row in ready_to_submit_project:
group_key = {"group_id": row["group_id"], "organism": row["organism"]}

Expand All @@ -211,7 +214,7 @@ def create_project(log_level, config_file):

project_set = construct_project_set_object(group_info, config, metadata_dict, row)
update_values = {
"status": Status.SUBMITTING.name,
"status": Status.SUBMITTING,
"center_name": group_info["institution"],
}
number_rows_updated = update_db_where_conditions(
Expand All @@ -231,7 +234,7 @@ def create_project(log_level, config_file):
project_creation_results: CreationResults = create_ena_project(config, project_set)
if project_creation_results.results:
update_values = {
"status": Status.SUBMITTED.name,
"status": Status.SUBMITTED,
"result": json.dumps(project_creation_results.results),
"finished_at": datetime.now(tz=pytz.utc),
}
Expand All @@ -253,7 +256,7 @@ def create_project(log_level, config_file):
logger.info(f"Project creation for group_id {row["group_id"]} succeeded!")
else:
update_values = {
"status": Status.HAS_ERRORS.name,
"status": Status.HAS_ERRORS,
"errors": json.dumps(project_creation_results.errors),
}
number_rows_updated = 0
Expand All @@ -275,10 +278,11 @@ def create_project(log_level, config_file):
entries_with_errors = find_errors_in_db(
db_config, "project_table", time_threshold=time_threshold
)
logger.info(
f"Found {len(entries_with_errors)} entries in project_table in status HAS_ERRORS"
f" for {time_threshold}m"
)
if len(entries_with_errors) > 0:
logger.info(
f"Found {len(entries_with_errors)} entries in project_table in status HAS_ERRORS"
f" for {time_threshold}m"
)
for row in entries_with_errors:
# TODO: Query ENA to check if project has in fact been created
# If created update project_table
Expand Down
54 changes: 30 additions & 24 deletions ena-submission/scripts/create_sample.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,13 +153,14 @@ def create_sample(log_level, config_file):

while True:
# Check submission_table for newly added sequences
conditions = {"status_all": StatusAll.SUBMITTED_PROJECT.name}
conditions = {"status_all": StatusAll.SUBMITTED_PROJECT}
ready_to_submit = find_conditions_in_db(
db_config, table_name="submission_table", conditions=conditions
)
logging.debug(
f"Found {len(ready_to_submit)} entries in submission_table in status SUBMITTED_PROJECT"
)
if len(ready_to_submit) > 0:
logging.debug(
f"Found {len(ready_to_submit)} entries in submission_table in status SUBMITTED_PROJECT"
)
for row in ready_to_submit:
seq_key = {"accession": row["accession"], "version": row["version"]}

Expand All @@ -168,16 +169,16 @@ def create_sample(log_level, config_file):
db_config, table_name="sample_table", conditions=seq_key
)
if len(corresponding_sample) == 1:
if corresponding_sample[0]["status"] == Status.SUBMITTED.name:
update_values = {"status_all": StatusAll.SUBMITTED_SAMPLE.name}
if corresponding_sample[0]["status"] == Status.SUBMITTED:
update_values = {"status_all": StatusAll.SUBMITTED_SAMPLE}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="submission_table",
conditions=seq_key,
update_values=update_values,
)
else:
update_values = {"status_all": StatusAll.SUBMITTING_SAMPLE.name}
update_values = {"status_all": StatusAll.SUBMITTING_SAMPLE}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="submission_table",
Expand All @@ -188,21 +189,22 @@ def create_sample(log_level, config_file):
# If not: create sample_entry, change status to SUBMITTING_SAMPLE
sample_table_entry = SampleTableEntry(**seq_key)
add_to_sample_table(db_config, sample_table_entry)
update_values = {"status_all": StatusAll.SUBMITTING_SAMPLE.name}
update_values = {"status_all": StatusAll.SUBMITTING_SAMPLE}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="submission_table",
conditions=seq_key,
update_values=update_values,
)
conditions = {"status_all": StatusAll.SUBMITTING_SAMPLE.name}
conditions = {"status_all": StatusAll.SUBMITTING_SAMPLE}
submitting_sample = find_conditions_in_db(
db_config, table_name="submission_table", conditions=conditions
)
logger.debug(
f"Found {len(submitting_sample)} entries in submission_table in"
" status SUBMITTING_SAMPLE"
)
if len(submitting_sample) > 0:
logger.debug(
f"Found {len(submitting_sample)} entries in submission_table in"
" status SUBMITTING_SAMPLE"
)
for row in submitting_sample:
seq_key = {"accession": row["accession"], "version": row["version"]}

Expand All @@ -212,9 +214,9 @@ def create_sample(log_level, config_file):
)
if (
len(corresponding_sample) == 1
and corresponding_sample[0]["status"] == Status.SUBMITTED.name
and corresponding_sample[0]["status"] == Status.SUBMITTED
):
update_values = {"status_all": StatusAll.SUBMITTED_SAMPLE.name}
update_values = {"status_all": StatusAll.SUBMITTED_SAMPLE}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="submission_table",
Expand All @@ -228,11 +230,14 @@ def create_sample(log_level, config_file):
)
raise RuntimeError(error_msg)
# Check sample_table for newly added sequences
conditions = {"status": Status.READY.name}
conditions = {"status": Status.READY}
ready_to_submit_sample = find_conditions_in_db(
db_config, table_name="sample_table", conditions=conditions
)
logger.debug(f"Found {len(ready_to_submit_sample)} entries in sample_table in status READY")
if len(ready_to_submit_sample) > 0:
logger.debug(
f"Found {len(ready_to_submit_sample)} entries in sample_table in status READY"
)
for row in ready_to_submit_sample:
seq_key = {"accession": row["accession"], "version": row["version"]}
sample_data_in_submission_table = find_conditions_in_db(
Expand All @@ -246,7 +251,7 @@ def create_sample(log_level, config_file):
sample_set = construct_sample_set_object(
config, organism_metadata, sample_metadata, center_name, row, organism
)
update_values = {"status": Status.SUBMITTING.name}
update_values = {"status": Status.SUBMITTING}
number_rows_updated = update_db_where_conditions(
db_config,
table_name="sample_table",
Expand All @@ -264,7 +269,7 @@ def create_sample(log_level, config_file):
sample_creation_results: CreationResults = create_ena_sample(config, sample_set)
if sample_creation_results.results:
update_values = {
"status": Status.SUBMITTED.name,
"status": Status.SUBMITTED,
"result": json.dumps(sample_creation_results.results),
"finished_at": datetime.now(tz=pytz.utc),
}
Expand All @@ -286,7 +291,7 @@ def create_sample(log_level, config_file):
logger.info(f"Sample creation for accession {row["accession"]} succeeded!")
else:
update_values = {
"status": Status.HAS_ERRORS.name,
"status": Status.HAS_ERRORS,
"errors": json.dumps(sample_creation_results.errors),
}
number_rows_updated = 0
Expand All @@ -308,10 +313,11 @@ def create_sample(log_level, config_file):
entries_with_errors = find_errors_in_db(
db_config, "sample_table", time_threshold=time_threshold
)
logger.info(
f"Found {len(entries_with_errors)} entries in sample_table "
f"in status HAS_ERRORS for {time_threshold}m"
)
if len(entries_with_errors) > 0:
logger.info(
f"Found {len(entries_with_errors)} entries in sample_table "
f"in status HAS_ERRORS for {time_threshold}m"
)
for row in entries_with_errors:
# TODO: Query ENA to check if sample has in fact been created
# If created update sample_table
Expand Down
Loading

0 comments on commit 05cd091

Please sign in to comment.