diff --git a/.github/workflows/dev-hail-search-release.yaml b/.github/workflows/dev-hail-search-release.yaml
index 18bad94549..9507e2b93d 100644
--- a/.github/workflows/dev-hail-search-release.yaml
+++ b/.github/workflows/dev-hail-search-release.yaml
@@ -47,11 +47,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
- - name: update image tag in the broad seqr chart
+ - name: update image tag in the dev broad seqr chart
uses: mikefarah/yq@v4.22.1
with:
cmd: >
- yq -i '.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/broad-seqr/values-dev.yaml
+ yq -i '.hail-search.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/dev-broad-seqr/values.yaml
- name: Commit and Push changes
uses: Andro999b/push@v1.3
diff --git a/.github/workflows/dev-release.yaml b/.github/workflows/dev-release.yaml
index 7df887327d..193110b0d0 100644
--- a/.github/workflows/dev-release.yaml
+++ b/.github/workflows/dev-release.yaml
@@ -47,11 +47,11 @@ jobs:
persist-credentials: false
fetch-depth: 0
- - name: update image tag in the broad seqr chart
+ - name: update image tag in the dev broad seqr chart
uses: mikefarah/yq@v4.22.1
with:
cmd: >
- yq -i '.seqr.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/broad-seqr/values-dev.yaml
+ yq -i '.seqr.image.tag = "${{ github.event.workflow_run.head_sha }}"' charts/dev-broad-seqr/values.yaml
- name: Commit and Push changes
uses: Andro999b/push@v1.3
diff --git a/requirements.txt b/requirements.txt
index d825a8bb0c..ca943f941f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -20,7 +20,7 @@ cffi==1.15.1
# via cryptography
charset-normalizer==3.0.1
# via requests
-cryptography==42.0.6
+cryptography==42.0.8
# via social-auth-core
defusedxml==0.7.1
# via
@@ -131,7 +131,7 @@ pytz==2022.7.1
# django-notifications-hq
redis==4.5.4
# via -r requirements.in
-requests==2.32.0
+requests==2.32.2
# via
# -r requirements.in
# django-anymail
diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py
index 7a938bca2e..a00e965e1b 100644
--- a/seqr/views/apis/report_api.py
+++ b/seqr/views/apis/report_api.py
@@ -442,7 +442,13 @@ def _add_row(row, family_id, row_type):
(FINDINGS_TABLE, genetic_findings_rows),
]
- files, warnings = _populate_gregor_files(file_data)
+ files, warnings, errors = _populate_gregor_files(file_data)
+
+ if errors and not request_json.get('overrideValidation'):
+ raise ErrorsWarningsException(errors, warnings)
+ else:
+ warnings = errors + warnings
+
write_multiple_files_to_gs(files, file_path, request.user, file_format='tsv')
return create_json_response({
@@ -700,10 +706,7 @@ def _populate_gregor_files(file_data):
for column, config in table_config.items():
_validate_column_data(column, file_name, data, column_validator=config, warnings=warnings, errors=errors)
- if errors:
- raise ErrorsWarningsException(errors, warnings)
-
- return files, warnings
+ return files, warnings, errors
def _load_data_model_validators():
diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py
index 056bdf0227..00ade3156d 100644
--- a/seqr/views/apis/report_api_tests.py
+++ b/seqr/views/apis/report_api_tests.py
@@ -622,6 +622,20 @@
],
]
+READ_TABLE_HEADER = [
+ 'aligned_dna_short_read_id', 'experiment_dna_short_read_id', 'aligned_dna_short_read_file',
+ 'aligned_dna_short_read_index_file', 'md5sum', 'reference_assembly', 'reference_assembly_uri',
+ 'reference_assembly_details', 'mean_coverage', 'alignment_software', 'analysis_details', 'quality_issues',
+]
+READ_SET_TABLE_HEADER = ['aligned_dna_short_read_set_id', 'aligned_dna_short_read_id']
+RNA_TABLE_HEADER = [
+ 'experiment_rna_short_read_id', 'analyte_id', 'experiment_sample_id', 'seq_library_prep_kit_method',
+ 'read_length', 'experiment_type', 'date_data_generation', 'sequencing_platform', 'library_prep_type',
+ 'single_or_paired_ends', 'within_site_batch_name', 'RIN', 'estimated_library_size', 'total_reads',
+ 'percent_rRNA', 'percent_mRNA', '5prime3prime_bias', 'percent_mtRNA', 'percent_Globin', 'percent_UMI',
+ 'percent_GC', 'percent_chrX_Y',
+]
+
class ReportAPITest(AirtableTest):
@@ -817,12 +831,13 @@ def test_gregor_export(self, mock_subprocess, mock_temp_dir, mock_open, mock_dat
'The following entries are missing recommended "age_at_enrollment" in the "participant" table: Broad_HG00731, Broad_NA20870, Broad_NA20872, Broad_NA20875, Broad_NA20876, Broad_NA20881, Broad_NA20888',
'The following entries are missing recommended "known_condition_name" in the "genetic_findings" table: Broad_HG00731_19_1912632, Broad_HG00731_19_1912633, Broad_HG00731_19_1912634, Broad_HG00731_1_248367227',
]
- self.assertListEqual(response.json()['warnings'], [
+ validation_warnings = [
'The following columns are specified as "enumeration" in the "participant" data model but are missing the allowed values definition: prior_testing',
'The following columns are included in the "participant" data model but have an unsupported data type: internal_project_id (reference)',
'The following columns are computed for the "participant" table but are missing from the data model: age_at_last_observation, ancestry_detail, missing_variant_case, pmid_id',
- ] + recommended_warnings)
- self.assertListEqual(response.json()['errors'], [
+ ] + recommended_warnings
+ self.assertListEqual(response.json()['warnings'], validation_warnings)
+ validation_errors = [
f'No data model found for "{file}" table' for file in reversed(EXPECTED_GREGOR_FILES) if file not in INVALID_MODEL_TABLES
] + [
'The following tables are required in the data model but absent from the reports: subject, dna_read_data_set',
@@ -838,29 +853,78 @@ def test_gregor_export(self, mock_subprocess, mock_temp_dir, mock_open, mock_dat
'The following entries have invalid values for "date_data_generation" (from Airtable) in the "experiment_rna_short_read" table. Allowed values have data type float. Invalid values: NA19679 (2023-02-11)',
'The following entries are missing required "experiment_id" (from Airtable) in the "genetic_findings" table: Broad_NA19675_1_21_3343353',
'The following entries have non-unique values for "experiment_id" (from Airtable) in the "genetic_findings" table: Broad_exome_VCGS_FAM203_621_D2 (Broad_HG00731_19_1912632, Broad_HG00731_19_1912633, Broad_HG00731_19_1912634, Broad_HG00731_1_248367227)',
- ])
+ ]
+ self.assertListEqual(response.json()['errors'], validation_errors)
+
+ mock_open.reset_mock()
+ response = self.client.post(
+ url, content_type='application/json', data=json.dumps({**body, 'overrideValidation': True})
+ )
+ self.assertEqual(response.status_code, 200)
+ expected_response = {
+ 'info': ['Successfully validated and uploaded Gregor Report for 9 families'],
+ 'warnings': validation_errors + validation_warnings,
+ }
+ self.assertDictEqual(response.json(), expected_response)
+ participant_file, read_file, read_set_file, rna_file, genetic_findings_file = self._get_expected_gregor_files(
+ mock_open, mock_subprocess, INVALID_MODEL_TABLES.keys()
+ )
+ self._assert_expected_file(participant_file, [
+ [c for c in PARTICIPANT_TABLE[0] if c not in {'pmid_id', 'ancestry_detail', 'age_at_last_observation', 'missing_variant_case'}],
+ [
+ 'Broad_NA19675_1', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', 'Yes', 'IKBKAP|CCDC102B|CMA - normal',
+ 'Broad_1', 'Broad_NA19678', 'Broad_NA19679', '', 'Self', '', 'Male', '', 'Middle Eastern or North African',
+ '', 'Affected', 'myopathy', '18', 'Unsolved',
+ ], [
+ 'Broad_NA19678', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', 'Broad_1', '0', '0', '', '',
+ '', 'Male', '', '', '', 'Unaffected', 'myopathy', '', 'Unaffected',
+ ], [
+ 'Broad_HG00731', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', 'Broad_2', 'Broad_HG00732',
+ 'Broad_HG00733', '', 'Self', '', 'Female', '', '', 'Hispanic or Latino', 'Affected',
+ 'microcephaly; seizures', '', 'Unsolved',
+ ]], additional_calls=10)
+ self._assert_expected_file(read_file, [READ_TABLE_HEADER, [
+ 'Broad_exome_VCGS_FAM203_621_D2_1', 'Broad_exome_VCGS_FAM203_621_D2',
+ 'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_COL_FAM1_1_D1.cram',
+ 'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_COL_FAM1_1_D1.crai', '129c28163df082', 'GRCh38', '', '',
+ '', 'BWA-MEM-2.3', 'DOI:10.5281/zenodo.4469317', '',
+ ]], additional_calls=1)
+ self._assert_expected_file(read_set_file, [
+ READ_SET_TABLE_HEADER,
+ ['Broad_exome_VCGS_FAM203_621_D2', 'Broad_exome_VCGS_FAM203_621_D2_1'],
+ ], additional_calls=1)
+ self._assert_expected_file(rna_file, [RNA_TABLE_HEADER, [
+ 'Broad_paired-end_NA19679', 'Broad_SM-N1P91', 'NA19679', 'Unknown', '151', 'paired-end', '2023-02-11',
+ 'NovaSeq', 'stranded poly-A pulldown', 'paired-end', 'LCSET-26942', '8.9818', '19480858', '106842386', '5.9',
+ '80.2', '1.05', '', '', '', '', '',
+ ]])
+ self._assert_expected_file(genetic_findings_file, [GENETIC_FINDINGS_TABLE[0], [
+ 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'SNV/INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '',
+ 'RP11', 'ENST00000258436.5', 'c.375_377delTCT', 'p.Leu126del', 'Heterozygous', '', 'de novo', '', '',
+ 'Candidate', 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', 'OMIM:615120',
+ 'Autosomal recessive|X-linked', 'Full', '', '', 'SR-ES', '', '', '', '', '', '', '',
+ ], [
+ 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh37', '1',
+ '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '',
+ 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '',
+ ], [
+ 'Broad_HG00731_19_1912634', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh38', '19',
+ '1912634', 'C', 'T', 'CA403171634', 'OR4G11P', 'ENST00000371839', '', '', 'Heterozygous', '', 'unknown',
+ 'Broad_HG00731_19_1912633', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', '', '', '', '',
+ '', '', '',
+ ]], additional_calls=2)
responses.calls.reset()
mock_subprocess.reset_mock()
+ mock_open.reset_mock()
responses.add(responses.GET, MOCK_DATA_MODEL_URL, body=MOCK_DATA_MODEL_RESPONSE, status=200)
response = self.client.post(url, content_type='application/json', data=json.dumps(body))
self.assertEqual(response.status_code, 200)
- expected_response = {
- 'info': ['Successfully validated and uploaded Gregor Report for 9 families'],
- 'warnings': recommended_warnings,
- }
+ expected_response['warnings'] = recommended_warnings
self.assertDictEqual(response.json(), expected_response)
- self._assert_expected_gregor_files(mock_open)
+ self._assert_expected_gregor_files(mock_open, mock_subprocess)
self._test_expected_gregor_airtable_calls()
- # test gsutil commands
- mock_subprocess.assert_has_calls([
- mock.call('gsutil ls gs://anvil-upload', stdout=-1, stderr=-2, shell=True), # nosec
- mock.call().wait(),
- mock.call('gsutil mv /mock/tmp/* gs://anvil-upload', stdout=-1, stderr=-2, shell=True), # nosec
- mock.call().wait(),
- ])
-
# Test multiple project with shared sample IDs
project = Project.objects.get(id=3)
project.consent_code = 'H'
@@ -887,6 +951,7 @@ def test_gregor_export(self, mock_subprocess, mock_temp_dir, mock_open, mock_dat
},
})
mock_open.reset_mock()
+ mock_subprocess.reset_mock()
response = self.client.post(url, content_type='application/json', data=json.dumps(body))
self.assertEqual(response.status_code, 200)
expected_response['info'][0] = expected_response['info'][0].replace('9', '10')
@@ -895,169 +960,160 @@ def test_gregor_export(self, mock_subprocess, mock_temp_dir, mock_open, mock_dat
expected_response['warnings'][2] = expected_response['warnings'][2].replace('Broad_NA20888', 'Broad_NA20885, Broad_NA20888, Broad_NA20889')
expected_response['warnings'][3] = expected_response['warnings'][3].replace('Broad_NA20888', 'Broad_NA20885, Broad_NA20888, Broad_NA20889')
self.assertDictEqual(response.json(), expected_response)
- self._assert_expected_gregor_files(mock_open, has_second_project=True)
+ self._assert_expected_gregor_files(mock_open, mock_subprocess, has_second_project=True)
self._test_expected_gregor_airtable_calls(additional_samples=['NA20885', 'NA20889'], additional_mondo_ids=['0008788'])
self.check_no_analyst_no_access(url)
- def _assert_expected_gregor_files(self, mock_open, has_second_project=False):
+ def _get_expected_gregor_files(self, mock_open, mock_subprocess, expected_files):
+ # test gsutil commands
+ mock_subprocess.assert_has_calls([
+ mock.call('gsutil ls gs://anvil-upload', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
+ mock.call('gsutil mv /mock/tmp/* gs://anvil-upload', stdout=-1, stderr=-2, shell=True), # nosec
+ mock.call().wait(),
+ ])
+
self.assertListEqual(
- mock_open.call_args_list, [mock.call(f'/mock/tmp/{file}.tsv', 'w') for file in EXPECTED_GREGOR_FILES])
- files = [
+ mock_open.call_args_list, [mock.call(f'/mock/tmp/{file}.tsv', 'w') for file in expected_files])
+ return [
[row.split('\t') for row in write_call.args[0].split('\n')]
for write_call in mock_open.return_value.__enter__.return_value.write.call_args_list
]
+
+ def _assert_expected_gregor_files(self, mock_open, mock_subprocess, has_second_project=False):
+ files = self._get_expected_gregor_files(mock_open, mock_subprocess, EXPECTED_GREGOR_FILES)
participant_file, family_file, phenotype_file, analyte_file, experiment_file, read_file, read_set_file, \
called_file, experiment_rna_file, aligned_rna_file, experiment_lookup_file, genetic_findings_file = files
- self.assertEqual(len(participant_file), 16 if has_second_project else 14)
- self.assertEqual(participant_file[0], PARTICIPANT_TABLE[0])
- row = next(r for r in participant_file if r[0] == 'Broad_NA19675_1')
- self.assertListEqual(PARTICIPANT_TABLE[1], row)
- hispanic_row = next(r for r in participant_file if r[0] == 'Broad_HG00731')
- self.assertListEqual(PARTICIPANT_TABLE[2], hispanic_row)
- solved_row = next(r for r in participant_file if r[0] == 'Broad_NA20876')
- self.assertIn(PARTICIPANT_TABLE[3], participant_file)
- self.assertListEqual(PARTICIPANT_TABLE[4], solved_row)
- multi_data_type_row = next(r for r in participant_file if r[0] == 'Broad_NA20888')
- expected_row = PARTICIPANT_TABLE[5]
- if not has_second_project:
- expected_row = expected_row[:1] + ['Broad_1kg project nme with unide'] + expected_row[2:7] + [
- 'Broad_8'] + expected_row[8:13] + ['Female', '', '', '', ''] + expected_row[18:]
- self.assertListEqual(expected_row, multi_data_type_row)
- self.assertEqual(PARTICIPANT_TABLE[5] in participant_file, has_second_project)
-
- self.assertEqual(len(family_file), 11 if has_second_project else 10)
- self.assertEqual(family_file[0], [
- 'family_id', 'consanguinity', 'consanguinity_detail',
- ])
- self.assertIn(['Broad_1', 'Present', ''], family_file)
+ single_project_row = PARTICIPANT_TABLE[5][:1] + ['Broad_1kg project nme with unide'] + PARTICIPANT_TABLE[5][2:7] + [
+ 'Broad_8'] + PARTICIPANT_TABLE[5][8:13] + ['Female', '', '', '', ''] + PARTICIPANT_TABLE[5][18:]
+ self._assert_expected_file(
+ participant_file,
+ expected_rows=PARTICIPANT_TABLE if has_second_project else PARTICIPANT_TABLE[:5] + [single_project_row],
+ absent_rows=[single_project_row] if has_second_project else PARTICIPANT_TABLE[5:],
+ additional_calls=9 if has_second_project else 8,
+ )
+
+ expected_rows = [
+ ['family_id', 'consanguinity', 'consanguinity_detail'],
+ ['Broad_1', 'Present', ''],
+ ]
+ absent_rows = []
fam_8_row = ['Broad_8', 'Unknown', '']
fam_11_row = ['Broad_11', 'None suspected', '']
if has_second_project:
- self.assertIn(fam_11_row, family_file)
- self.assertNotIn(fam_8_row, family_file)
+ expected_rows.append(fam_11_row)
+ absent_rows.append(fam_8_row)
else:
- self.assertIn(fam_8_row, family_file)
- self.assertNotIn(fam_11_row, family_file)
-
- self.assertEqual(len(phenotype_file), 14 if has_second_project else 10)
- self.assertEqual(phenotype_file[0], PHENOTYPE_TABLE[0])
- for row in PHENOTYPE_TABLE[1:5]:
- self.assertIn(row, phenotype_file)
- for row in PHENOTYPE_TABLE[5:]:
- self.assertEqual(row in phenotype_file, has_second_project)
-
- self.assertEqual(len(analyte_file), 6 if has_second_project else 5)
- self.assertEqual(analyte_file[0], [
- 'analyte_id', 'participant_id', 'analyte_type', 'analyte_processing_details', 'primary_biosample',
- 'primary_biosample_id', 'primary_biosample_details', 'tissue_affected_status',
- ])
- row = next(r for r in analyte_file if r[1] == 'Broad_NA19675_1')
- self.assertListEqual(
+ expected_rows.append(fam_8_row)
+ absent_rows.append(fam_11_row)
+ self._assert_expected_file(
+ family_file, expected_rows, absent_rows=absent_rows, additional_calls=8 if has_second_project else 7,
+ )
+
+ self._assert_expected_file(
+ phenotype_file,
+ expected_rows=PHENOTYPE_TABLE if has_second_project else PHENOTYPE_TABLE[:5],
+ absent_rows=None if has_second_project else PHENOTYPE_TABLE[5:],
+ additional_calls=7 if has_second_project else 5,
+ )
+
+ expected_rows = [
+ [
+ 'analyte_id', 'participant_id', 'analyte_type', 'analyte_processing_details', 'primary_biosample',
+ 'primary_biosample_id', 'primary_biosample_details', 'tissue_affected_status',
+ ],
['Broad_SM-AGHT', 'Broad_NA19675_1', 'DNA', '', 'UBERON:0003714', '', '', 'No'],
- row)
- self.assertIn(
- ['Broad_SM-N1P91', 'Broad_NA19679', 'RNA', '', 'CL: 0000057', '', '', 'Yes'], analyte_file)
- self.assertIn(
- ['Broad_SM-L5QMP', 'Broad_NA20888', '', '', '', '', '', 'No'], analyte_file)
- self.assertEqual(
- ['Broad_SM-L5QMWP', 'Broad_NA20888', '', '', '', '', '', 'No'] in analyte_file,
- has_second_project
+ ['Broad_SM-N1P91', 'Broad_NA19679', 'RNA', '', 'CL: 0000057', '', '', 'Yes'],
+ ['Broad_SM-L5QMP', 'Broad_NA20888', '', '', '', '', '', 'No'],
+ ]
+ absent_rows = []
+ (expected_rows if has_second_project else absent_rows).append(
+ ['Broad_SM-L5QMWP', 'Broad_NA20888', '', '', '', '', '', 'No']
)
+ self._assert_expected_file(analyte_file, expected_rows, absent_rows=absent_rows, additional_calls=1)
- num_airtable_rows = 4 if has_second_project else 3
- self.assertEqual(len(experiment_file), num_airtable_rows)
- self.assertEqual(experiment_file[0], EXPERIMENT_TABLE[0])
- self.assertIn(EXPERIMENT_TABLE[1], experiment_file)
- self.assertIn(EXPERIMENT_TABLE[2], experiment_file)
- self.assertEqual(EXPERIMENT_TABLE[3] in experiment_file, has_second_project)
-
- self.assertEqual(len(read_file), num_airtable_rows)
- self.assertEqual(read_file[0], [
- 'aligned_dna_short_read_id', 'experiment_dna_short_read_id', 'aligned_dna_short_read_file',
- 'aligned_dna_short_read_index_file', 'md5sum', 'reference_assembly', 'reference_assembly_uri', 'reference_assembly_details',
- 'mean_coverage', 'alignment_software', 'analysis_details', 'quality_issues',
- ])
- self.assertIn([
- 'Broad_exome_VCGS_FAM203_621_D2_1', 'Broad_exome_VCGS_FAM203_621_D2',
- 'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_COL_FAM1_1_D1.cram',
- 'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_COL_FAM1_1_D1.crai',
- '129c28163df082', 'GRCh38', '', '', '', 'BWA-MEM-2.3', 'DOI:10.5281/zenodo.4469317', '',
- ], read_file)
- self.assertIn([
+ self._assert_expected_file(
+ experiment_file,
+ expected_rows=EXPERIMENT_TABLE if has_second_project else EXPERIMENT_TABLE[:3],
+ absent_rows=None if has_second_project else EXPERIMENT_TABLE[3:],
+ )
+
+ expected_rows = [READ_TABLE_HEADER, [
'Broad_exome_NA20888_1', 'Broad_exome_NA20888',
'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_NA20888.cram',
'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_NA20888.crai', 'a6f6308866765ce8', 'GRCh38', '', '',
'42.8', 'BWA-MEM-2.3', '', '',
- ], read_file)
- self.assertEqual([
+ ]]
+ absent_rows = []
+ (expected_rows if has_second_project else absent_rows).append([
'Broad_genome_NA20888_1_1', 'Broad_genome_NA20888_1',
'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_NA20888_1.cram',
'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/Broad_NA20888_1.crai', '2aa33e8c32020b1c', 'GRCh38', '', '',
'36.1', 'BWA-MEM-2.3', '', '',
- ] in read_file, has_second_project)
+ ])
+ self._assert_expected_file(read_file, expected_rows, absent_rows=absent_rows, additional_calls=1)
- self.assertEqual(len(read_set_file), num_airtable_rows)
- self.assertEqual(read_set_file[0], ['aligned_dna_short_read_set_id', 'aligned_dna_short_read_id'])
- self.assertIn(['Broad_exome_VCGS_FAM203_621_D2', 'Broad_exome_VCGS_FAM203_621_D2_1'], read_set_file)
- self.assertIn(['Broad_exome_NA20888', 'Broad_exome_NA20888_1'], read_set_file)
- self.assertEqual(['Broad_genome_NA20888_1', 'Broad_genome_NA20888_1_1'] in read_set_file, has_second_project)
+ expected_rows = [
+ READ_SET_TABLE_HEADER,
+ ['Broad_exome_VCGS_FAM203_621_D2', 'Broad_exome_VCGS_FAM203_621_D2_1'],
+ ['Broad_exome_NA20888', 'Broad_exome_NA20888_1'],
+ ]
+ absent_rows = []
+ (expected_rows if has_second_project else absent_rows).append(
+ ['Broad_genome_NA20888_1', 'Broad_genome_NA20888_1_1']
+ )
+ self._assert_expected_file(read_set_file, expected_rows, absent_rows=absent_rows)
- self.assertEqual(len(called_file), 2)
- self.assertEqual(called_file[0], [
+ self._assert_expected_file(called_file, [[
'called_variants_dna_short_read_id', 'aligned_dna_short_read_set_id', 'called_variants_dna_file', 'md5sum',
'caller_software', 'variant_types', 'analysis_details',
- ])
- self.assertIn([
+ ], [
'SX2-3', 'Broad_exome_VCGS_FAM203_621_D2', 'gs://fc-fed09429-e563-44a7-aaeb-776c8336ba02/COL_FAM1_1_D1.SV.vcf',
'129c28163df082', 'gatk4.1.2', 'SNV', 'DOI:10.5281/zenodo.4469317',
- ], called_file)
-
- self.assertEqual(len(experiment_rna_file), 2)
- self.assertEqual(experiment_rna_file[0], [
- 'experiment_rna_short_read_id', 'analyte_id', 'experiment_sample_id', 'seq_library_prep_kit_method',
- 'read_length', 'experiment_type', 'date_data_generation', 'sequencing_platform', 'library_prep_type',
- 'single_or_paired_ends', 'within_site_batch_name', 'RIN', 'estimated_library_size', 'total_reads',
- 'percent_rRNA', 'percent_mRNA', '5prime3prime_bias', 'percent_mtRNA', 'percent_Globin', 'percent_UMI',
- 'percent_GC', 'percent_chrX_Y',
- ])
- self.assertEqual(experiment_rna_file[1], [
+ ]])
+
+ self._assert_expected_file(experiment_rna_file, [RNA_TABLE_HEADER, [
'Broad_paired-end_NA19679', 'Broad_SM-N1P91', 'NA19679', 'Unknown', '151', 'paired-end', '2023-02-11',
'NovaSeq', 'stranded poly-A pulldown', 'paired-end', 'LCSET-26942', '8.9818', '19480858', '106842386',
'5.9', '80.2', '1.05', '', '', '', '', '',
- ])
+ ]])
- self.assertEqual(len(aligned_rna_file), 2)
- self.assertEqual(aligned_rna_file[0], [
+ self._assert_expected_file(aligned_rna_file, [[
'aligned_rna_short_read_id', 'experiment_rna_short_read_id', 'aligned_rna_short_read_file',
'aligned_rna_short_read_index_file', 'md5sum', 'reference_assembly', 'reference_assembly_uri',
'reference_assembly_details', 'mean_coverage', 'gene_annotation', 'gene_annotation_details',
'alignment_software', 'alignment_log_file', 'alignment_postprocessing', 'percent_uniquely_aligned',
'percent_multimapped', 'percent_unaligned', 'quality_issues'
- ])
- self.assertEqual(aligned_rna_file[1], [
+ ], [
'Broad_paired-end_NA19679_1', 'Broad_paired-end_NA19679', 'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/NA19679.Aligned.out.cram',
'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/NA19679.Aligned.out.crai', 'f6490b8ebdf2', 'GRCh38',
'gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta', '', '', 'GENCODEv26', '',
'STARv2.7.10b', 'gs://fc-eb352699-d849-483f-aefe-9d35ce2b21ac/NA19679.Log.final.out', '', '80.53', '17.08',
'1.71', ''
- ])
+ ]])
- self.assertEqual(len(experiment_lookup_file), num_airtable_rows + 1)
- self.assertEqual(experiment_lookup_file[0], EXPERIMENT_LOOKUP_TABLE[0])
- self.assertIn(EXPERIMENT_LOOKUP_TABLE[1], experiment_lookup_file)
- self.assertIn(EXPERIMENT_LOOKUP_TABLE[2], experiment_lookup_file)
- self.assertIn(EXPERIMENT_LOOKUP_TABLE[3], experiment_lookup_file)
- self.assertEqual(EXPERIMENT_LOOKUP_TABLE[4] in experiment_lookup_file, has_second_project)
-
- self.assertEqual(len(genetic_findings_file), 8 if has_second_project else 6)
- self.assertEqual(genetic_findings_file[0], GENETIC_FINDINGS_TABLE[0])
- self.assertIn(GENETIC_FINDINGS_TABLE[1], genetic_findings_file)
- self.assertIn(GENETIC_FINDINGS_TABLE[2], genetic_findings_file)
- if has_second_project:
- self.assertIn(GENETIC_FINDINGS_TABLE[3], genetic_findings_file)
- self.assertIn(GENETIC_FINDINGS_TABLE[4], genetic_findings_file)
+ self._assert_expected_file(
+ experiment_lookup_file,
+ expected_rows=EXPERIMENT_LOOKUP_TABLE if has_second_project else EXPERIMENT_LOOKUP_TABLE[:4],
+ absent_rows=None if has_second_project else EXPERIMENT_LOOKUP_TABLE[4:],
+ )
+
+ self._assert_expected_file(
+ genetic_findings_file,
+ expected_rows=GENETIC_FINDINGS_TABLE if has_second_project else GENETIC_FINDINGS_TABLE[:3],
+ absent_rows=None if has_second_project else EXPERIMENT_LOOKUP_TABLE[3:],
+ additional_calls=3,
+ )
+
+ def _assert_expected_file(self, actual_rows, expected_rows, additional_calls=0, absent_rows=None):
+ self.assertEqual(len(actual_rows), len(expected_rows) + additional_calls)
+ self.assertEqual(expected_rows[0], actual_rows[0])
+ for row in expected_rows[1:]:
+ self.assertIn(row, actual_rows)
+ for row in absent_rows or []:
+ self.assertNotIn(row, actual_rows)
def _test_expected_gregor_airtable_calls(self, additional_samples=None, additional_mondo_ids=None):
mondo_ids = ['0044970'] + (additional_mondo_ids or [])
@@ -1205,7 +1261,7 @@ def test_variant_metadata(self):
'ClinGen_allele_ID': 'CA1501729',
'clinvar': {'alleleId': None, 'clinicalSignificance': '', 'goldStars': None, 'variationId': None},
'condition_id': 'MONDO:0044970',
- 'condition_inheritance': None,
+ 'condition_inheritance': 'Unknown',
'displayName': '2',
'familyGuid': 'F000002_2',
'family_id': '2',
@@ -1231,7 +1287,7 @@ def test_variant_metadata(self):
'chrom': '19',
'ClinGen_allele_ID': 'CA403171634',
'condition_id': 'MONDO:0044970',
- 'condition_inheritance': None,
+ 'condition_inheritance': 'Unknown',
'displayName': '2',
'end': 1912634,
'familyGuid': 'F000002_2',
diff --git a/seqr/views/apis/users_api.py b/seqr/views/apis/users_api.py
index 5e341ca7dc..69f55a5b4f 100644
--- a/seqr/views/apis/users_api.py
+++ b/seqr/views/apis/users_api.py
@@ -49,9 +49,12 @@ def get_all_user_group_options(request):
@login_and_policies_required
def get_project_collaborator_options(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user)
+ user_fields = {'display_name', 'username', 'email'}
users = get_project_collaborators_by_username(
- request.user, project, fields={'display_name', 'username', 'email'}, expand_user_groups=True,
+ request.user, project, fields=user_fields, expand_user_groups=True,
)
+ if not users:
+ users = {request.user.username: get_json_for_user(request.user, user_fields)}
return create_json_response(users)
diff --git a/seqr/views/apis/users_api_tests.py b/seqr/views/apis/users_api_tests.py
index 5579edee11..8a563f50ea 100644
--- a/seqr/views/apis/users_api_tests.py
+++ b/seqr/views/apis/users_api_tests.py
@@ -60,6 +60,7 @@ def test_get_project_collaborator_options(self):
users.update(self.COLLABORATOR_JSON)
users.pop('analysts@firecloud.org', None)
self.assertDictEqual(response_json, users)
+ return url
def test_get_all_collaborator_options(self):
url = reverse(get_all_collaborator_options)
@@ -425,7 +426,7 @@ def _assert_403_response(self, response, **kwargs):
_test_delete_collaborator_group_response = _assert_403_response
def test_get_project_collaborator_options(self, *args, **kwargs):
- super(AnvilUsersAPITest, self).test_get_project_collaborator_options(*args, **kwargs)
+ url = super(AnvilUsersAPITest, self).test_get_project_collaborator_options(*args, **kwargs)
self.mock_list_workspaces.assert_not_called()
self.assertEqual(self.mock_get_ws_acl.call_count, 1)
self.mock_get_ws_acl.assert_called_with(
@@ -436,6 +437,10 @@ def test_get_project_collaborator_options(self, *args, **kwargs):
self.mock_get_groups.assert_not_called()
self.mock_get_group_members.assert_called_with(self.collaborator_user, 'Analysts', use_sa_credentials=True)
+ self.mock_get_ws_acl.side_effect = lambda *args: {}
+ response = self.client.get(url)
+ self.assertEqual(response.status_code, 200)
+ self.assertDictEqual(response.json(), {'test_user_collaborator': MAIN_COLLABORATOR_JSON['test_user_collaborator']})
def test_set_password(self):
super(AnvilUsersAPITest, self).test_set_password()
diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py
index fefa27e1d7..4c1f58cad7 100644
--- a/seqr/views/utils/anvil_metadata_utils.py
+++ b/seqr/views/utils/anvil_metadata_utils.py
@@ -361,7 +361,7 @@ def _get_parsed_saved_discovery_variants_by_family(
variant_fields += ['svType', 'svName', 'end']
parsed_variant = {
- 'chrom': chrom,
+ 'chrom': 'MT' if chrom == 'M' else chrom,
'pos': pos,
'variant_reference_assembly': GENOME_VERSION_LOOKUP[variant_json['genomeVersion']],
'gene_id': gene_id,
@@ -624,7 +624,7 @@ def _get_mondo_condition_data(mondo_id):
inheritance = HumanPhenotypeOntology.objects.get(hpo_id=inheritance['id']).name.replace(' inheritance', '')
return {
'known_condition_name': data['name'],
- 'condition_inheritance': inheritance,
+ 'condition_inheritance': inheritance or 'Unknown',
}
except Exception:
return {}
diff --git a/ui/pages/Report/components/Gregor.jsx b/ui/pages/Report/components/Gregor.jsx
index ca1e08420f..0421634949 100644
--- a/ui/pages/Report/components/Gregor.jsx
+++ b/ui/pages/Report/components/Gregor.jsx
@@ -2,13 +2,20 @@ import React from 'react'
import { Header } from 'semantic-ui-react'
import { validators } from 'shared/components/form/FormHelpers'
-import { ButtonRadioGroup } from 'shared/components/form/Inputs'
+import { ButtonRadioGroup, InlineToggle } from 'shared/components/form/Inputs'
import UploadFormPage from 'shared/components/page/UploadFormPage'
import { CONSENT_CODES } from 'shared/utils/constants'
import { HttpRequestHelper } from 'shared/utils/httpRequestHelper'
const FIELDS = [
-
+ {
+ name: 'overrideValidation',
+ label: 'Upload with Validation Errors',
+ component: InlineToggle,
+ asFormInput: true,
+ fullHeight: true,
+ inline: false,
+ },
{
name: 'deliveryPath',
label: 'AnVIL Delivery Bucket Path',
diff --git a/ui/shared/components/form/Inputs.jsx b/ui/shared/components/form/Inputs.jsx
index 492a94a066..867929c1fa 100644
--- a/ui/shared/components/form/Inputs.jsx
+++ b/ui/shared/components/form/Inputs.jsx
@@ -486,7 +486,7 @@ BooleanCheckbox.propTypes = {
export const AlignedBooleanCheckbox = AlignedCheckboxGroup.withComponent(BooleanCheckbox)
-const BaseInlineToggle = styled(({ divided, fullHeight, asFormInput, padded, ...props }) => )`
+const BaseInlineToggle = styled(({ divided, fullHeight, asFormInput, padded, inline = true, ...props }) => )`
${props => (props.asFormInput ?
`label {
font-weight: 700;