From 0ac1039105b1938f788958e4c2372d17cb6b1a88 Mon Sep 17 00:00:00 2001 From: Charles Haynes <33608920+haynescd@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:22:59 -0400 Subject: [PATCH] Add generic-assay-data-counts endpoint (#11059) * Add generic-assay-data-counts endpoint * Update GenericAssayFilter to support NA's * Add tests for generic assay data counts endpoint (categorical) --------- Co-authored-by: alisman --- .../persistence/StudyViewRepository.java | 4 + .../mybatisclickhouse/StudyViewMapper.java | 4 + .../StudyViewMyBatisRepository.java | 10 ++- .../service/StudyViewColumnarService.java | 4 + .../impl/StudyViewColumnarServiceImpl.java | 9 ++- .../StudyViewColumnStoreController.java | 29 +++++++ .../StudyViewFilterMapper.xml | 54 ++++++++----- .../mybatisclickhouse/StudyViewMapper.xml | 49 ++++++++++++ .../specs/generic-assay-data-counts.json | 75 +++++++++++++++++++ 9 files changed, 215 insertions(+), 23 deletions(-) create mode 100644 test/api-e2e/specs/generic-assay-data-counts.json diff --git a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java index e73d1225916..6dab21a2081 100644 --- a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java +++ b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java @@ -7,6 +7,7 @@ import org.cbioportal.model.ClinicalDataCount; import org.cbioportal.model.ClinicalEventTypeCount; import org.cbioportal.model.CopyNumberCountByGene; +import org.cbioportal.model.GenericAssayDataCountItem; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.MolecularProfile; @@ -16,6 +17,7 @@ import org.cbioportal.model.StudyViewFilterContext; import org.cbioportal.web.parameter.ClinicalDataType; import org.cbioportal.web.parameter.GenericAssayDataBinFilter; +import org.cbioportal.web.parameter.GenericAssayDataFilter; import org.cbioportal.web.parameter.GenomicDataBinFilter; import org.cbioportal.web.parameter.GenomicDataFilter; @@ -68,6 +70,8 @@ public interface StudyViewRepository { int getTotalSampleTreatmentCount(StudyViewFilterContext studyViewFilterContext); List getCNACounts(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters); + + List getGenericAssayDataCounts(StudyViewFilterContext studyViewFilterContext, List genericAssayDataFilters); Map getMutationCounts(StudyViewFilterContext studyViewFilterContext, GenomicDataFilter genomicDataFilter); diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java index 304612d6c81..00333c43a3c 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java @@ -9,6 +9,7 @@ import org.cbioportal.model.ClinicalEventTypeCount; import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.GenePanelToGene; +import org.cbioportal.model.GenericAssayDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.PatientTreatment; @@ -18,6 +19,7 @@ import org.cbioportal.persistence.helper.AlterationFilterHelper; import org.cbioportal.persistence.helper.StudyViewFilterHelper; import org.cbioportal.web.parameter.GenericAssayDataBinFilter; +import org.cbioportal.web.parameter.GenericAssayDataFilter; import org.cbioportal.web.parameter.GenomicDataBinFilter; import org.cbioportal.web.parameter.GenomicDataFilter; @@ -66,6 +68,8 @@ public interface StudyViewMapper { int getTotalSampleTreatmentCounts(@Param("studyViewFilterHelper") StudyViewFilterHelper studyViewFilterHelper); List getCNACounts(StudyViewFilterHelper studyViewFilterHelper, List genomicDataFilters); + + List getGenericAssayDataCounts(StudyViewFilterHelper studyViewFilterHelper, List genericAssayDataFilters); Map getMutationCounts(StudyViewFilterHelper studyViewFilterHelper, GenomicDataFilter genomicDataFilter); diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java index 4f0e5cde5d0..5419d12d1a9 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java @@ -6,6 +6,7 @@ import org.cbioportal.model.ClinicalDataCount; import org.cbioportal.model.ClinicalEventTypeCount; import org.cbioportal.model.GenePanelToGene; +import org.cbioportal.model.GenericAssayDataCountItem; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.CopyNumberCountByGene; @@ -20,6 +21,7 @@ import org.cbioportal.persistence.helper.StudyViewFilterHelper; import org.cbioportal.web.parameter.ClinicalDataType; import org.cbioportal.web.parameter.GenericAssayDataBinFilter; +import org.cbioportal.web.parameter.GenericAssayDataFilter; import org.cbioportal.web.parameter.GenomicDataBinFilter; import org.cbioportal.web.parameter.GenomicDataFilter; import org.springframework.beans.factory.annotation.Autowired; @@ -245,17 +247,19 @@ private Map> getGenericAssayProfilesMap() { @Override public List getCNACounts(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters) { - return mapper.getCNACounts(createStudyViewFilterHelper(studyViewFilterContext), genomicDataFilters); } - public Map getMutationCounts(StudyViewFilterContext studyViewFilterContext, GenomicDataFilter genomicDataFilter) { + @Override + public List getGenericAssayDataCounts(StudyViewFilterContext studyViewFilterContext, List genericAssayDataFilters) { + return mapper.getGenericAssayDataCounts(createStudyViewFilterHelper(studyViewFilterContext), genericAssayDataFilters); + } + public Map getMutationCounts(StudyViewFilterContext studyViewFilterContext, GenomicDataFilter genomicDataFilter) { return mapper.getMutationCounts(createStudyViewFilterHelper(studyViewFilterContext), genomicDataFilter); } public List getMutationCountsByType(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters) { - return mapper.getMutationCountsByType(createStudyViewFilterHelper(studyViewFilterContext), genomicDataFilters); } diff --git a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java index 9e59c326945..d4dafd536ca 100644 --- a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java +++ b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java @@ -5,6 +5,7 @@ import org.cbioportal.model.ClinicalData; import org.cbioportal.model.ClinicalDataCountItem; import org.cbioportal.model.ClinicalEventTypeCount; +import org.cbioportal.model.GenericAssayDataCountItem; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.CopyNumberCountByGene; @@ -14,6 +15,7 @@ import org.cbioportal.service.exception.StudyNotFoundException; import org.cbioportal.web.parameter.ClinicalDataType; import org.cbioportal.web.parameter.GenericAssayDataBinFilter; +import org.cbioportal.web.parameter.GenericAssayDataFilter; import org.cbioportal.web.parameter.GenomicDataBinFilter; import org.cbioportal.web.parameter.GenomicDataFilter; import org.cbioportal.web.parameter.StudyViewFilter; @@ -46,6 +48,8 @@ public interface StudyViewColumnarService { SampleTreatmentReport getSampleTreatmentReport(StudyViewFilter studyViewFilter); List getCNACountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); + + List getGenericAssayDataCounts(StudyViewFilter studyViewFilter, List genericAssayDataFilters); List getMutationCountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); diff --git a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java index fc9af001d41..ea715b98652 100644 --- a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java +++ b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java @@ -7,6 +7,7 @@ import org.cbioportal.model.ClinicalDataCountItem; import org.cbioportal.model.ClinicalEventTypeCount; import org.cbioportal.model.CopyNumberCountByGene; +import org.cbioportal.model.GenericAssayDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.PatientTreatmentReport; import org.cbioportal.model.GenomicDataCountItem; @@ -21,6 +22,7 @@ import org.cbioportal.web.parameter.ClinicalDataType; import org.cbioportal.web.parameter.CustomSampleIdentifier; import org.cbioportal.web.parameter.GenericAssayDataBinFilter; +import org.cbioportal.web.parameter.GenericAssayDataFilter; import org.cbioportal.web.parameter.GenomicDataBinFilter; import org.cbioportal.web.parameter.GenomicDataFilter; import org.cbioportal.web.parameter.StudyViewFilter; @@ -143,7 +145,12 @@ public List getSampleClinicalData(StudyViewFilter studyViewFilter, public List getCNACountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters) { return studyViewRepository.getCNACounts(createContext(studyViewFilter), genomicDataFilters); } - + + @Override + public List getGenericAssayDataCounts(StudyViewFilter studyViewFilter, List genericAssayDataFilters) { + return studyViewRepository.getGenericAssayDataCounts(createContext(studyViewFilter), genericAssayDataFilters); + } + @Override public List getMutationCountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters) { List genomicDataCountItemList = new ArrayList<>(); diff --git a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java index 0d0df0de0ef..e5ebbfecda0 100644 --- a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java +++ b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java @@ -20,6 +20,7 @@ import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.DensityPlotData; import org.cbioportal.model.GenericAssayDataBin; +import org.cbioportal.model.GenericAssayDataCountItem; import org.cbioportal.model.GenomicDataBin; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.PatientTreatmentReport; @@ -40,6 +41,8 @@ import org.cbioportal.web.parameter.ClinicalDataFilter; import org.cbioportal.web.parameter.DataBinMethod; import org.cbioportal.web.parameter.GenericAssayDataBinCountFilter; +import org.cbioportal.web.parameter.GenericAssayDataCountFilter; +import org.cbioportal.web.parameter.GenericAssayDataFilter; import org.cbioportal.web.parameter.GenomicDataBinCountFilter; import org.cbioportal.web.parameter.GenomicDataCountFilter; import org.cbioportal.web.parameter.GenomicDataFilter; @@ -399,6 +402,32 @@ public ResponseEntity> fetchGenomicDataCounts( return new ResponseEntity<>(result, HttpStatus.OK); } + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") + @PostMapping(value = "/column-store/generic-assay-data-counts/fetch", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(description = "Fetch generic assay data counts by study view filter") + @ApiResponse(responseCode = "200", description = "OK", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = GenericAssayDataCountItem.class)))) + public ResponseEntity> fetchGenericAssayDataCounts( + @Parameter(required = true, description = "Generic assay data count filter") @Valid @RequestBody(required = false) GenericAssayDataCountFilter genericAssayDataCountFilter, + @Parameter(hidden = true) // prevent reference to this attribute in the swagger-ui interface + @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, + @Parameter(hidden = true) // prevent reference to this attribute in the swagger-ui interface. this + // attribute is needed for the @PreAuthorize tag above. + @Valid @RequestAttribute(required = false, value = "interceptedGenericAssayDataCountFilter") GenericAssayDataCountFilter interceptedGenericAssayDataCountFilter) { + + List gaFilters = interceptedGenericAssayDataCountFilter.getGenericAssayDataFilters(); + StudyViewFilter studyViewFilter = interceptedGenericAssayDataCountFilter.getStudyViewFilter(); + // when there is only one filter, it means study view is doing a single chart filter operation + // remove filter from studyViewFilter to return all data counts + // the reason we do this is to make sure after chart get filtered, user can still see unselected portion of the chart + + if (gaFilters.size() == 1) { + studyViewFilterUtil.removeSelfFromGenericAssayFilter(gaFilters.getFirst().getStableId(), studyViewFilter); + } + + return new ResponseEntity<>(studyViewColumnarService.getGenericAssayDataCounts(studyViewFilter, gaFilters), HttpStatus.OK); + } + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") @PostMapping(value = "/column-store/mutation-data-counts/fetch", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(description = "Fetch mutation data counts by GenomicDataCountFilter") diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml index 1ff28729b63..4e4fb92d61b 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml @@ -442,6 +442,10 @@ empty(attribute_value) + OR + + + = 'NA' ( @@ -541,13 +545,11 @@ - - SELECT sample_unique_id, value + + SELECT sample_unique_id, value, datatype FROM generic_assay_data_derived WHERE profile_type = #{genericAssayDataFilter.profileType} AND entity_stable_id = #{genericAssayDataFilter.stableId} - - AND datatype = 'LIMIT-VALUE' @@ -569,8 +571,9 @@ SELECT DISTINCT sd.sample_unique_id FROM sample_derived sd - LEFT JOIN () AS generic_numerical_query ON sd.sample_unique_id = generic_numerical_query.sample_unique_id - WHERE value IS null OR + LEFT JOIN () AS generic_numerical_query ON sd.sample_unique_id = generic_numerical_query.sample_unique_id + WHERE datatype = 'LIMIT-VALUE' + AND value IS null OR = 'NA' @@ -582,8 +585,10 @@ SELECT DISTINCT sample_unique_id - FROM () AS generic_numerical_query + FROM () AS generic_numerical_query WHERE + datatype = 'LIMIT-VALUE' + AND != 'NA' @@ -639,18 +644,29 @@ SELECT ${unique_id} - FROM ${table_name} - WHERE entity_stable_id = '${genericAssayDataFilter.stableId}' AND - profile_type='${genericAssayDataFilter.profileType}' - - - AND ( - - - - ) ILIKE '${dataFilterValue.value}' - - + FROM sample_derived sd + LEFT JOIN () AS generic_assay_query + ON sd.sample_unique_id = generic_assay_query.sample_unique_id + + datatype != 'LIMIT-VALUE' + + + + value IS null OR + + + = 'NA' + + + ( + + + + ) ILIKE '${dataFilterValue.value}' + + + + diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml index dc94c3fc3aa..e7b082753db 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml @@ -282,6 +282,47 @@ 'NA' as value, cast(((SELECT * FROM ()) - coalesce((SELECT cna_count FROM cna_sum LIMIT 1), 0)) as INTEGER) as count + +