From a484e2ee5e6d719f5433c3016aa2c4a06d4d2e5e Mon Sep 17 00:00:00 2001 From: Onur Sumer Date: Mon, 24 Jun 2024 15:33:59 -0400 Subject: [PATCH] Move ClinicalCategoricalFilter to repository (#10847) * use clinical_data_derived instead of sample_clinical_attribute_numeric_mv and patient_clinical_attribute_numeric_mv * use clinical_attribute_meta instead of sample_clinical_attribute_numeric_mv and patient_clinical_attribute_numeric_mv * remove unused clinical data count methods and SQL * fix numericalClinicalDataCountFilter * Move CategoricalClinicalAttributeFilter to repository * remove unused columns * Add override to methods --------- Co-authored-by: haynescd --- .../persistence/StudyViewRepository.java | 37 +++--- .../mybatisclickhouse/StudyViewMapper.java | 3 +- .../StudyViewMyBatisRepository.java | 119 +++++++++++++----- .../service/AlterationCountService.java | 6 +- .../impl/AlterationCountServiceImpl.java | 26 ++-- .../impl/StudyViewColumnarServiceImpl.java | 78 ++---------- .../StudyViewFilterMapper.xml | 16 +-- .../mybatisclickhouse/StudyViewMapper.xml | 50 ++------ 8 files changed, 146 insertions(+), 189 deletions(-) diff --git a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java index 51ff94c7a17..98c1a17633e 100644 --- a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java +++ b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java @@ -1,15 +1,13 @@ package org.cbioportal.persistence; import org.cbioportal.model.AlterationCountByGene; +import org.cbioportal.model.ClinicalAttribute; import org.cbioportal.model.CaseListDataCount; import org.cbioportal.model.ClinicalData; import org.cbioportal.model.ClinicalDataCount; import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.Sample; -import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; -import org.cbioportal.persistence.enums.ClinicalAttributeDataType; -import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.StudyViewFilter; import java.util.List; @@ -17,35 +15,30 @@ import java.util.Set; public interface StudyViewRepository { - List getFilteredSamples(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getFilteredSamples(StudyViewFilter studyViewFilter); - List getSampleClinicalData(StudyViewFilter studyViewFilter, List attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getSampleClinicalData(StudyViewFilter studyViewFilter, List attributeIds); - List getPatientClinicalData(StudyViewFilter studyViewFilter, List attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getPatientClinicalData(StudyViewFilter studyViewFilter, List attributeIds); - List getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getMutatedGenes(StudyViewFilter studyViewFilter); - List getCnaGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); - List getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getStructuralVariantGenes(StudyViewFilter studyViewFilter); + List getCnaGenes(StudyViewFilter studyViewFilter); - List getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List filteredAttributes); + List getClinicalDataCounts(StudyViewFilter studyViewFilter, List filteredAttributes); - List getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List filteredAttributes); + List getGenomicDataCounts(StudyViewFilter studyViewFilter); - List getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List filteredAttributes); - - List getGenomicDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); - - - List getCaseListDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getClinicalAttributes(); - List getClinicalDataAttributeNames(ClinicalAttributeDataSource clinicalAttributeDataSource, ClinicalAttributeDataType dataType); + List getCaseListDataCounts(StudyViewFilter studyViewFilter); - Map getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType); + Map getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType); - int getFilteredSamplesCount(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + int getFilteredSamplesCount(StudyViewFilter studyViewFilter); - Map> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType); + Map> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, String alterationType); - int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType); + int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, String alterationType); } diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java index 3407f40c145..71d0e76a1ab 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java @@ -3,6 +3,7 @@ import org.apache.ibatis.annotations.MapKey; import org.cbioportal.model.AlterationCountByGene; import org.cbioportal.model.CaseListDataCount; +import org.cbioportal.model.ClinicalAttribute; import org.cbioportal.model.ClinicalData; import org.cbioportal.model.ClinicalDataCount; import org.cbioportal.model.CopyNumberCountByGene; @@ -42,7 +43,7 @@ List getClinicalDataCounts(StudyViewFilter studyViewFilter, C List getCaseListDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters); - List getClinicalAttributeNames(String tableName); + List getClinicalAttributes(); List getSampleClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List attributeIds); diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java index 01bd7aa6bb3..ebe94444794 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java @@ -1,5 +1,6 @@ package org.cbioportal.persistence.mybatisclickhouse; import org.cbioportal.model.AlterationCountByGene; +import org.cbioportal.model.ClinicalAttribute; import org.cbioportal.model.CaseListDataCount; import org.cbioportal.model.ClinicalData; import org.cbioportal.model.ClinicalDataCount; @@ -9,13 +10,13 @@ import org.cbioportal.model.Sample; import org.cbioportal.persistence.StudyViewRepository; import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; -import org.cbioportal.persistence.enums.ClinicalAttributeDataType; import org.cbioportal.persistence.helper.AlterationFilterHelper; import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.StudyViewFilter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -24,6 +25,9 @@ @Repository public class StudyViewMyBatisRepository implements StudyViewRepository { + private Map> clinicalAttributesMap = new HashMap<>(); + + private static final List FILTERED_CLINICAL_ATTR_VALUES = Collections.emptyList(); private final StudyViewMapper mapper; @@ -32,94 +36,94 @@ public StudyViewMyBatisRepository(StudyViewMapper mapper) { this.mapper = mapper; } @Override - public List getFilteredSamples(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { - + public List getFilteredSamples(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getFilteredSamples(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter)); } @Override - public List getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + public List getMutatedGenes(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getMutatedGenes(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), AlterationFilterHelper.build(studyViewFilter.getAlterationFilter())); } @Override - public List getCnaGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + public List getCnaGenes(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getCnaGenes(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), AlterationFilterHelper.build(studyViewFilter.getAlterationFilter())); } @Override - public List getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + public List getStructuralVariantGenes(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getStructuralVariantGenes(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), AlterationFilterHelper.build(studyViewFilter.getAlterationFilter())); } @Override - public List getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List filteredAttributes) { + public List getClinicalDataCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES ); } @Override - public List getGenomicDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + public List getGenomicDataCounts(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getGenomicDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter)); } - - @Override - public List getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List filteredAttributes) { - return mapper.getSampleClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), - filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES ); - } - - @Override - public List getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List filteredAttributes) { - return mapper.getPatientClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), - filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES); + public List getClinicalAttributes() { + return mapper.getClinicalAttributes(); } @Override - public List getCaseListDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + public List getCaseListDataCounts(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getCaseListDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter)); } - @Override - public List getClinicalDataAttributeNames(ClinicalAttributeDataSource clinicalAttributeDataSource, ClinicalAttributeDataType dataType) { - String tableName = clinicalAttributeDataSource.getValue().toLowerCase() + "_clinical_attribute_" + dataType.getValue().toLowerCase() + "_mv"; - return mapper.getClinicalAttributeNames(tableName); - } private boolean shouldApplyPatientIdFilters(CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { return categorizedClinicalDataCountFilter.getPatientCategoricalClinicalDataFilters() != null && !categorizedClinicalDataCountFilter.getPatientCategoricalClinicalDataFilters().isEmpty() || categorizedClinicalDataCountFilter.getPatientNumericalClinicalDataFilters() != null && !categorizedClinicalDataCountFilter.getPatientNumericalClinicalDataFilters().isEmpty(); } - public List getSampleClinicalData(StudyViewFilter studyViewFilter, List attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + @Override + public List getSampleClinicalData(StudyViewFilter studyViewFilter, List attributeIds) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getSampleClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds); } - public List getPatientClinicalData(StudyViewFilter studyViewFilter, List attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + @Override + public List getPatientClinicalData(StudyViewFilter studyViewFilter, List attributeIds) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getPatientClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds); } - public Map getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType) { + @Override + public Map getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getTotalProfiledCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), alterationType); } @Override - public int getFilteredSamplesCount(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { + public int getFilteredSamplesCount(StudyViewFilter studyViewFilter) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getFilteredSamplesCount(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter)); } @Override - public Map> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType) { + public Map> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, String alterationType) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getMatchingGenePanelIds(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), alterationType) .stream() @@ -128,9 +132,62 @@ public Map> getMatchingGenePanelIds(StudyViewFilter studyVie } @Override - public int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType) { + public int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, String alterationType) { + CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); return mapper.getTotalProfiledCountByAlterationType(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), alterationType); } + private void buildClinicalAttributeNameMap() { + clinicalAttributesMap = this.getClinicalAttributes() + .stream() + .collect(Collectors.groupingBy(ca -> ca.getPatientAttribute() ? ClinicalAttributeDataSource.PATIENT : ClinicalAttributeDataSource.SAMPLE)); + } + + private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter) { + if (clinicalAttributesMap.isEmpty()) { + buildClinicalAttributeNameMap(); + } + + if (studyViewFilter.getClinicalDataFilters() == null) { + return CategorizedClinicalDataCountFilter.getBuilder().build(); + } + + List patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) + .stream().filter(ca -> ca.getDatatype().equals("STRING")) + .map(ClinicalAttribute::getAttrId) + .toList(); + + List patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) + .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) + .map(ClinicalAttribute::getAttrId) + .toList(); + + List sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(ca -> ca.getDatatype().equals("STRING")) + .map(ClinicalAttribute::getAttrId) + .toList(); + + List sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) + .map(ClinicalAttribute::getAttrId) + .toList(); + + return CategorizedClinicalDataCountFilter.getBuilder() + .setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() + .stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .build(); + } + + } \ No newline at end of file diff --git a/src/main/java/org/cbioportal/service/AlterationCountService.java b/src/main/java/org/cbioportal/service/AlterationCountService.java index 2e02a4be518..289806033ba 100644 --- a/src/main/java/org/cbioportal/service/AlterationCountService.java +++ b/src/main/java/org/cbioportal/service/AlterationCountService.java @@ -77,9 +77,9 @@ Pair, Long> getPatientCnaGeneCounts(List getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); - List getCnaGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getMutatedGenes(StudyViewFilter studyViewFilter); + List getCnaGenes(StudyViewFilter studyViewFilter); - List getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter); + List getStructuralVariantGenes(StudyViewFilter studyViewFilter); } diff --git a/src/main/java/org/cbioportal/service/impl/AlterationCountServiceImpl.java b/src/main/java/org/cbioportal/service/impl/AlterationCountServiceImpl.java index c1770e41e18..67079dc5b63 100644 --- a/src/main/java/org/cbioportal/service/impl/AlterationCountServiceImpl.java +++ b/src/main/java/org/cbioportal/service/impl/AlterationCountServiceImpl.java @@ -16,7 +16,6 @@ import org.cbioportal.persistence.StudyViewRepository; import org.cbioportal.service.AlterationCountService; import org.cbioportal.service.util.AlterationEnrichmentUtil; -import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.StudyViewFilter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.lang.NonNull; @@ -256,33 +255,30 @@ public Pair, Long> getPatientCnaGeneCounts(List getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { - var alterationCountByGenes = studyViewRepository.getMutatedGenes(studyViewFilter, categorizedClinicalDataCountFilter); - return populateAlterationCounts(alterationCountByGenes, studyViewFilter, categorizedClinicalDataCountFilter, AlterationType.MUTATION_EXTENDED); + public List getMutatedGenes(StudyViewFilter studyViewFilter) { + var alterationCountByGenes = studyViewRepository.getMutatedGenes(studyViewFilter); + return populateAlterationCounts(alterationCountByGenes, studyViewFilter, AlterationType.MUTATION_EXTENDED); } - public List getCnaGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { - var copyNumberCountByGenes = studyViewRepository.getCnaGenes(studyViewFilter, categorizedClinicalDataCountFilter); - return populateAlterationCounts(copyNumberCountByGenes, studyViewFilter, categorizedClinicalDataCountFilter, AlterationType.COPY_NUMBER_ALTERATION); + public List getCnaGenes(StudyViewFilter studyViewFilter) { + var copyNumberCountByGenes = studyViewRepository.getCnaGenes(studyViewFilter); + return populateAlterationCounts(copyNumberCountByGenes, studyViewFilter, AlterationType.COPY_NUMBER_ALTERATION); } @Override - public List getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) { - var alterationCountByGenes = studyViewRepository.getStructuralVariantGenes(studyViewFilter, categorizedClinicalDataCountFilter); - return populateAlterationCounts(alterationCountByGenes, studyViewFilter, categorizedClinicalDataCountFilter, AlterationType.STRUCTURAL_VARIANT); + public List getStructuralVariantGenes(StudyViewFilter studyViewFilter) { + var alterationCountByGenes = studyViewRepository.getStructuralVariantGenes(studyViewFilter); + return populateAlterationCounts(alterationCountByGenes, studyViewFilter, AlterationType.STRUCTURAL_VARIANT); } private < T extends AlterationCountByGene> List populateAlterationCounts(@NonNull List alterationCounts, @NonNull StudyViewFilter studyViewFilter, - @NonNull CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, @NonNull AlterationType alterationType) { var updatedAlterationCounts = alterationCounts.stream().map(SerializationUtils::clone).toList(); var profiledCountsMap = studyViewRepository.getTotalProfiledCounts(studyViewFilter, - categorizedClinicalDataCountFilter, alterationType.toString()); - var profiledCountWithoutGenePanelData = studyViewRepository.getTotalProfiledCountsByAlterationType(studyViewFilter, categorizedClinicalDataCountFilter, alterationType.toString()); - var matchingGenePanelIdsMap = studyViewRepository.getMatchingGenePanelIds(studyViewFilter, - categorizedClinicalDataCountFilter, alterationType.toString()); + var profiledCountWithoutGenePanelData = studyViewRepository.getTotalProfiledCountsByAlterationType(studyViewFilter, alterationType.toString()); + var matchingGenePanelIdsMap = studyViewRepository.getMatchingGenePanelIds(studyViewFilter, alterationType.toString()); updatedAlterationCounts.parallelStream() .forEach(alterationCountByGene -> { diff --git a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java index 6a48daf03ad..5202b2589c6 100644 --- a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java +++ b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java @@ -9,26 +9,19 @@ import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.Sample; import org.cbioportal.persistence.StudyViewRepository; -import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; -import org.cbioportal.persistence.enums.ClinicalAttributeDataType; import org.cbioportal.service.AlterationCountService; import org.cbioportal.service.StudyViewColumnarService; -import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.StudyViewFilter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; @Service public class StudyViewColumnarServiceImpl implements StudyViewColumnarService { - private final Map> clinicalAttributeNameMap = new HashMap<>(); - private final StudyViewRepository studyViewRepository; @@ -43,40 +36,31 @@ public StudyViewColumnarServiceImpl(StudyViewRepository studyViewRepository, Alt @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()") @Override public List getFilteredSamples(StudyViewFilter studyViewFilter) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return studyViewRepository.getFilteredSamples(studyViewFilter, categorizedClinicalDataCountFilter); + return studyViewRepository.getFilteredSamples(studyViewFilter); } @Override public List getMutatedGenes(StudyViewFilter studyViewFilter) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return alterationCountService.getMutatedGenes(studyViewFilter, categorizedClinicalDataCountFilter); + return alterationCountService.getMutatedGenes(studyViewFilter); } @Override public List getGenomicDataCounts(StudyViewFilter studyViewFilter) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return studyViewRepository.getGenomicDataCounts(studyViewFilter, categorizedClinicalDataCountFilter); + return studyViewRepository.getGenomicDataCounts(studyViewFilter); } public List getCnaGenes(StudyViewFilter studyViewFilter) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return alterationCountService.getCnaGenes(studyViewFilter, categorizedClinicalDataCountFilter); + return alterationCountService.getCnaGenes(studyViewFilter); } @Override public List getStructuralVariantGenes(StudyViewFilter studyViewFilter) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return alterationCountService.getStructuralVariantGenes(studyViewFilter, categorizedClinicalDataCountFilter); + return alterationCountService.getStructuralVariantGenes(studyViewFilter); } @Override public List getClinicalDataCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - - - - return studyViewRepository.getClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, filteredAttributes) + return studyViewRepository.getClinicalDataCounts(studyViewFilter, filteredAttributes) .stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) .entrySet().parallelStream().map(e -> { ClinicalDataCountItem item = new ClinicalDataCountItem(); @@ -88,60 +72,18 @@ public List getClinicalDataCounts(StudyViewFilter studyVi @Override public List getCaseListDataCounts(StudyViewFilter studyViewFilter) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return studyViewRepository.getCaseListDataCounts(studyViewFilter, categorizedClinicalDataCountFilter); - } - - private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter) { - if (clinicalAttributeNameMap.isEmpty()) { - buildClinicalAttributeNameMap(); - } - - if (studyViewFilter.getClinicalDataFilters() == null) { - return CategorizedClinicalDataCountFilter.getBuilder().build(); - } - - final String patientCategoricalKey = ClinicalAttributeDataSource.PATIENT.getValue() + ClinicalAttributeDataType.CATEGORICAL.getValue(); - final String patientNumericKey = ClinicalAttributeDataSource.PATIENT.getValue() + ClinicalAttributeDataType.NUMERIC.getValue(); - final String sampleCategoricalKey = ClinicalAttributeDataSource.SAMPLE.getValue() + ClinicalAttributeDataType.CATEGORICAL.getValue(); - final String sampleNumericKey = ClinicalAttributeDataSource.SAMPLE.getValue() + ClinicalAttributeDataType.NUMERIC.getValue(); - - return CategorizedClinicalDataCountFilter.getBuilder() - .setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() - .stream().filter(clinicalDataFilter -> clinicalAttributeNameMap.get(patientCategoricalKey).contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> clinicalAttributeNameMap.get(patientNumericKey).contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> clinicalAttributeNameMap.get(sampleCategoricalKey).contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> clinicalAttributeNameMap.get(sampleNumericKey).contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .build(); - } - - private void buildClinicalAttributeNameMap() { - List clinicalAttributeDataSources = List.of(ClinicalAttributeDataSource.values()); - for (ClinicalAttributeDataSource clinicalAttributeDataSource : clinicalAttributeDataSources) { - String categoricalKey = clinicalAttributeDataSource.getValue() + ClinicalAttributeDataType.CATEGORICAL; - String numericKey = clinicalAttributeDataSource.getValue() + ClinicalAttributeDataType.NUMERIC; - clinicalAttributeNameMap.put(categoricalKey, studyViewRepository.getClinicalDataAttributeNames(clinicalAttributeDataSource, ClinicalAttributeDataType.CATEGORICAL)); - clinicalAttributeNameMap.put(numericKey, studyViewRepository.getClinicalDataAttributeNames(clinicalAttributeDataSource, ClinicalAttributeDataType.NUMERIC)); - } + return studyViewRepository.getCaseListDataCounts(studyViewFilter); } + @Override public List getPatientClinicalData(StudyViewFilter studyViewFilter, List attributeIds) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return studyViewRepository.getPatientClinicalData(studyViewFilter, attributeIds, categorizedClinicalDataCountFilter); + return studyViewRepository.getPatientClinicalData(studyViewFilter, attributeIds); } @Override public List getSampleClinicalData(StudyViewFilter studyViewFilter, List attributeIds) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter); - return studyViewRepository.getSampleClinicalData(studyViewFilter, attributeIds, categorizedClinicalDataCountFilter); + return studyViewRepository.getSampleClinicalData(studyViewFilter, attributeIds); } diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml index 29d52a9e7df..7932a1841d5 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml @@ -91,7 +91,7 @@ - + @@ -115,7 +115,7 @@ - + @@ -135,22 +135,24 @@ SELECT ${unique_id} FROM ${table_name} - WHERE attribute_name = '${clinicalDataFilter.attributeId}' + WHERE attribute_name = '${clinicalDataFilter.attributeId}' AND + type='${type}' AND + (attribute_value = '' OR match(attribute_value, '^[\d\.]+$')) - AND attribute_value = -1000000 + AND attribute_value = '' - AND abs(minus(attribute_value, ${dataFilterValue.start})) < exp(-11) + AND abs(minus(cast(attribute_value as float), ${dataFilterValue.start})) < exp(-11) - AND attribute_value > ${dataFilterValue.start} + AND cast(attribute_value as float) > ${dataFilterValue.start} - AND attribute_value <= ${dataFilterValue.end} + AND cast(attribute_value as float) <= ${dataFilterValue.end} diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml index 9170b676872..b7d785e4991 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml @@ -167,31 +167,7 @@ GROUP BY s.cancer_study_identifier, sl.stable_id, sl.name; - - SELECT - attribute_name as attributeId, - CASE WHEN attribute_value = 'NULL' THEN 'NA' ELSE attribute_value END AS value, - Count(*) as count - FROM ${table_name_prefix}_clinical_attribute_categorical_view - - patient_unique_id IN ( - - INTERSECT - - ) - AND UPPER(attribute_value) NOT IN - - #{filteredAttributeValue} - - AND attribute_name IN - - #{attributeId} - - - GROUP BY attribute_name, - attribute_value - - + SELECT attribute_name as attributeId, @@ -250,20 +226,6 @@ value - - - - - - SELECT sample_unique_id FROM sample_derived @@ -280,10 +242,14 @@ - SELECT - DISTINCT(attribute_name) - FROM ${tableName}; + attr_id as attrId, + datatype as dataType, + patient_attribute as patientAttribute, + cs.cancer_study_identifier cancerStudyIdentifier + FROM clinical_attribute_meta cam + JOIN cancer_study cs on cs.cancer_study_id = cam.cancer_study_id