diff --git a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java index 14c77b47f04..7359ad50c16 100644 --- a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java +++ b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java @@ -1,6 +1,7 @@ package org.cbioportal.persistence.helper; import org.cbioportal.model.ClinicalAttribute; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter; import org.cbioportal.web.parameter.CustomSampleIdentifier; @@ -13,10 +14,12 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public final class StudyViewFilterHelper { public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFilter, @Nullable Map> clinicalAttributesMap, + @Nullable Map> genericAssayProfilesMap, @Nullable List customDataSamples) { if (Objects.isNull(studyViewFilter)) { studyViewFilter = new StudyViewFilter(); @@ -24,10 +27,13 @@ public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFil if (Objects.isNull(clinicalAttributesMap)) { clinicalAttributesMap = new EnumMap<>(ClinicalAttributeDataSource.class); } + if (Objects.isNull(genericAssayProfilesMap)) { + genericAssayProfilesMap = new EnumMap<>(ClinicalAttributeDataSource.class); + } if (Objects.isNull(customDataSamples)) { customDataSamples = new ArrayList<>(); } - return new StudyViewFilterHelper(studyViewFilter, clinicalAttributesMap, customDataSamples); + return new StudyViewFilterHelper(studyViewFilter, clinicalAttributesMap, genericAssayProfilesMap, customDataSamples); } private final StudyViewFilter studyViewFilter; @@ -37,9 +43,10 @@ public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFil private StudyViewFilterHelper(@NonNull StudyViewFilter studyViewFilter, @NonNull Map> clinicalAttributesMap, + @NonNull Map> genericAssayProfilesMap, @NonNull List customDataSamples) { this.studyViewFilter = studyViewFilter; - this.categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter, clinicalAttributesMap); + this.categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter, clinicalAttributesMap, genericAssayProfilesMap); this.customDataSamples = customDataSamples; this.applyPatientIdFilters = shouldApplyPatientIdFilters(); } @@ -56,46 +63,77 @@ public List customDataSamples() { return this.customDataSamples; } - private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter, Map> clinicalAttributesMap) { - - if (studyViewFilter.getClinicalDataFilters() == null || clinicalAttributesMap.isEmpty()) { + private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter, Map> clinicalAttributesMap, Map> genericAssayProfilesMap) { + if ((studyViewFilter.getClinicalDataFilters() == null || clinicalAttributesMap.isEmpty()) && + (studyViewFilter.getGenericAssayDataFilters() == null || genericAssayProfilesMap.isEmpty()) && + studyViewFilter.getGenomicDataFilters() == null) + { return CategorizedClinicalDataCountFilter.getBuilder().build(); } - List patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) - .stream().filter(ca -> ca.getDatatype().equals("STRING")) - .map(ClinicalAttribute::getAttrId) - .toList(); + CategorizedClinicalDataCountFilter.Builder builder = CategorizedClinicalDataCountFilter.getBuilder(); + + if (studyViewFilter.getClinicalDataFilters() != null) { + List patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) + .stream().filter(ca -> ca.getDatatype().equals("STRING")) + .map(ClinicalAttribute::getAttrId) + .toList(); - List patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) - .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) - .map(ClinicalAttribute::getAttrId) - .toList(); + List patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) + .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) + .map(ClinicalAttribute::getAttrId) + .toList(); - List sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(ca -> ca.getDatatype().equals("STRING")) - .map(ClinicalAttribute::getAttrId) - .toList(); + List sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(ca -> ca.getDatatype().equals("STRING")) + .map(ClinicalAttribute::getAttrId) + .toList(); - List sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) - .map(ClinicalAttribute::getAttrId) - .toList(); + List sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) + .map(ClinicalAttribute::getAttrId) + .toList(); + + builder.setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() + .stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())) + .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() + .filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) + .collect(Collectors.toList())); + } + if (studyViewFilter.getGenomicDataFilters() != null) { + builder.setSampleNumericalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() + .filter(genomicDataFilter -> !genomicDataFilter.getProfileType().equals("cna") && !genomicDataFilter.getProfileType().equals("gistic")) + .collect(Collectors.toList())); + builder.setSampleCategoricalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() + .filter(genomicDataFilter -> genomicDataFilter.getProfileType().equals("cna") || genomicDataFilter.getProfileType().equals("gistic")) + .collect(Collectors.toList())); + } + if (studyViewFilter.getGenericAssayDataFilters() != null) { + // TODO: Support patient level profiles and data filtering + List sampleCategoricalProfileTypes = genericAssayProfilesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY")) + .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) + .toList(); - return CategorizedClinicalDataCountFilter.getBuilder() - .setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() - .stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) - .toList()) - .build(); + List sampleNumericalProfileTypes = genericAssayProfilesMap.get(ClinicalAttributeDataSource.SAMPLE) + .stream().filter(profile -> profile.getDatatype().equals("LIMIT-VALUE")) + .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) + .toList(); + builder.setSampleNumericalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() + .filter(genericAssayDataFilter -> sampleNumericalProfileTypes.contains(genericAssayDataFilter.getProfileType())) + .collect(Collectors.toList())); + builder.setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() + .filter(genericAssayDataFilter -> sampleCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType())) + .collect(Collectors.toList())); + } + return builder.build(); } public boolean shouldApplyPatientIdFilters() { diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java index acf77a81ae0..981ac3ee799 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java @@ -68,10 +68,8 @@ public interface StudyViewMapper { List getMutationCountsByType(StudyViewFilterHelper studyViewFilterHelper, List genomicDataFilters); // TODO: update this with new parameters - List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, - boolean applyPatientIdFilters, List attributeIds, List filteredAttributeValues); - List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, - boolean applyPatientIdFilters, List attributeIds, List filteredAttributeValues); + List getGenomicDataBinCounts(StudyViewFilterHelper studyViewFilterHelper, List attributeIds); + List getGenericAssayDataBinCounts(StudyViewFilterHelper studyViewFilterHelper, List attributeIds); List getGenericAssayProfiles(); } diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java index d09a4493054..dc5d147e88b 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java @@ -105,7 +105,7 @@ public List getMolecularProfileSampleCounts(StudyViewFilterCon } public StudyViewFilterHelper createStudyViewFilterHelper(StudyViewFilterContext studyViewFilterContext) { - return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter(), getClinicalAttributeNameMap(), studyViewFilterContext.customDataFilterSamples()); + return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter(), getClinicalAttributeNameMap(), getGenericAssayProfilesMap(), studyViewFilterContext.customDataFilterSamples()); } @Override @@ -208,27 +208,15 @@ public List getSampleTreatments(StudyViewFilterContext studyVie public int getTotalSampleTreatmentCount(StudyViewFilterContext studyViewFilterContext) { return mapper.getTotalSampleTreatmentCounts(createStudyViewFilterHelper(studyViewFilterContext)); } - - // TODO: update parameter name + @Override - public List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractDataCountFilters(studyViewFilter); - return mapper.getGenomicDataBinCounts(studyViewFilter, categorizedClinicalDataCountFilter, - // TODO: This might need to update with patient level information - // setting false to indicate this is sample level data - false, - filteredAttributes, Collections.emptyList() ); + public List getGenomicDataBinCounts(StudyViewFilterContext studyViewFilterContext, List attributeIds) { + return mapper.getGenomicDataBinCounts(createStudyViewFilterHelper(studyViewFilterContext), attributeIds); } - // TODO: update parameter name @Override - public List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { - CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractDataCountFilters(studyViewFilter); - return mapper.getGenericAssayDataBinCounts(studyViewFilter, categorizedClinicalDataCountFilter, - // TODO: This might need to update with patient level information - // setting false to indicate this is sample level data - false, - filteredAttributes, Collections.emptyList() ); + public List getGenericAssayDataBinCounts(StudyViewFilterContext studyViewFilterContext, List attributeIds) { + return mapper.getGenericAssayDataBinCounts(createStudyViewFilterHelper(studyViewFilterContext), attributeIds); } private void buildClinicalAttributeNameMap() { @@ -249,6 +237,13 @@ private Map> getClinicalAtt } return clinicalAttributesMap; } + + private Map> getGenericAssayProfilesMap() { + if (genericAssayProfilesMap.isEmpty()) { + buildGenericAssayProfilesMap(); + } + return genericAssayProfilesMap; + } @Override public List getCNACounts(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters) { @@ -266,96 +261,4 @@ public List getMutationCountsByType(StudyViewFilterContext return mapper.getMutationCountsByType(createStudyViewFilterHelper(studyViewFilterContext), genomicDataFilters); } - // TODO: need to update this into the new format - private CategorizedClinicalDataCountFilter extractDataCountFilters(final StudyViewFilter studyViewFilter) { - if (clinicalAttributesMap.isEmpty()) { - buildClinicalAttributeNameMap(); - } - - if (genericAssayProfilesMap.isEmpty()) { - buildGenericAssayProfilesMap(); - } - - if (studyViewFilter.getClinicalDataFilters() == null && studyViewFilter.getGenomicDataFilters() == null && studyViewFilter.getGenericAssayDataFilters() == null) { - return CategorizedClinicalDataCountFilter.getBuilder().build(); - } - - List patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) - .stream().filter(ca -> ca.getDatatype().equals("STRING")) - .map(ClinicalAttribute::getAttrId) - .toList(); - - List patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT) - .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) - .map(ClinicalAttribute::getAttrId) - .toList(); - - List sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(ca -> ca.getDatatype().equals("STRING")) - .map(ClinicalAttribute::getAttrId) - .toList(); - - List sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(ca -> ca.getDatatype().equals("NUMBER")) - .map(ClinicalAttribute::getAttrId) - .toList(); - - CategorizedClinicalDataCountFilter.Builder builder = CategorizedClinicalDataCountFilter.getBuilder(); - if (studyViewFilter.getClinicalDataFilters() != null) { - builder.setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters() - .stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())) - .setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream() - .filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId())) - .collect(Collectors.toList())); - } - if (studyViewFilter.getGenomicDataFilters() != null) { - // .setPatientCategoricalGenomicDataFilters(studyViewFilter.getClinicalDataFilters() - // .stream().filter(genomicDataFilter -> patientCategoricalAttributes.contains(genomicDataFilter.getAttributeId())) - // .collect(Collectors.toList())) - // .setPatientNumericalGenomicDataFilters(studyViewFilter.getClinicalDataFilters().stream() - // .filter(genomicDataFilter -> patientNumericalAttributes.contains(genomicDataFilter.getAttributeId())) - // .collect(Collectors.toList())) - // .setSampleCategoricalGenomicDataFilters(studyViewFilter.getClinicalDataFilters().stream() - // .filter(genomicDataFilter -> sampleCategoricalAttributes.contains(genomicDataFilter.getAttributeId())) - // .collect(Collectors.toList())) - builder.setSampleNumericalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() - .filter(genomicDataFilter -> genomicDataFilter.getProfileType() != "cna" && genomicDataFilter.getProfileType() != "gistic") - .collect(Collectors.toList())); - } - if (studyViewFilter.getGenericAssayDataFilters() != null) { - // .setPatientCategoricalGenericAssayDataFilters(studyViewFilter.getClinicalDataFilters() - // .stream().filter(genericAssayDataFilter -> patientCategoricalAttributes.contains(genericAssayDataFilter.getAttributeId())) - // .collect(Collectors.toList())) - // .setPatientNumericalGenericAssayDataFilters(studyViewFilter.getClinicalDataFilters().stream() - // .filter(genericAssayDataFilter -> patientNumericalAttributes.contains(genericAssayDataFilter.getAttributeId())) - // .collect(Collectors.toList())) - // .setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getClinicalDataFilters().stream() - // .filter(genericAssayDataFilter -> sampleCategoricalAttributes.contains(genericAssayDataFilter.getAttributeId())) - // .collect(Collectors.toList())) - // TODO: (required) get profile and filter the profiles - List sampleCategoricalProfileTypes = genericAssayProfilesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY")) - .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) - .toList(); - - List sampleNumericalProfileTypes = genericAssayProfilesMap.get(ClinicalAttributeDataSource.SAMPLE) - .stream().filter(profile -> profile.getDatatype().equals("LIMIT-VALUE")) - .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) - .toList(); - builder.setSampleNumericalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() - .filter(genericAssayDataFilter -> sampleNumericalProfileTypes.contains(genericAssayDataFilter.getProfileType())) - .collect(Collectors.toList())); - builder.setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() - .filter(genericAssayDataFilter -> sampleCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType())) - .collect(Collectors.toList())); - } - return builder.build(); - } } \ No newline at end of file diff --git a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java index 86b61ad9d30..5c7c1756ed8 100644 --- a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java +++ b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java @@ -83,7 +83,7 @@ public SampleTreatmentReport getSampleTreatmentReport(StudyViewFilter studyViewF @Override public List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { - return studyViewRepository.getGenomicDataBinCounts(studyViewFilter, filteredAttributes) + return studyViewRepository.getGenomicDataBinCounts(createContext(studyViewFilter), filteredAttributes) .stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) .entrySet().parallelStream().map(e -> { ClinicalDataCountItem item = new ClinicalDataCountItem(); @@ -95,7 +95,7 @@ public List getGenomicDataBinCounts(StudyViewFilter study @Override public List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { - return studyViewRepository.getGenericAssayDataBinCounts(studyViewFilter, filteredAttributes) + return studyViewRepository.getGenericAssayDataBinCounts(createContext(studyViewFilter), filteredAttributes) .stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) .entrySet().parallelStream().map(e -> { ClinicalDataCountItem item = new ClinicalDataCountItem(); diff --git a/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java b/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java index 19d09f81107..8dd973f8628 100644 --- a/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java +++ b/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java @@ -2,19 +2,15 @@ import org.cbioportal.model.*; import org.cbioportal.service.StudyViewColumnarService; -import org.cbioportal.service.exception.MolecularProfileNotFoundException; import org.cbioportal.web.columnar.util.NewClinicalDataBinUtil; import org.cbioportal.web.parameter.*; import org.cbioportal.web.util.DataBinner; -import org.cbioportal.web.util.StudyViewFilterApplier; -import org.cbioportal.web.util.StudyViewFilterUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Component; import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Collections.emptyList; @@ -22,10 +18,7 @@ public class BasicDataBinner { private final StudyViewColumnarService studyViewColumnarService; private final DataBinner dataBinner; - - @Autowired - private StudyViewFilterUtil studyViewFilterUtil; - + @Autowired public BasicDataBinner( StudyViewColumnarService studyViewColumnarService, @@ -45,90 +38,6 @@ private List convertCountsToData(List clinicalD .toList(); } -// // main function to get data bin counts -// // data bin counts filter must be a interface that implements DataBinCountFilter -// // data -// @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()") -// public List fetchClinicalDataBinCounts( -// DataBinMethod dataBinMethod, -// T dataBinCountFilter, -// boolean shouldRemoveSelfFromFilter -// ) { -// List attributes = dataBinCountFilter.getAttributes(); -// StudyViewFilter studyViewFilter = dataBinCountFilter.getStudyViewFilter(); -// -// if (shouldRemoveSelfFromFilter) { -// studyViewFilter = NewClinicalDataBinUtil.removeSelfFromFilter(dataBinCountFilter); -// } -// -// List attributeIds = attributes.stream().map(ClinicalDataBinFilter::getAttributeId).collect(Collectors.toList()); -// -// // a new StudyView filter to partially filter by study and sample ids only -// // we need this additional partial filter because we always need to know the bins generated for the initial state -// // which allows us to keep the number of bins and bin ranges consistent even if there are additional data filters. -// // we only want to update the counts for each bin, we don't want to regenerate the bins for the filtered data. -// // NOTE: partial filter is only needed when dataBinMethod == DataBinMethod.STATIC but that's always the case -// // for the frontend implementation. we can't really use dataBinMethod == DataBinMethod.DYNAMIC because of the -// // complication it brings to the frontend visualization and filtering -// StudyViewFilter partialFilter = new StudyViewFilter(); -// partialFilter.setStudyIds(studyViewFilter.getStudyIds()); -// partialFilter.setSampleIdentifiers(studyViewFilter.getSampleIdentifiers()); -// -// // we need the clinical data for the partial filter in order to generate the bins for initial state -// // we use the filtered data to calculate the counts for each bin, we do not regenerate bins for the filtered data -// List unfilteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(partialFilter, attributeIds); -// List filteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(studyViewFilter, attributeIds); -// -// // TODO ignoring conflictingPatientAttributeIds for now -// List unfilteredClinicalData = convertCountsToData( -// unfilteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList() -// ); -// List filteredClinicalData = convertCountsToData( -// filteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList() -// ); -// -// // TODO: only do this for clinical data, because all genomic data and generic assay data are currently no patient level data -// Map attributeDatatypeMap = studyViewColumnarService.getClinicalAttributeDatatypeMap(); -// -// Map> unfilteredClinicalDataByAttributeId = -// unfilteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId)); -// -// Map> filteredClinicalDataByAttributeId = -// filteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId)); -// -// List clinicalDataBins = Collections.emptyList(); -// -// // TODO: need to update attributeDatatypeMap to ignore patient level data -// if (dataBinMethod == DataBinMethod.STATIC) { -// if (!unfilteredClinicalData.isEmpty()) { -// clinicalDataBins = NewClinicalDataBinUtil.calculateStaticDataBins( -// dataBinner, -// attributes, -// attributeDatatypeMap, -// unfilteredClinicalDataByAttributeId, -// filteredClinicalDataByAttributeId -// ); -// } -// } -// // TODO: need to update attributeDatatypeMap to ignore patient level data -// else { // dataBinMethod == DataBinMethod.DYNAMIC -// // TODO we should consider removing dynamic binning support -// // we never use dynamic binning in the frontend because number of bins and the bin ranges can change -// // each time there is a new filter which makes the frontend implementation complicated -// if (!filteredClinicalData.isEmpty()) { -// clinicalDataBins = NewClinicalDataBinUtil.calculateDynamicDataBins( -// dataBinner, -// attributes, -// attributeDatatypeMap, -// filteredClinicalDataByAttributeId -// ); -// } -// } -// -// return clinicalDataBins; -// } - - @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()") public List getDataBins( DataBinMethod dataBinMethod, @@ -142,8 +51,7 @@ public uniqueKeys = dataBinFilters.stream().map(this::getDataBinFilterUniqueKey).collect(Collectors.toList()); // a new StudyView filter to partially filter by study and sample ids only @@ -277,17 +185,6 @@ private String getDataBinFilterUniqueKey(S dataBinFilt return null; } -// private String fetchData(StudyViewFilter studyViewFilter, List uniqueKeys) { -// if (dataBinFilter instanceof ClinicalDataBinFilter clinicalDataBinFilter) { -// return clinicalDataBinFilter.getAttributeId(); -// } else if (dataBinFilter instanceof GenomicDataBinFilter genomicDataBinFilter) { -// return genomicDataBinFilter.getHugoGeneSymbol() + genomicDataBinFilter.getProfileType(); -// } else if (dataBinFilter instanceof GenericAssayDataBinFilter genericAssayDataBinFilter) { -// return genericAssayDataBinFilter.getStableId() + genericAssayDataBinFilter.getProfileType(); -// } -// return null; -// } - private List calculateStaticDataBins( DataBinner dataBinner, List dataBinFilters, @@ -372,86 +269,6 @@ private ClinicalDataBin dataBinToClinicalDataBin(ClinicalDataBinFilter attribute return clinicalDataBin; } -// private List fetchDataReference( -// S dataBinCountFilter, -// StudyViewFilter studyViewFilter, -// List sampleIds, -// List studyIds -// ) { -// -// // TODO: replace this with sample filtering by using study view filter in the repository -// List filteredSampleIdentifiers = apply(studyViewFilter); -// studyViewFilterUtil.extractStudyAndSampleIds(filteredSampleIdentifiers, studyIds, sampleIds); -// -// List molecularProfiles = molecularProfileService.getMolecularProfilesInStudies(studyIds, -// Projection.SUMMARY.name()); -// -// Map> molecularProfileMap = molecularProfileUtil -// .categorizeMolecularProfilesByStableIdSuffixes(molecularProfiles); -// -// if (dataBinCountFilter instanceof GenomicDataBinCountFilter genomicDataBinCountFilter) { -// List genomicDataBinFilters = genomicDataBinCountFilter.getGenomicDataBinFilters(); -// -// Set hugoGeneSymbols = genomicDataBinFilters.stream().map(GenomicDataBinFilter::getHugoGeneSymbol) -// .collect(Collectors.toSet()); -// -// Map geneSymbolIdMap = geneService -// .fetchGenes(new ArrayList<>(hugoGeneSymbols), GeneIdType.HUGO_GENE_SYMBOL.name(), -// Projection.SUMMARY.name()) -// .stream().collect(Collectors.toMap(Gene::getHugoGeneSymbol, Gene::getEntrezGeneId)); -// -// return genomicDataBinFilters.stream().flatMap(genomicDataBinFilter -> { -// -// Map studyIdToMolecularProfileIdMap = molecularProfileMap -// .getOrDefault(genomicDataBinFilter.getProfileType(), new ArrayList<>()).stream() -// .collect(Collectors.toMap(MolecularProfile::getCancerStudyIdentifier, -// MolecularProfile::getStableId)); -// -// return invokeDataFunc(sampleIds, studyIds, -// List.of(geneSymbolIdMap.get(genomicDataBinFilter.getHugoGeneSymbol()).toString()), -// studyIdToMolecularProfileIdMap, studyViewFilterUtil.getDataBinFilterUniqueKey(genomicDataBinFilter), -// fetchMolecularData); -// }).collect(Collectors.toList()); -// } else if (dataBinCountFilter instanceof GenericAssayDataBinCountFilter genomicDataBinCountFilter) { -// -// List genericAssayDataBinFilters = genomicDataBinCountFilter -// .getGenericAssayDataBinFilters(); -// -// return genericAssayDataBinFilters.stream().flatMap(genericAssayDataBinFilter -> { -// -// Map studyIdToMolecularProfileIdMap = molecularProfileMap -// .getOrDefault(genericAssayDataBinFilter.getProfileType(), new ArrayList<>()) -// .stream().collect(Collectors.toMap(MolecularProfile::getCancerStudyIdentifier, -// MolecularProfile::getStableId)); -// -// return invokeDataFunc(sampleIds, studyIds, Collections.singletonList(genericAssayDataBinFilter.getStableId()), -// studyIdToMolecularProfileIdMap, studyViewFilterUtil.getDataBinFilterUniqueKey(genericAssayDataBinFilter), -// fetchGenericAssayData); -// -// }).collect(Collectors.toList()); -// -// } -// -// return new ArrayList<>(); -// } -// -// public List getUniqkeyKeys(List studyIds, List caseIds) { -// List uniqkeyKeys = new ArrayList(); -// for (int i = 0; i < caseIds.size(); i++) { -// uniqkeyKeys.add(studyViewFilterUtil.getCaseUniqueKey(studyIds.get(i), caseIds.get(i))); -// } -// return uniqkeyKeys; -// } -// -//// private T transform(S dataBinFilter, DataBin dataBin) { -//// if (dataBinFilter instanceof GenomicDataBinFilter genomicDataBinFilter) { -//// return (T) dataBintoGenomicDataBin(genomicDataBinFilter, dataBin); -//// } else if (dataBinFilter instanceof GenericAssayDataBinFilter genericAssayDataBinFilter) { -//// return (T) dataBintoGenericAssayDataBin(genericAssayDataBinFilter, dataBin); -//// } -//// return null; -//// } - private GenomicDataBin dataBintoGenomicDataBin(GenomicDataBinFilter genomicDataBinFilter, DataBin dataBin) { GenomicDataBin genomicDataBin = new GenomicDataBin(); genomicDataBin.setCount(dataBin.getCount()); @@ -487,73 +304,4 @@ private GenericAssayDataBin dataBintoGenericAssayDataBin(GenericAssayDataBinFilt return genericAssayDataBin; } -// private Stream invokeDataFunc(List sampleIds, List studyIds, -// List stableIds, Map studyIdToMolecularProfileIdMap, String attributeId, -// FourParameterFunction, List, List, String, List> dataFunc) { -// -// List mappedSampleIds = new ArrayList<>(); -// List mappedProfileIds = new ArrayList<>(); -// -// for (int i = 0; i < sampleIds.size(); i++) { -// String studyId = studyIds.get(i); -// if (studyIdToMolecularProfileIdMap.containsKey(studyId)) { -// mappedSampleIds.add(sampleIds.get(i)); -// mappedProfileIds.add(studyIdToMolecularProfileIdMap.get(studyId)); -// } -// } -// -// if (mappedSampleIds.isEmpty()) { -// return Stream.of(); -// } -// return dataFunc.apply(mappedProfileIds, mappedSampleIds, stableIds, attributeId) -// .stream(); -// } - - @FunctionalInterface - private interface FourParameterFunction { - R apply(T t, U u, V v, W w); - } -// FourParameterFunction, List, List, String, List> fetchMolecularData = ( -// mappedProfileIds, mappedSampleIds, stableIds, attributeId) -> -// molecularDataService.getMolecularDataInMultipleMolecularProfiles(mappedProfileIds, mappedSampleIds, -// stableIds.stream().map(Integer::parseInt).toList(), Projection.SUMMARY.name()) -// .stream().map(geneMolecularData -> -// transformDataToClinicalData(geneMolecularData, attributeId, geneMolecularData.getValue())) -// .collect(Collectors.toList()); -// -// FourParameterFunction, List, List, String, List> fetchGenericAssayData = ( -// mappedProfileIds, mappedSampleIds, stableIds, attributeId) -> { -// -// try { -// return genericAssayService -// .fetchGenericAssayData(mappedProfileIds, mappedSampleIds, stableIds, Projection.SUMMARY.name()) -// .stream().map(genericAssayData -> -// transformDataToClinicalData(genericAssayData, attributeId, genericAssayData.getValue()) -// ).collect(Collectors.toList()); -// } catch (MolecularProfileNotFoundException e) { -// return new ArrayList<>(); -// } -// }; - -// private ClinicalData transformDataToClinicalData(S data, String attributeId, String attributeValue) { -// ClinicalData clinicalData = new ClinicalData(); -// -// if (data instanceof MolecularData molecularData) { -// clinicalData.setPatientId(molecularData.getPatientId()); -// clinicalData.setSampleId(molecularData.getSampleId()); -// clinicalData.setStudyId(molecularData.getStudyId()); -// } else if (data instanceof Mutation mutationData) { -// clinicalData.setPatientId(mutationData.getPatientId()); -// clinicalData.setSampleId(mutationData.getSampleId()); -// clinicalData.setStudyId(mutationData.getStudyId()); -// } else { -// return clinicalData; -// } -// -// clinicalData.setAttrValue(attributeValue); -// clinicalData.setAttrId(attributeId); -// -// return clinicalData; -// } - } diff --git a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java index 615a4f679fe..fc55e05481b 100644 --- a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java +++ b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java @@ -544,7 +544,6 @@ public ResponseEntity> fetchGenomicDataBinCounts( @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, @RequestAttribute(required = false, value = "interceptedGenomicDataBinCountFilter") GenomicDataBinCountFilter interceptedGenomicDataBinCountFilter ) { - // TODO: create a general data binner that can be used in clinical data, genomic data and generic assay data endpoints List genomicDataBins = basicDataBinner.getDataBins( dataBinMethod, interceptedGenomicDataBinCountFilter, @@ -564,7 +563,6 @@ public ResponseEntity> fetchGenericAssayDataBinCounts( @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, @RequestAttribute(required = false, value = "interceptedGenericAssayDataBinCountFilter") GenericAssayDataBinCountFilter interceptedGenericAssayDataBinCountFilter ) { - // TODO: create a general data binner that can be used in clinical data, genomic data and generic assay data endpoints List genericAssayDataBins = basicDataBinner.getDataBins( dataBinMethod, interceptedGenericAssayDataBinCountFilter, diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml index 586a01bff72..9e49b493642 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml @@ -149,23 +149,15 @@ - - - - - - - - - + - - + + @@ -173,8 +165,14 @@ - - + + + + + + + + @@ -182,8 +180,8 @@ - - + + @@ -383,18 +381,39 @@ - - AND match(alteration_value, '^[\d\.]+$') + + AND match(alteration_value, '^>?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(alteration_value, '^<?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(alteration_value, '^[-+]?[0-9]*[.,]?[0-9]+$') + + - AND abs(minus(cast(alteration_value as float), ${dataFilterValue.start})) < exp(-11) + AND abs( + minus( + + + , + ${dataFilterValue.start} + ) + ) < exp(-11) - AND cast(alteration_value as float) > ${dataFilterValue.start} + AND + + + > ${dataFilterValue.start} - AND cast(alteration_value as float) <= ${dataFilterValue.end} + AND + + + <= ${dataFilterValue.end} @@ -417,18 +436,39 @@ - - AND match(value, '^[\d\.]+$') + + AND match(value, '^>?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(value, '^<?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(value, '^[-+]?[0-9]*[.,]?[0-9]+$') + + - AND abs(minus(cast(value as float), ${dataFilterValue.start})) < exp(-11) + AND abs( + minus( + + + , + ${dataFilterValue.start} + ) + ) < exp(-11) - AND cast(value as float) > ${dataFilterValue.start} + AND + + + > ${dataFilterValue.start} - AND cast(value as float) <= ${dataFilterValue.end} + AND + + + <= ${dataFilterValue.end} @@ -436,15 +476,19 @@ - + SELECT ${unique_id} FROM ${table_name} WHERE entity_stable_id = '${genericAssayDataFilter.stableId}' AND profile_type='${genericAssayDataFilter.profileType}' - + - AND value = '${dataFilterValue.value}' + AND ( + + + + ) = '${dataFilterValue.value}' diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml index e113abe2b7d..4e546638391 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml @@ -452,35 +452,6 @@ ) - - @@ -585,14 +556,41 @@ GROUP BY treatments.treatment; + +