diff --git a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java index b5c7748ffe9..3b60e17fcd4 100644 --- a/src/main/java/org/cbioportal/persistence/StudyViewRepository.java +++ b/src/main/java/org/cbioportal/persistence/StudyViewRepository.java @@ -9,6 +9,7 @@ import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.model.PatientTreatment; import org.cbioportal.model.Sample; import org.cbioportal.model.SampleTreatment; @@ -67,4 +68,10 @@ public interface StudyViewRepository { Map getMutationCounts(StudyViewFilterContext studyViewFilterContext, GenomicDataFilter genomicDataFilter); List getMutationCountsByType(StudyViewFilterContext studyViewFilterContext, List genomicDataFilters); + + List getGenomicDataBinCounts(StudyViewFilterContext studyViewFilterContext, List filteredAttributes); + + List getGenericAssayDataBinCounts(StudyViewFilterContext studyViewFilterContext, List filteredAttributes); + + List getGenericAssayProfiles(); } diff --git a/src/main/java/org/cbioportal/persistence/enums/ClinicalAttributeDataSource.java b/src/main/java/org/cbioportal/persistence/enums/DataSource.java similarity index 71% rename from src/main/java/org/cbioportal/persistence/enums/ClinicalAttributeDataSource.java rename to src/main/java/org/cbioportal/persistence/enums/DataSource.java index ed320c1cebd..09b81b79edb 100644 --- a/src/main/java/org/cbioportal/persistence/enums/ClinicalAttributeDataSource.java +++ b/src/main/java/org/cbioportal/persistence/enums/DataSource.java @@ -1,11 +1,11 @@ package org.cbioportal.persistence.enums; -public enum ClinicalAttributeDataSource { +public enum DataSource { PATIENT("PATIENT"),SAMPLE("SAMPLE"); private final String value; - ClinicalAttributeDataSource(String value) { + DataSource(String value) { this.value = value; } diff --git a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java index a5d16a6de94..c94600c57d8 100644 --- a/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java +++ b/src/main/java/org/cbioportal/persistence/helper/StudyViewFilterHelper.java @@ -1,44 +1,109 @@ package org.cbioportal.persistence.helper; +import org.cbioportal.model.MolecularProfile; +import org.cbioportal.persistence.enums.DataSource; import org.cbioportal.web.parameter.ClinicalDataFilter; +import org.cbioportal.web.parameter.CategorizedGenericAssayDataCountFilter; +import org.cbioportal.web.parameter.CategorizedGenomicDataCountFilter; import org.cbioportal.web.parameter.CustomSampleIdentifier; import org.cbioportal.web.parameter.StudyViewFilter; import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; import java.util.ArrayList; +import java.util.EnumMap; import java.util.List; +import java.util.Map; import java.util.Objects; public final class StudyViewFilterHelper { public static StudyViewFilterHelper build(@Nullable StudyViewFilter studyViewFilter, - @Nullable List customDataSamples) { + @Nullable Map> genericAssayProfilesMap, + @Nullable List customDataSamples) { if (Objects.isNull(studyViewFilter)) { studyViewFilter = new StudyViewFilter(); } + if (Objects.isNull(genericAssayProfilesMap)) { + genericAssayProfilesMap = new EnumMap<>(DataSource.class); + } if (Objects.isNull(customDataSamples)) { customDataSamples = new ArrayList<>(); } - return new StudyViewFilterHelper(studyViewFilter, customDataSamples); + return new StudyViewFilterHelper(studyViewFilter, genericAssayProfilesMap, customDataSamples); } private final StudyViewFilter studyViewFilter; + private final CategorizedGenomicDataCountFilter categorizedGenomicDataCountFilter; + private final CategorizedGenericAssayDataCountFilter categorizedGenericAssayDataCountFilter; private final List customDataSamples; - private StudyViewFilterHelper(@NonNull StudyViewFilter studyViewFilter, - @NonNull List customDataSamples) { + private StudyViewFilterHelper(@NonNull StudyViewFilter studyViewFilter, + @NonNull Map> genericAssayProfilesMap, + @NonNull List customDataSamples) { this.studyViewFilter = studyViewFilter; - + this.categorizedGenomicDataCountFilter = extractGenomicDataCountFilters(studyViewFilter); + this.categorizedGenericAssayDataCountFilter = extractGenericAssayDataCountFilters(studyViewFilter, genericAssayProfilesMap); this.customDataSamples = customDataSamples; } public StudyViewFilter studyViewFilter() { return studyViewFilter; } + + public CategorizedGenomicDataCountFilter categorizedGenomicDataCountFilter() { + return categorizedGenomicDataCountFilter; + } + public CategorizedGenericAssayDataCountFilter categorizedGenericAssayDataCountFilter() { + return categorizedGenericAssayDataCountFilter; + } + public List customDataSamples() { return this.customDataSamples; } + + private CategorizedGenomicDataCountFilter extractGenomicDataCountFilters(final StudyViewFilter studyViewFilter) { + if (studyViewFilter.getGenomicDataFilters() == null) + { + return CategorizedGenomicDataCountFilter.getBuilder().build(); + } + + CategorizedGenomicDataCountFilter.Builder builder = CategorizedGenomicDataCountFilter.getBuilder(); + + builder.setSampleNumericalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() + .filter(genomicDataFilter -> !genomicDataFilter.getProfileType().equals("cna") && !genomicDataFilter.getProfileType().equals("gistic")) + .toList()); + builder.setSampleCategoricalGenomicDataFilters(studyViewFilter.getGenomicDataFilters().stream() + .filter(genomicDataFilter -> genomicDataFilter.getProfileType().equals("cna") || genomicDataFilter.getProfileType().equals("gistic")) + .toList()); + return builder.build(); + } + + private CategorizedGenericAssayDataCountFilter extractGenericAssayDataCountFilters(final StudyViewFilter studyViewFilter, Map> genericAssayProfilesMap) { + if ((studyViewFilter.getGenericAssayDataFilters() == null || genericAssayProfilesMap.isEmpty())) + { + return CategorizedGenericAssayDataCountFilter.getBuilder().build(); + } + + CategorizedGenericAssayDataCountFilter.Builder builder = CategorizedGenericAssayDataCountFilter.getBuilder(); + + // TODO: Support patient level profiles and data filtering + List sampleCategoricalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE) + .stream().filter(profile -> profile.getDatatype().equals("CATEGORICAL") || profile.getDatatype().equals("BINARY")) + .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) + .toList(); + List sampleNumericalProfileTypes = genericAssayProfilesMap.get(DataSource.SAMPLE) + .stream().filter(profile -> profile.getDatatype().equals("LIMIT-VALUE")) + .map(profile -> profile.getStableId().replace(profile.getCancerStudyIdentifier() + "_", "")) + .toList(); + builder.setSampleNumericalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() + .filter(genericAssayDataFilter -> sampleNumericalProfileTypes.contains(genericAssayDataFilter.getProfileType())) + .toList()); + builder.setSampleCategoricalGenericAssayDataFilters(studyViewFilter.getGenericAssayDataFilters().stream() + .filter(genericAssayDataFilter -> sampleCategoricalProfileTypes.contains(genericAssayDataFilter.getProfileType())) + .toList()); + return builder.build(); + } public boolean isCategoricalClinicalDataFilter(ClinicalDataFilter clinicalDataFilter) { var filterValue = clinicalDataFilter.getValues().getFirst(); diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java index 6dabd46d0df..5bc659e5c5a 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java @@ -12,6 +12,7 @@ import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.PatientTreatment; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.model.Sample; import org.cbioportal.model.SampleTreatment; import org.cbioportal.persistence.helper.AlterationFilterHelper; @@ -65,4 +66,9 @@ public interface StudyViewMapper { Map getMutationCounts(StudyViewFilterHelper studyViewFilterHelper, GenomicDataFilter genomicDataFilter); List getMutationCountsByType(StudyViewFilterHelper studyViewFilterHelper, List genomicDataFilters); + + List getGenomicDataBinCounts(StudyViewFilterHelper studyViewFilterHelper, List attributeIds); + List getGenericAssayDataBinCounts(StudyViewFilterHelper studyViewFilterHelper, List attributeIds); + + List getGenericAssayProfiles(); } diff --git a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java index d522b2f3b9b..41687281c5d 100644 --- a/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java +++ b/src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java @@ -9,12 +9,13 @@ import org.cbioportal.model.GenomicDataCountItem; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.CopyNumberCountByGene; +import org.cbioportal.model.MolecularProfile; import org.cbioportal.model.PatientTreatment; import org.cbioportal.model.Sample; import org.cbioportal.model.SampleTreatment; import org.cbioportal.model.StudyViewFilterContext; import org.cbioportal.persistence.StudyViewRepository; -import org.cbioportal.persistence.enums.ClinicalAttributeDataSource; +import org.cbioportal.persistence.enums.DataSource; import org.cbioportal.persistence.helper.AlterationFilterHelper; import org.cbioportal.persistence.helper.StudyViewFilterHelper; import org.cbioportal.web.parameter.ClinicalDataType; @@ -24,6 +25,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,9 +35,9 @@ @Repository public class StudyViewMyBatisRepository implements StudyViewRepository { - private Map> clinicalAttributesMap = new HashMap<>(); - - + private Map> clinicalAttributesMap = new EnumMap<>(DataSource.class); + private Map> genericAssayProfilesMap = new EnumMap<>(DataSource.class); + private static final List FILTERED_CLINICAL_ATTR_VALUES = Collections.emptyList(); private final StudyViewMapper mapper; @@ -101,7 +103,7 @@ public List getMolecularProfileSampleCounts(StudyViewFilterCon } public StudyViewFilterHelper createStudyViewFilterHelper(StudyViewFilterContext studyViewFilterContext) { - return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter(), studyViewFilterContext.customDataFilterSamples()); + return StudyViewFilterHelper.build(studyViewFilterContext.studyViewFilter() , getGenericAssayProfilesMap(), studyViewFilterContext.customDataFilterSamples()); } @Override @@ -109,6 +111,11 @@ public List getClinicalAttributes() { return mapper.getClinicalAttributes(); } + @Override + public List getGenericAssayProfiles() { + return mapper.getGenericAssayProfiles(); + } + @Override public Map getClinicalAttributeDatatypeMap() { if (clinicalAttributesMap.isEmpty()) { @@ -118,11 +125,11 @@ public Map getClinicalAttributeDatatypeMap() { Map attributeDatatypeMap = new HashMap<>(); clinicalAttributesMap - .get(ClinicalAttributeDataSource.SAMPLE) + .get(DataSource.SAMPLE) .forEach(attribute -> attributeDatatypeMap.put(attribute.getAttrId(), ClinicalDataType.SAMPLE)); clinicalAttributesMap - .get(ClinicalAttributeDataSource.PATIENT) + .get(DataSource.PATIENT) .forEach(attribute -> attributeDatatypeMap.put(attribute.getAttrId(), ClinicalDataType.PATIENT)); return attributeDatatypeMap; @@ -199,18 +206,34 @@ public List getSampleTreatments(StudyViewFilterContext studyVie public int getTotalSampleTreatmentCount(StudyViewFilterContext studyViewFilterContext) { return mapper.getTotalSampleTreatmentCounts(createStudyViewFilterHelper(studyViewFilterContext)); } + + @Override + public List getGenomicDataBinCounts(StudyViewFilterContext studyViewFilterContext, List attributeIds) { + return mapper.getGenomicDataBinCounts(createStudyViewFilterHelper(studyViewFilterContext), attributeIds); + } + @Override + public List getGenericAssayDataBinCounts(StudyViewFilterContext studyViewFilterContext, List attributeIds) { + return mapper.getGenericAssayDataBinCounts(createStudyViewFilterHelper(studyViewFilterContext), attributeIds); + } + private void buildClinicalAttributeNameMap() { clinicalAttributesMap = this.getClinicalAttributes() .stream() - .collect(Collectors.groupingBy(ca -> ca.getPatientAttribute() ? ClinicalAttributeDataSource.PATIENT : ClinicalAttributeDataSource.SAMPLE)); + .collect(Collectors.groupingBy(ca -> ca.getPatientAttribute().booleanValue() ? DataSource.PATIENT : DataSource.SAMPLE)); } - - private Map> getClinicalAttributeNameMap() { - if (clinicalAttributesMap.isEmpty()) { - buildClinicalAttributeNameMap(); + + private void buildGenericAssayProfilesMap() { + genericAssayProfilesMap = this.getGenericAssayProfiles() + .stream() + .collect(Collectors.groupingBy(ca -> ca.getPatientLevel().booleanValue() ? DataSource.PATIENT : DataSource.SAMPLE)); + } + + private Map> getGenericAssayProfilesMap() { + if (genericAssayProfilesMap.isEmpty()) { + buildGenericAssayProfilesMap(); } - return clinicalAttributesMap; + return genericAssayProfilesMap; } @Override diff --git a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java index 369f33aab41..4a631a67204 100644 --- a/src/main/java/org/cbioportal/service/StudyViewColumnarService.java +++ b/src/main/java/org/cbioportal/service/StudyViewColumnarService.java @@ -45,6 +45,10 @@ public interface StudyViewColumnarService { List getCNACountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); List getMutationCountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); + + List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes); + + List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes); List getMutationTypeCountsByGeneSpecific(StudyViewFilter studyViewFilter, List genomicDataFilters); } diff --git a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java index c3ebffa4a4f..b1ada8fac31 100644 --- a/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java +++ b/src/main/java/org/cbioportal/service/impl/StudyViewColumnarServiceImpl.java @@ -81,6 +81,16 @@ public SampleTreatmentReport getSampleTreatmentReport(StudyViewFilter studyViewF return treatmentCountReportService.getSampleTreatmentReport(createContext(studyViewFilter)); } + @Override + public List getGenomicDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { + return generateDataCountItemsFromDataCounts(studyViewRepository.getGenomicDataBinCounts(createContext(studyViewFilter), filteredAttributes)); + } + + @Override + public List getGenericAssayDataBinCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { + return generateDataCountItemsFromDataCounts(studyViewRepository.getGenericAssayDataBinCounts(createContext(studyViewFilter), filteredAttributes)); + } + public List getCnaGenes(StudyViewFilter studyViewFilter) { return alterationCountService.getCnaGenes(createContext(studyViewFilter)); } @@ -97,14 +107,7 @@ public Map getClinicalAttributeDatatypeMap() { @Override public List getClinicalDataCounts(StudyViewFilter studyViewFilter, List filteredAttributes) { - return studyViewRepository.getClinicalDataCounts(createContext(studyViewFilter), filteredAttributes) - .stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) - .entrySet().parallelStream().map(e -> { - ClinicalDataCountItem item = new ClinicalDataCountItem(); - item.setAttributeId(e.getKey()); - item.setCounts(e.getValue()); - return item; - }).collect(Collectors.toList()); + return generateDataCountItemsFromDataCounts(studyViewRepository.getClinicalDataCounts(createContext(studyViewFilter), filteredAttributes)); } @Override @@ -156,8 +159,16 @@ private StudyViewFilterContext createContext(StudyViewFilter studyViewFilter) { List customSampleIdentifiers = customDataFilterUtil.extractCustomDataSamples(studyViewFilter); return new StudyViewFilterContext(studyViewFilter, customSampleIdentifiers); } - - + + private List generateDataCountItemsFromDataCounts(List dataCounts) { + return dataCounts.stream().collect(Collectors.groupingBy(ClinicalDataCount::getAttributeId)) + .entrySet().parallelStream().map(e -> { + ClinicalDataCountItem item = new ClinicalDataCountItem(); + item.setAttributeId(e.getKey()); + item.setCounts(e.getValue()); + return item; + }).toList(); + } public static List mergeCaseListCounts(List counts) { Map> countsPerListType = counts.stream() diff --git a/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java b/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java new file mode 100644 index 00000000000..d6978e35a5e --- /dev/null +++ b/src/main/java/org/cbioportal/web/columnar/BasicDataBinner.java @@ -0,0 +1,316 @@ +package org.cbioportal.web.columnar; + +import org.cbioportal.model.*; +import org.cbioportal.service.StudyViewColumnarService; +import org.cbioportal.web.columnar.util.NewClinicalDataBinUtil; +import org.cbioportal.web.parameter.*; +import org.cbioportal.web.util.DataBinner; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +// BasicDataBinner is a generalized class derived from ClinicalDataBinner +// BasicDataBinner should eventually deprecate ClinicalDataBinner +// we are using BasicDataBinner for genomic data bin counts and generic assay bin counts now +// but BasicDataBinner can support clinical data counts too +// after we switched clinical data counts to use this, then We can remove ClinicalDataBinner +@Component +public class BasicDataBinner { + private final StudyViewColumnarService studyViewColumnarService; + private final DataBinner dataBinner; + + @Autowired + public BasicDataBinner( + StudyViewColumnarService studyViewColumnarService, + DataBinner dataBinner + ) { + this.studyViewColumnarService = studyViewColumnarService; + this.dataBinner = dataBinner; + } + + // convert from counts to clinical data + private List convertCountsToData(List clinicalDataCounts) + { + return clinicalDataCounts + .stream() + .map(NewClinicalDataBinUtil::generateClinicalDataFromClinicalDataCount) + .flatMap(Collection::stream) + .toList(); + } + + @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()") + public List getDataBins( + DataBinMethod dataBinMethod, + T dataBinCountFilter, + boolean shouldRemoveSelfFromFilter) { + // get data bin filters based on the type of the filter + // either Genomic data or Generic Assay data or clinical data + List dataBinFilters = fetchDataBinFilters(dataBinCountFilter); + StudyViewFilter studyViewFilter = dataBinCountFilter.getStudyViewFilter(); + // define result variables + List resultDataBins = Collections.emptyList(); + // if no data bin filters or no study view filer object is passed in + // return empty result + if (dataBinFilters.isEmpty() || studyViewFilter == null) { + return resultDataBins; + } + + if (shouldRemoveSelfFromFilter && dataBinFilters.size() == 1) { + removeSelfFromFilter(dataBinFilters.get(0), studyViewFilter); + } + + List uniqueKeys = dataBinFilters.stream().map(this::getDataBinFilterUniqueKey).toList(); + + // a new StudyView filter to partially filter by study and sample ids only + // we need this additional partial filter because we always need to know the bins generated for the initial state + // which allows us to keep the number of bins and bin ranges consistent even if there are additional data filters. + // we only want to update the counts for each bin, we don't want to regenerate the bins for the filtered data. + // NOTE: partial filter is only needed when dataBinMethod == DataBinMethod.STATIC but that's always the case + // for the frontend implementation. we can't really use dataBinMethod == DataBinMethod.DYNAMIC because of the + // complication it brings to the frontend visualization and filtering + StudyViewFilter partialFilter = new StudyViewFilter(); + partialFilter.setStudyIds(studyViewFilter.getStudyIds()); + partialFilter.setSampleIdentifiers(studyViewFilter.getSampleIdentifiers()); + + // we need to fetch data for the partial filter in order to generate the bins for initial state + // we use the filtered data to calculate the counts for each bin, we do not regenerate bins for the filtered data + List unfilteredClinicalDataCounts; + List filteredClinicalDataCounts; + Map attributeDatatypeMap; + switch (dataBinCountFilter) { + // TODO: first case is to support clinical data, but clinical data is not using this now. We should update controller to use this method later + case ClinicalDataBinCountFilter clinicalDataBinCountFilter -> { + unfilteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(partialFilter, uniqueKeys); + filteredClinicalDataCounts = studyViewColumnarService.getClinicalDataCounts(studyViewFilter, uniqueKeys); + attributeDatatypeMap = studyViewColumnarService.getClinicalAttributeDatatypeMap(); + } + case GenomicDataBinCountFilter genomicDataBinCountFilter -> { + unfilteredClinicalDataCounts = studyViewColumnarService.getGenomicDataBinCounts(partialFilter, uniqueKeys); + filteredClinicalDataCounts = studyViewColumnarService.getGenomicDataBinCounts(studyViewFilter, uniqueKeys); + attributeDatatypeMap = Collections.emptyMap(); + } + case GenericAssayDataBinCountFilter genericAssayDataBinCountFilter -> { + unfilteredClinicalDataCounts = studyViewColumnarService.getGenericAssayDataBinCounts(partialFilter, uniqueKeys); + filteredClinicalDataCounts = studyViewColumnarService.getGenericAssayDataBinCounts(studyViewFilter, uniqueKeys); + attributeDatatypeMap = Collections.emptyMap(); + } + default -> { + unfilteredClinicalDataCounts = Collections.emptyList(); + filteredClinicalDataCounts = Collections.emptyList(); + attributeDatatypeMap = Collections.emptyMap(); + } + } + + // TODO ignoring conflictingPatientAttributeIds for now + List unfilteredClinicalData = convertCountsToData( + unfilteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList() + ); + List filteredClinicalData = convertCountsToData( + filteredClinicalDataCounts.stream().flatMap(c -> c.getCounts().stream()).toList() + ); + + Map> unfilteredClinicalDataByAttributeId = + unfilteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId)); + + Map> filteredClinicalDataByAttributeId = + filteredClinicalData.stream().collect(Collectors.groupingBy(Binnable::getAttrId)); + + // TODO: need to update attributeDatatypeMap to include patient level data for Generic Assay Profiles + if (dataBinMethod == DataBinMethod.STATIC) { + if (!unfilteredClinicalData.isEmpty()) { + resultDataBins = calculateStaticDataBins( + dataBinner, + dataBinFilters, + attributeDatatypeMap, + unfilteredClinicalDataByAttributeId, + filteredClinicalDataByAttributeId + ); + } + } + // TODO: need to update attributeDatatypeMap to include patient level data for Generic Assay Profiles + else { // dataBinMethod == DataBinMethod.DYNAMIC + // TODO we should consider removing dynamic binning support + // we never use dynamic binning in the frontend because number of bins and the bin ranges can change + // each time there is a new filter which makes the frontend implementation complicated + if (!filteredClinicalData.isEmpty()) { + resultDataBins = calculateDynamicDataBins( + dataBinner, + dataBinFilters, + attributeDatatypeMap, + filteredClinicalDataByAttributeId + ); + } + } + + return resultDataBins; + } + + private void removeSelfFromFilter(S dataBinFilter, StudyViewFilter studyViewFilter) { + switch (dataBinFilter) { + case ClinicalDataBinFilter clinicalDataBinFilter when studyViewFilter.getClinicalDataFilters() != null -> + studyViewFilter.getClinicalDataFilters().removeIf(f -> f.getAttributeId().equals(clinicalDataBinFilter.getAttributeId())); + case GenomicDataBinFilter genomicDataBinFilter when studyViewFilter.getGenomicDataFilters() != null -> + studyViewFilter.getGenomicDataFilters().removeIf(f -> + f.getHugoGeneSymbol().equals(genomicDataBinFilter.getHugoGeneSymbol()) + && f.getProfileType().equals(genomicDataBinFilter.getProfileType()) + ); + case GenericAssayDataBinFilter genericAssayDataBinFilter when studyViewFilter.getGenericAssayDataFilters() != null -> + studyViewFilter.getGenericAssayDataFilters().removeIf(f -> + f.getStableId().equals(genericAssayDataBinFilter.getStableId()) + && f.getProfileType().equals(genericAssayDataBinFilter.getProfileType()) + ); + default -> { + // Do not remove any filters + } + } + } + + private List fetchDataBinFilters(T dataBinCountFilter) { + switch (dataBinCountFilter) { + case ClinicalDataBinCountFilter clinicalDataBinCountFilter -> { + return (List) clinicalDataBinCountFilter.getAttributes(); + } + case GenomicDataBinCountFilter genomicDataBinCountFilter -> { + return (List) genomicDataBinCountFilter.getGenomicDataBinFilters(); + } + case GenericAssayDataBinCountFilter genericAssayDataBinCountFilter -> { + return (List) genericAssayDataBinCountFilter.getGenericAssayDataBinFilters(); + } + default -> { + return new ArrayList<>(); + } + } + } + + private String getDataBinFilterUniqueKey(S dataBinFilter) { + switch (dataBinFilter) { + case ClinicalDataBinFilter clinicalDataBinFilter -> { + return clinicalDataBinFilter.getAttributeId(); + } + case GenomicDataBinFilter genomicDataBinFilter -> { + return genomicDataBinFilter.getHugoGeneSymbol() + genomicDataBinFilter.getProfileType(); + } + case GenericAssayDataBinFilter genericAssayDataBinFilter -> { + return genericAssayDataBinFilter.getStableId() + genericAssayDataBinFilter.getProfileType(); + } + default -> { + return null; + } + } + } + + private List calculateStaticDataBins( + DataBinner dataBinner, + List dataBinFilters, + Map attributeDatatypeMap, + Map> unfilteredClinicalDataByAttributeId, + Map> filteredClinicalDataByAttributeId + ) { + List result = new ArrayList<>(); + + for (T dataBinFilter : dataBinFilters) { + // if there is data for requested attribute + if (attributeDatatypeMap.isEmpty() || attributeDatatypeMap.containsKey(getDataBinFilterUniqueKey(dataBinFilter))) { + List dataBins = dataBinner + .calculateClinicalDataBins( + dataBinFilter, + filteredClinicalDataByAttributeId.getOrDefault(getDataBinFilterUniqueKey(dataBinFilter), emptyList()), + unfilteredClinicalDataByAttributeId.getOrDefault(getDataBinFilterUniqueKey(dataBinFilter), emptyList()) + ) + .stream() + .map(dataBin -> (U) transform(dataBinFilter, dataBin)) + .toList(); + + result.addAll(dataBins); + } + } + + return result; + } + + private List calculateDynamicDataBins( + DataBinner dataBinner, + List dataBinFilters, + Map attributeDatatypeMap, + Map> filteredClinicalDataByAttributeId + ) { + List result = new ArrayList<>(); + + for (T dataBinFilter : dataBinFilters) { + // if there is data for requested attribute + if (attributeDatatypeMap.isEmpty() || attributeDatatypeMap.containsKey(getDataBinFilterUniqueKey(dataBinFilter))) { + List dataBins = dataBinner + .calculateDataBins( + dataBinFilter, + filteredClinicalDataByAttributeId.getOrDefault(getDataBinFilterUniqueKey(dataBinFilter), emptyList()) + ) + .stream() + .map(dataBin -> (U) transform(dataBinFilter, dataBin)) + .toList(); + result.addAll(dataBins); + } + } + + return result; + } + + private T transform(S dataBinFilter, DataBin dataBin) { + switch (dataBinFilter) { + case ClinicalDataBinFilter clinicalDataBinFilter -> { + return (T) dataBinToClinicalDataBin(clinicalDataBinFilter, dataBin); + } + case GenomicDataBinFilter genomicDataBinFilter -> { + return (T) dataBintoGenomicDataBin(genomicDataBinFilter, dataBin); + } + case GenericAssayDataBinFilter genericAssayDataBinFilter -> { + return (T) dataBintoGenericAssayDataBin(genericAssayDataBinFilter, dataBin); + } + default -> { + return null; + } + } + } + + private ClinicalDataBin dataBinToClinicalDataBin(ClinicalDataBinFilter attribute, DataBin dataBin) { + ClinicalDataBin clinicalDataBin = new ClinicalDataBin(); + clinicalDataBin.setAttributeId(attribute.getAttributeId()); + setCommonDataBinProperties(dataBin, clinicalDataBin); + return clinicalDataBin; + } + + private GenomicDataBin dataBintoGenomicDataBin(GenomicDataBinFilter genomicDataBinFilter, DataBin dataBin) { + GenomicDataBin genomicDataBin = new GenomicDataBin(); + genomicDataBin.setHugoGeneSymbol(genomicDataBinFilter.getHugoGeneSymbol()); + genomicDataBin.setProfileType(genomicDataBinFilter.getProfileType()); + setCommonDataBinProperties(dataBin, genomicDataBin); + return genomicDataBin; + } + + private GenericAssayDataBin dataBintoGenericAssayDataBin(GenericAssayDataBinFilter genericAssayDataBinFilter, + DataBin dataBin) { + GenericAssayDataBin genericAssayDataBin = new GenericAssayDataBin(); + genericAssayDataBin.setStableId(genericAssayDataBinFilter.getStableId()); + genericAssayDataBin.setProfileType(genericAssayDataBinFilter.getProfileType()); + setCommonDataBinProperties(dataBin, genericAssayDataBin); + return genericAssayDataBin; + } + + private void setCommonDataBinProperties(DataBin originalDataBin, U targetDatabin) { + targetDatabin.setCount(originalDataBin.getCount()); + if (originalDataBin.getSpecialValue() != null) { + targetDatabin.setSpecialValue(originalDataBin.getSpecialValue()); + } + if (originalDataBin.getStart() != null) { + targetDatabin.setStart(originalDataBin.getStart()); + } + if (originalDataBin.getEnd() != null) { + targetDatabin.setEnd(originalDataBin.getEnd()); + } + } + +} diff --git a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java index ffeff0ad7b3..fc55e05481b 100644 --- a/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java +++ b/src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java @@ -19,6 +19,8 @@ import org.cbioportal.model.ClinicalViolinPlotData; import org.cbioportal.model.CopyNumberCountByGene; import org.cbioportal.model.DensityPlotData; +import org.cbioportal.model.GenericAssayDataBin; +import org.cbioportal.model.GenomicDataBin; import org.cbioportal.model.GenomicDataCount; import org.cbioportal.model.PatientTreatmentReport; import org.cbioportal.model.Sample; @@ -34,6 +36,8 @@ import org.cbioportal.web.parameter.ClinicalDataCountFilter; import org.cbioportal.web.parameter.ClinicalDataFilter; import org.cbioportal.web.parameter.DataBinMethod; +import org.cbioportal.web.parameter.GenericAssayDataBinCountFilter; +import org.cbioportal.web.parameter.GenomicDataBinCountFilter; import org.cbioportal.web.parameter.GenomicDataCountFilter; import org.cbioportal.web.parameter.GenomicDataFilter; import org.cbioportal.web.parameter.MutationOption; @@ -70,6 +74,8 @@ public class StudyViewColumnStoreController { private final StudyViewColumnarService studyViewColumnarService; private final ClinicalDataBinner clinicalDataBinner; + + private final BasicDataBinner basicDataBinner; private final ClinicalDataDensityPlotService clinicalDataDensityPlotService; private final ViolinPlotService violinPlotService; @@ -79,11 +85,13 @@ public class StudyViewColumnStoreController { @Autowired public StudyViewColumnStoreController(StudyViewColumnarService studyViewColumnarService, ClinicalDataBinner clinicalDataBinner, + BasicDataBinner basicDataBinner, ClinicalDataDensityPlotService clinicalDataDensityPlotService, ViolinPlotService violinPlotService ) { this.studyViewColumnarService = studyViewColumnarService; this.clinicalDataBinner = clinicalDataBinner; + this.basicDataBinner = basicDataBinner; this.clinicalDataDensityPlotService = clinicalDataDensityPlotService; this.violinPlotService = violinPlotService; } @@ -524,4 +532,43 @@ public ResponseEntity fetchSampleTreatmentCounts( // // return new ResponseEntity<>(clinicalDataBins, HttpStatus.OK); // } + + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") + @PostMapping(value = "/column-store/genomic-data-bin-counts/fetch", + consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponse(responseCode = "200", description = "OK", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = GenomicDataBin.class)))) + public ResponseEntity> fetchGenomicDataBinCounts( + @RequestParam(defaultValue = "DYNAMIC") DataBinMethod dataBinMethod, + @RequestBody(required = false) GenomicDataBinCountFilter genomicDataBinCountFilter, + @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, + @RequestAttribute(required = false, value = "interceptedGenomicDataBinCountFilter") GenomicDataBinCountFilter interceptedGenomicDataBinCountFilter + ) { + List genomicDataBins = basicDataBinner.getDataBins( + dataBinMethod, + interceptedGenomicDataBinCountFilter, + true + ); + return new ResponseEntity<>(genomicDataBins, HttpStatus.OK); + } + + @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection', T(org.cbioportal.utils.security.AccessLevel).READ)") + @PostMapping(value = "/column-store/generic-assay-data-bin-counts/fetch", + consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponse(responseCode = "200", description = "OK", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = GenericAssayDataBin.class)))) + public ResponseEntity> fetchGenericAssayDataBinCounts( + @RequestParam(defaultValue = "DYNAMIC") DataBinMethod dataBinMethod, + @RequestBody(required = false) GenericAssayDataBinCountFilter genericAssayDataBinCountFilter, + @RequestAttribute(required = false, value = "involvedCancerStudies") Collection involvedCancerStudies, + @RequestAttribute(required = false, value = "interceptedGenericAssayDataBinCountFilter") GenericAssayDataBinCountFilter interceptedGenericAssayDataBinCountFilter + ) { + List genericAssayDataBins = basicDataBinner.getDataBins( + dataBinMethod, + interceptedGenericAssayDataBinCountFilter, + true + ); + return new ResponseEntity<>(genericAssayDataBins, HttpStatus.OK); + } + } diff --git a/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java b/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java index 734d9bb53e7..22ce588453e 100644 --- a/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java +++ b/src/main/java/org/cbioportal/web/parameter/CategorizedClinicalDataCountFilter.java @@ -11,6 +11,7 @@ public static Builder getBuilder() { private final List sampleCategoricalClinicalDataFilters; private final List patientNumericalClinicalDataFilters; private final List patientCategoricalClinicalDataFilters; + private CategorizedClinicalDataCountFilter(Builder builder) { this.sampleCategoricalClinicalDataFilters = builder.sampleCategoricalClinicalDataFilters; this.sampleNumericalClinicalDataFilters = builder.sampleNumericalClinicalDataFilters; @@ -33,8 +34,7 @@ public List getPatientNumericalClinicalDataFilters() { public List getPatientCategoricalClinicalDataFilters() { return patientCategoricalClinicalDataFilters; } - - + public static class Builder { private List sampleNumericalClinicalDataFilters; private List sampleCategoricalClinicalDataFilters; @@ -42,7 +42,7 @@ public static class Builder { private List patientCategoricalClinicalDataFilters; private Builder(){ - + } public Builder setSampleCategoricalClinicalDataFilters(List sampleCategoricalClinicalDataFilters) { this.sampleCategoricalClinicalDataFilters = sampleCategoricalClinicalDataFilters; diff --git a/src/main/java/org/cbioportal/web/parameter/CategorizedGenericAssayDataCountFilter.java b/src/main/java/org/cbioportal/web/parameter/CategorizedGenericAssayDataCountFilter.java new file mode 100644 index 00000000000..3d9face92c8 --- /dev/null +++ b/src/main/java/org/cbioportal/web/parameter/CategorizedGenericAssayDataCountFilter.java @@ -0,0 +1,72 @@ +package org.cbioportal.web.parameter; + +import java.util.List; + +public final class CategorizedGenericAssayDataCountFilter { + + public static Builder getBuilder() { + return new Builder(); + } + + private final List sampleNumericalGenericAssayDataFilters; + private final List sampleCategoricalGenericAssayDataFilters; + private final List patientNumericalGenericAssayDataFilters; + private final List patientCategoricalGenericAssayDataFilters; + private CategorizedGenericAssayDataCountFilter(Builder builder) { + this.sampleCategoricalGenericAssayDataFilters = builder.sampleCategoricalGenericAssayDataFilters; + this.sampleNumericalGenericAssayDataFilters = builder.sampleNumericalGenericAssayDataFilters; + this.patientCategoricalGenericAssayDataFilters = builder.patientCategoricalGenericAssayDataFilters; + this.patientNumericalGenericAssayDataFilters = builder.patientNumericalGenericAssayDataFilters; + } + + public List getSampleNumericalGenericAssayDataFilters() { + return sampleNumericalGenericAssayDataFilters; + } + + public List getSampleCategoricalGenericAssayDataFilters() { + return sampleCategoricalGenericAssayDataFilters; + } + + public List getPatientNumericalGenericAssayDataFilters() { + return patientNumericalGenericAssayDataFilters; + } + + public List getPatientCategoricalGenericAssayDataFilters() { + return patientCategoricalGenericAssayDataFilters; + } + + public static class Builder { + private List sampleNumericalGenericAssayDataFilters; + private List sampleCategoricalGenericAssayDataFilters; + private List patientNumericalGenericAssayDataFilters; + private List patientCategoricalGenericAssayDataFilters; + + private Builder(){ + + } + + public Builder setSampleCategoricalGenericAssayDataFilters(List sampleCategoricalGenericAssayDataFilters) { + this.sampleCategoricalGenericAssayDataFilters = sampleCategoricalGenericAssayDataFilters; + return this; + } + + public Builder setSampleNumericalGenericAssayDataFilters(List sampleNumericalGenericAssayDataFilters) { + this.sampleNumericalGenericAssayDataFilters = sampleNumericalGenericAssayDataFilters; + return this; + } + + public Builder setPatientCategoricalGenericAssayDataFilters(List patientCategoricalGenericAssayDataFilters) { + this.patientCategoricalGenericAssayDataFilters = patientCategoricalGenericAssayDataFilters; + return this; + } + + public Builder setPatientNumericalGenericAssayDataFilters(List patientNumericalGenericAssayDataFilters) { + this.patientNumericalGenericAssayDataFilters = patientNumericalGenericAssayDataFilters; + return this; + } + + public CategorizedGenericAssayDataCountFilter build() { + return new CategorizedGenericAssayDataCountFilter(this); + } + } +} diff --git a/src/main/java/org/cbioportal/web/parameter/CategorizedGenomicDataCountFilter.java b/src/main/java/org/cbioportal/web/parameter/CategorizedGenomicDataCountFilter.java new file mode 100644 index 00000000000..01e67c010fc --- /dev/null +++ b/src/main/java/org/cbioportal/web/parameter/CategorizedGenomicDataCountFilter.java @@ -0,0 +1,73 @@ +package org.cbioportal.web.parameter; + +import java.util.List; + +public final class CategorizedGenomicDataCountFilter { + + public static Builder getBuilder() { + return new Builder(); + } + + private final List sampleNumericalGenomicDataFilters; + private final List sampleCategoricalGenomicDataFilters; + private final List patientNumericalGenomicDataFilters; + private final List patientCategoricalGenomicDataFilters; + + private CategorizedGenomicDataCountFilter(Builder builder) { + this.sampleCategoricalGenomicDataFilters = builder.sampleCategoricalGenomicDataFilters; + this.sampleNumericalGenomicDataFilters = builder.sampleNumericalGenomicDataFilters; + this.patientCategoricalGenomicDataFilters = builder.patientCategoricalGenomicDataFilters; + this.patientNumericalGenomicDataFilters = builder.patientNumericalGenomicDataFilters; + } + + public List getSampleNumericalGenomicDataFilters() { + return sampleNumericalGenomicDataFilters; + } + + public List getSampleCategoricalGenomicDataFilters() { + return sampleCategoricalGenomicDataFilters; + } + + public List getPatientNumericalGenomicDataFilters() { + return patientNumericalGenomicDataFilters; + } + + public List getPatientCategoricalGenomicDataFilters() { + return patientCategoricalGenomicDataFilters; + } + + public static class Builder { + private List sampleNumericalGenomicDataFilters; + private List sampleCategoricalGenomicDataFilters; + private List patientNumericalGenomicDataFilters; + private List patientCategoricalGenomicDataFilters; + + private Builder(){ + + } + + public Builder setSampleCategoricalGenomicDataFilters(List sampleCategoricalGenomicDataFilters) { + this.sampleCategoricalGenomicDataFilters = sampleCategoricalGenomicDataFilters; + return this; + } + + public Builder setSampleNumericalGenomicDataFilters(List sampleNumericalGenomicDataFilters) { + this.sampleNumericalGenomicDataFilters = sampleNumericalGenomicDataFilters; + return this; + } + + public Builder setPatientCategoricalGenomicDataFilters(List patientCategoricalGenomicDataFilters) { + this.patientCategoricalGenomicDataFilters = patientCategoricalGenomicDataFilters; + return this; + } + + public Builder setPatientNumericalGenomicDataFilters(List patientNumericalGenomicDataFilters) { + this.patientNumericalGenomicDataFilters = patientNumericalGenomicDataFilters; + return this; + } + + public CategorizedGenomicDataCountFilter build() { + return new CategorizedGenomicDataCountFilter(this); + } + } +} diff --git a/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java b/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java index 0fb4e83dc2c..16e5483d6dd 100644 --- a/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java +++ b/src/main/java/org/cbioportal/web/util/StudyViewFilterApplier.java @@ -723,7 +723,6 @@ public List getUniqkeyKeys(List studyIds, List caseIds) public List getDataBins( DataBinMethod dataBinMethod, T dataBinCountFilter) { List dataBinFilters = fetchDataBinFilters(dataBinCountFilter); - StudyViewFilter studyViewFilter = dataBinCountFilter.getStudyViewFilter(); if (dataBinFilters.size() == 1) { @@ -733,6 +732,7 @@ public resultDataBins; List filteredSampleIds = new ArrayList<>(); List filteredStudyIds = new ArrayList<>(); + List filteredData = fetchData(dataBinCountFilter, studyViewFilter, filteredSampleIds, filteredStudyIds); @@ -740,7 +740,7 @@ public > filteredClinicalDataByAttributeId = filteredData.stream() .collect(Collectors.groupingBy(Binnable::getAttrId)); - + if (dataBinMethod == DataBinMethod.STATIC) { StudyViewFilter filter = studyViewFilter == null ? null : new StudyViewFilter(); diff --git a/src/main/resources/db-scripts/clickhouse/clickhouse.sql b/src/main/resources/db-scripts/clickhouse/clickhouse.sql index 9e89e8506f9..ffb18b1979d 100644 --- a/src/main/resources/db-scripts/clickhouse/clickhouse.sql +++ b/src/main/resources/db-scripts/clickhouse/clickhouse.sql @@ -323,10 +323,116 @@ FROM WHERE alteration_value != 'NA') AS subquery JOIN sample_derived sd ON sd.internal_id = subquery.sample_id; +CREATE TABLE IF NOT EXISTS genetic_alteration_numerical_derived +( + sample_unique_id String, + cancer_study_identifier LowCardinality(String), + hugo_gene_symbol String, + profile_type LowCardinality(String), + alteration_value String + ) + ENGINE = MergeTree() + ORDER BY (profile_type, cancer_study_identifier, hugo_gene_symbol, sample_unique_id ); + +INSERT INTO TABLE genetic_alteration_numerical_derived +SELECT + sample_unique_id, + cancer_study_identifier, + hugo_gene_symbol, + profile_type, + alteration_value +FROM + (SELECT + sample_id, + hugo_gene_symbol, + profile_type, + alteration_value + FROM + (SELECT + g.hugo_gene_symbol AS hugo_gene_symbol, + replaceOne(stable_id, concat(cs.cancer_study_identifier, '_'), '') as profile_type, -- Compute profile_type + arrayMap(x -> (x = '' ? NULL : x), splitByString(',', assumeNotNull(trim(trailing ',' from ga.values)))) AS alteration_value, + arrayMap(x -> (x = '' ? NULL : toInt32(x)), splitByString(',', assumeNotNull(trim(trailing ',' from gps.ordered_sample_list)))) AS sample_id + FROM + genetic_profile gp + JOIN cancer_study cs ON cs.cancer_study_id = gp.cancer_study_id + JOIN genetic_profile_samples gps ON gp.genetic_profile_id = gps.genetic_profile_id + JOIN genetic_alteration ga ON gp.genetic_profile_id = ga.genetic_profile_id + JOIN gene g ON ga.genetic_entity_id = g.genetic_entity_id + WHERE + gp.genetic_alteration_type != 'COPY_NUMBER_ALTERATION') + ARRAY JOIN alteration_value, sample_id + ) AS subquery + JOIN sample_derived sd ON sd.internal_id = subquery.sample_id; + +CREATE TABLE IF NOT EXISTS generic_assay_data_derived +( + sample_unique_id String, + genetic_entity_id String, + value String, + generic_assay_type String, + profile_stable_id String, + entity_stable_id String, + datatype String, + patient_level NUMERIC, + profile_type String +) + ENGINE = MergeTree() + ORDER BY (profile_type, entity_stable_id, sample_unique_id); + +INSERT INTO TABLE generic_assay_data_derived +SELECT + sd.sample_unique_id as sample_unique_id, + genetic_entity_id, + value, + generic_assay_type, + profile_stable_id, + entity_stable_id, + datatype, + patient_level, + replaceOne(profile_stable_id, concat(cs.cancer_study_identifier, '_'), '') as profile_type +FROM + (SELECT + sample_id, + genetic_entity_id, + value, + cancer_study_id, + generic_assay_type, + genetic_profile_id, + profile_stable_id, + entity_stable_id, + patient_level, + datatype + FROM + (SELECT + sample_id as sample_unique_id, + gp.cancer_study_id AS cancer_study_id, + ga.genetic_entity_id as genetic_entity_id, + gp.genetic_profile_id as genetic_profile_id, + gp.generic_assay_type as generic_assay_type, + gp.stable_id as profile_stable_id, + ge.stable_id as entity_stable_id, + gp.datatype as datatype, + gp.patient_level as patient_level, + arrayMap(x -> (x = '' ? NULL : x), splitByString(',', assumeNotNull(trim(trailing ',' from ga.values)))) AS value, + arrayMap(x -> (x = '' ? NULL : toInt64(x)), splitByString(',', assumeNotNull(trim(trailing ',' from gps.ordered_sample_list)))) AS sample_id + FROM genetic_profile gp + JOIN genetic_profile_samples gps ON gp.genetic_profile_id = gps.genetic_profile_id + JOIN genetic_alteration ga ON gp.genetic_profile_id = ga.genetic_profile_id + JOIN genetic_entity ge on ga.genetic_entity_id = ge.id + WHERE + gp.generic_assay_type IS NOT NULL + ) + ARRAY JOIN value, sample_id) AS subquery + JOIN cancer_study cs ON cs.cancer_study_id = subquery.cancer_study_id + JOIN sample_derived sd ON sd.internal_id = subquery.sample_id; + OPTIMIZE TABLE sample_to_gene_panel_derived; OPTIMIZE TABLE gene_panel_to_gene_derived; OPTIMIZE TABLE sample_derived; OPTIMIZE TABLE genomic_event_derived; OPTIMIZE TABLE clinical_data_derived; OPTIMIZE TABLE clinical_event_derived; -OPTIMIZE TABLE genetic_alteration_cna_derived; \ No newline at end of file +OPTIMIZE TABLE genetic_alteration_cna_derived; +OPTIMIZE TABLE genetic_alteration_numerical_derived; +OPTIMIZE TABLE generic_assay_data_derived; diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml index ca289ebc4b5..7c78f4db52e 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml @@ -131,18 +131,42 @@ - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -157,7 +181,7 @@ - + @@ -361,6 +385,131 @@ + + SELECT ${unique_id} + FROM ${table_name} + WHERE hugo_gene_symbol = '${genomicDataFilter.hugoGeneSymbol}' AND + profile_type='${genomicDataFilter.profileType}' + + + + AND + + + + + + AND match(alteration_value, '^>?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(alteration_value, '^<?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(alteration_value, '^[-+]?[0-9]*[.,]?[0-9]+$') + + + + + AND abs( + minus( + + + , + ${dataFilterValue.start} + ) + ) < exp(-11) + + + + AND + + + > ${dataFilterValue.start} + + + AND + + + <= ${dataFilterValue.end} + + + + + + + + + + + SELECT ${unique_id} + FROM ${table_name} + WHERE entity_stable_id = '${genericAssayDataFilter.stableId}' AND + profile_type='${genericAssayDataFilter.profileType}' + + + + AND + + + + + + AND match(value, '^>?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(value, '^<?=?[-+]?[0-9]*[.,]?[0-9]+$') + + + AND match(value, '^[-+]?[0-9]*[.,]?[0-9]+$') + + + + + AND abs( + minus( + + + , + ${dataFilterValue.start} + ) + ) < exp(-11) + + + + AND + + + > ${dataFilterValue.start} + + + AND + + + <= ${dataFilterValue.end} + + + + + + + + + + SELECT ${unique_id} + FROM ${table_name} + WHERE entity_stable_id = '${genericAssayDataFilter.stableId}' AND + profile_type='${genericAssayDataFilter.profileType}' + + + AND ( + + + + ) = '${dataFilterValue.value}' + + + + diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml index 2da2eff0f0b..ddd65175da7 100644 --- a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml +++ b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml @@ -551,6 +551,69 @@ GROUP BY treatments.treatment; + + + + + +