fetch missing patient clinical data for violin plot (#11092)

src/main/java/org/cbioportal/web/columnar/StudyViewColumnStoreController.java

@@ -73,6 +73,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.cbioportal.web.columnar.util.ClinicalDataViolinPlotUtil.convertPatientClinicalDataToSampleClinicalData;
+import static org.cbioportal.web.columnar.util.ClinicalDataViolinPlotUtil.filterNonEmptyClinicalData;
 
 @InternalApi
 @RestController()
@@ -349,17 +353,26 @@ public ResponseEntity<ClinicalViolinPlotData> fetchClinicalDataViolinPlots(
                 .ifPresent(f->interceptedStudyViewFilter.getClinicalDataFilters().remove(f));
         }
 
+        List<String> attributeIds = List.of(numericalAttributeId, categoricalAttributeId);
+
         // Filter out clinical data with empty attribute values due to Clickhouse migration
-        List<ClinicalData> sampleClinicalDataList = studyViewColumnarService.getSampleClinicalData(interceptedStudyViewFilter, List.of(numericalAttributeId, categoricalAttributeId))
-            .stream()
-            .filter(clinicalData -> !clinicalData.getAttrValue().isEmpty())
-            .toList();
+        List<ClinicalData> sampleClinicalDataList = filterNonEmptyClinicalData(
+            studyViewColumnarService.getSampleClinicalData(interceptedStudyViewFilter, attributeIds)
+        );
+        List<ClinicalData> patientClinicalDataList = filterNonEmptyClinicalData(
+            studyViewColumnarService.getPatientClinicalData(interceptedStudyViewFilter, attributeIds)
+        );
+
+        List<ClinicalData> combinedClinicalDataList = Stream.concat(
+            sampleClinicalDataList.stream(),
+            convertPatientClinicalDataToSampleClinicalData(patientClinicalDataList, filteredSamples).stream()
+        ).toList();
 
         // Only mutation count can use log scale
         boolean useLogScale = logScale && numericalAttributeId.equals("MUTATION_COUNT");
 
         ClinicalViolinPlotData result = violinPlotService.getClinicalViolinPlotData(
-            sampleClinicalDataList,
+            combinedClinicalDataList,
             filteredSamples,
             axisStart,
             axisEnd,
@@ -370,7 +383,7 @@ public ResponseEntity<ClinicalViolinPlotData> fetchClinicalDataViolinPlots(
 
         return new ResponseEntity<>(result, HttpStatus.OK);
     }
-
+        
     @Hidden
     @PreAuthorize("hasPermission(#involvedCancerStudies, 'Collection<CancerStudyId>', T(org.cbioportal.utils.security.AccessLevel).READ)")
     @PostMapping(value = "/column-store/genomic-data-counts/fetch", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

src/main/java/org/cbioportal/web/columnar/util/ClinicalDataViolinPlotUtil.java

@@ -0,0 +1,58 @@
+package org.cbioportal.web.columnar.util;
+
+import org.cbioportal.model.ClinicalData;
+import org.cbioportal.model.Sample;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class ClinicalDataViolinPlotUtil {
+    public static List<ClinicalData> filterNonEmptyClinicalData(List<ClinicalData> clinicalData) {
+        return clinicalData
+            .stream()
+            .filter(data -> !data.getAttrValue().isEmpty())
+            .toList();
+    }
+
+    public static List<ClinicalData> convertPatientClinicalDataToSampleClinicalData(
+        List<ClinicalData> patientClinicalDataList,
+        List<Sample> filteredSamples
+    ) {
+        List<ClinicalData> sampleClinicalDataList = new ArrayList<>();
+
+        Map<String, Map<String, List<Sample>>> patientToSamples = filteredSamples
+            .stream()
+            .collect(Collectors.groupingBy(
+                s -> s.getCancerStudyIdentifier() + "_" + s.getPatientStableId(),
+                Collectors.groupingBy(Sample::getCancerStudyIdentifier)
+            ));
+
+        // put all clinical data into sample form
+        for (ClinicalData d: patientClinicalDataList) {
+            List<Sample> samplesForPatient = patientToSamples.get(d.getPatientId()).get(d.getStudyId());
+            if (samplesForPatient != null) {
+                for (Sample s: samplesForPatient) {
+                    ClinicalData newData = new ClinicalData();
+
+                    newData.setInternalId(s.getInternalId());
+                    newData.setAttrId(d.getAttrId());
+                    newData.setPatientId(d.getPatientId());
+                    newData.setStudyId(d.getStudyId());
+                    newData.setAttrValue(d.getAttrValue());
+                    newData.setSampleId(s.getCancerStudyIdentifier() + "_" + s.getStableId());
+
+                    sampleClinicalDataList.add(newData);
+                }
+            } else {
+                // TODO ignoring for now rather than throwing an error
+                // patient has no samples - this shouldn't happen and could affect the integrity
+                //  of the data analysis
+                // return new ResponseEntity<>(null, HttpStatus.INTERNAL_SERVER_ERROR);
+            }
+        }
+
+        return sampleClinicalDataList;
+    }
+}