check for all possible NA values not just empty string

cBioPortal · Jun 25, 2024 · abbb255 · abbb255
1 parent 6bfba14
commit abbb255
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 11 deletions.
diff --git a/...resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewClinicalDataBinMapper.xml b/...resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewClinicalDataBinMapper.xml
@@ -39,7 +39,10 @@
     <select id="getPatientClinicalDataCountsForBinning">
         SELECT
             attribute_name as attributeId,
-            if(attribute_value='', 'NA', attribute_value) AS value,
+            <include refid="normalizeAttributeValueNA">
+                <property name="attribute_value" value="attribute_value"/>
+                <property name="as_value" value="value"/>
+            </include>,
             count(value) as count
         FROM clinical_data_derived
         <include refid="patientClinicalDataFromStudyViewFilter" />
@@ -50,7 +53,10 @@
     <select id="getSampleClinicalDataCountsForBinning">
         SELECT
             attribute_name as attributeId,
-            if(attribute_value='', 'NA', attribute_value) AS value,
+            <include refid="normalizeAttributeValueNA">
+                <property name="attribute_value" value="attribute_value"/>
+                <property name="as_value" value="value"/>
+            </include>,
             count(value) as count
         FROM clinical_data_derived
         <include refid="sampleClinicalDataFromStudyViewFilter" />

diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewFilterMapper.xml
@@ -140,7 +140,10 @@
         <foreach item="dataFilterValue" collection="clinicalDataFilter.values" open=" AND ((" separator=") OR (" close="))">
             <trim prefix="" prefixOverrides="AND">
                 <if test="dataFilterValue.value eq 'NA'">
-                    AND attribute_value = ''
+                    AND
+                    <include refid="isAttributeValueNA">
+                        <property name="attribute_value" value="attribute_value"/>
+                    </include>
                 </if>
                 <if test="dataFilterValue.start != null || dataFilterValue.end != null">
                     AND match(attribute_value, '^[\d\.]+$')

diff --git a/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml b/src/main/resources/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.xml
@@ -76,13 +76,9 @@
 
     <!-- for /clinical-data-counts/fetch (returns ClinicalData) which will then be converted to clinicalDataCountItems -->
     <select id="getClinicalDataCounts" resultType="org.cbioportal.model.ClinicalDataCount">
-        <include refid="getCategoricalClinicalDataCountsQuerySample">
-            <property name="table_name_prefix" value="sample"/>
-        </include>
+        <include refid="getCategoricalClinicalDataCountsQuerySample" />
         UNION ALL
-        <include refid="getCategoricalClinicalDataCountsQueryPatient">
-            <property name="table_name_prefix" value="patient"/>
-        </include>
+        <include refid="getCategoricalClinicalDataCountsQueryPatient" />
     </select>
 
 
@@ -122,7 +118,10 @@
     <sql id="getCategoricalClinicalDataCountsQuerySample">
         SELECT
         attribute_name as attributeId,
-        if(attribute_value='', 'NA', attribute_value) AS value,
+        <include refid="normalizeAttributeValueNA">
+            <property name="attribute_value" value="attribute_value"/>
+            <property name="as_value" value="value"/>
+        </include>,
         count(value) as count
         FROM clinical_data_derived
         <where>
@@ -152,7 +151,10 @@
     <sql id="getCategoricalClinicalDataCountsQueryPatient">
         SELECT
         attribute_name as attributeId,
-        if(attribute_value='', 'NA', attribute_value) AS value,
+        <include refid="normalizeAttributeValueNA">
+            <property name="attribute_value" value="attribute_value"/>
+            <property name="as_value" value="value"/>
+        </include>,
         count(value) as count
         FROM clinical_data_derived
         <where>
@@ -281,4 +283,19 @@
             AND patient_unique_id IN (<include refid="patientUniqueIdsFromStudyViewFilter"/>)
         </if> 
     </sql>
+
+    <sql id="isAttributeValueNA">
+        ${attribute_value}='' OR upperUTF8(${attribute_value})='NA' OR upperUTF8(${attribute_value})='NAN' OR upperUTF8(${attribute_value})='N/A'
+    </sql>
+
+    <sql id="normalizeAttributeValueNA">
+        if(
+          <include refid="isAttributeValueNA">
+              <property name="attribute_value" value="${attribute_value}"/>
+          </include>,
+          'NA',
+          ${attribute_value}
+        )
+        AS ${as_value}
+    </sql>
 </mapper>