analysis: improve QC trace for individuals taking into account its pa…

…rts (inferred sex, Mendelian errors and relatedness), #TASK-6772, #TASK-6766

opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java

@@ -52,6 +52,7 @@
 import java.util.stream.Collectors;
 
 import static org.opencb.opencga.core.models.common.InternalStatus.READY;
+import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING;
 import static org.opencb.opencga.core.models.common.QualityControlStatus.NONE;
 import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_FAMILIES;
 import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.JSON;
@@ -102,7 +103,8 @@ protected void run() throws ToolException {
 
                 // Set quality control status to COMPUTING to prevent multiple family QCs from running simultaneously
                 // for the same family
-                if (!setComputingStatus(family.getId(), FAMILY_QC_TYPE)) {
+                QualityControlStatus qcStatus = new QualityControlStatus(COMPUTING, "Performing " + FAMILY_QC_TYPE + " QC");
+                if (!setQualityControlStatus(qcStatus, family.getId(), FAMILY_QC_TYPE)) {
                     continue;
                 }
 
@@ -223,7 +225,6 @@ public static void checkParameters(FamilyQcAnalysisParams params, String studyId
     }
 
     private void updateFamilyQualityControl(List<Family> families) throws ToolException {
-        final String extension = ".qc.json";
         ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper();
         objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
         ObjectReader objectReader = JacksonUtils.getDefaultObjectMapper().readerFor(FamilyQualityControl.class);
@@ -234,21 +235,21 @@ private void updateFamilyQualityControl(List<Family> families) throws ToolExcept
 
             // Check output file
             String msg;
-            Path qcPath = getOutDir().resolve(family.getId()).resolve(family.getId() + extension);
+            Path qcPath = getOutDir().resolve(family.getId()).resolve(family.getId() + QC_JSON_EXTENSION);
             if (!Files.exists(qcPath)) {
-                msg = "Quality control error for family " + family.getId() + ": file " + qcPath.getFileName() + " not found";
+                msg = "Failure: file " + qcPath.getFileName() + " not found";
                 familyQc = new FamilyQualityControl();
                 qcStatus = new QualityControlStatus(NONE, msg);
                 addError(new ToolException(msg));
                 logger.error(msg);
             } else {
                 try {
-                    msg = "Computed successfully for family " + family.getId();
+                    msg = "Success";
                     familyQc = objectReader.readValue(qcPath.toFile());
                     qcStatus = new QualityControlStatus(READY, msg);
                     logger.info(msg);
                 } catch (IOException e) {
-                    msg = "Quality control error for family " + family.getId() + ": error parsing JSON file " + qcPath.getFileName();
+                    msg = "Failure: error parsing JSON file " + qcPath.getFileName();
                     familyQc = new FamilyQualityControl();
                     qcStatus = new QualityControlStatus(NONE, msg);
                     addError(e);

opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java

@@ -23,6 +23,9 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.opencb.biodata.models.clinical.qc.InferredSexReport;
+import org.opencb.biodata.models.clinical.qc.MendelianErrorReport;
+import org.opencb.biodata.models.clinical.qc.RelatednessReport;
 import org.opencb.biodata.models.variant.avro.VariantType;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
@@ -36,6 +39,7 @@
 import org.opencb.opencga.core.models.family.FamilyQualityControl;
 import org.opencb.opencga.core.models.individual.Individual;
 import org.opencb.opencga.core.models.individual.IndividualQualityControl;
+import org.opencb.opencga.core.models.individual.IndividualQualityControlStatus;
 import org.opencb.opencga.core.models.individual.IndividualUpdateParams;
 import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams;
 import org.opencb.opencga.core.response.OpenCGAResult;
@@ -45,16 +49,18 @@
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 import java.util.stream.Collectors;
 
-import static org.opencb.opencga.analysis.utils.VariantQcAnalysisExecutorUtils.QC_JSON_EXTENSION;
 import static org.opencb.opencga.core.models.common.InternalStatus.READY;
+import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING;
 import static org.opencb.opencga.core.models.common.QualityControlStatus.NONE;
+import static org.opencb.opencga.core.models.individual.IndividualQualityControlStatus.*;
 import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_INDIVIDUALS;
 import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.JSON;
 import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.VCF_GZ;
@@ -117,7 +123,9 @@ protected void run() throws ToolException {
 
                 // Set quality control status to COMPUTING to prevent multiple individual QCs from running simultaneously
                 // for the same individual
-                if (!setComputingStatus(individual.getId(), INDIVIDUAL_QC_TYPE)) {
+                IndividualQualityControlStatus qcStatus = new IndividualQualityControlStatus(COMPUTING,
+                        "Performing " + INDIVIDUAL_QC_TYPE + " QC");
+                if (!setQualityControlStatus(qcStatus, individual.getId(), INDIVIDUAL_QC_TYPE)) {
                     continue;
                 }
 
@@ -202,46 +210,87 @@ protected void run() throws ToolException {
     private void updateIndividualQualityControl(List<Individual> individuals) throws ToolException {
         ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper();
         objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
-        ObjectReader objectReader = JacksonUtils.getDefaultObjectMapper().readerFor(FamilyQualityControl.class);
+        ObjectReader inferredSexReportReader = JacksonUtils.getDefaultObjectMapper().readerFor(InferredSexReport.class);
+        ObjectReader mendelianErrorReportReader = JacksonUtils.getDefaultObjectMapper().readerFor(MendelianErrorReport.class);
+        ObjectReader relatednessReportReader = JacksonUtils.getDefaultObjectMapper().readerFor(RelatednessReport.class);
 
         for (Individual individual : individuals) {
-            IndividualQualityControl individualQc;
-            QualityControlStatus qcStatus;
-
-            // Check output file
+            // Check output files
             String msg;
-            Path qcPath = getOutDir().resolve(individual.getId()).resolve(individual.getId() + QC_JSON_EXTENSION);
+            Path qcPath = getOutDir().resolve(individual.getId());
             if (!Files.exists(qcPath)) {
-                msg = "Quality control error for individual " + individual.getId() + ": file " + qcPath.getFileName() + " not found";
-                individualQc = new IndividualQualityControl();
-                qcStatus = new QualityControlStatus(NONE, msg);
-                addError(new ToolException(msg));
+                msg = "Quality control error for individual " + individual.getId() + ": folder " + qcPath + " not found."
+                        + " None quality control was performed.";
                 logger.error(msg);
+                addError(new ToolException(msg));
             } else {
-                try {
-                    msg = "Computed successfully for individual " + individual.getId();
-                    individualQc = objectReader.readValue(qcPath.toFile());
-                    qcStatus = new QualityControlStatus(READY, msg);
-                    logger.info(msg);
-                } catch (IOException e) {
-                    msg = "Quality control error for individual " + individual.getId() + ": error parsing JSON file " + qcPath.getFileName();
-                    individualQc = new IndividualQualityControl();
-                    qcStatus = new QualityControlStatus(NONE, msg);
-                    addError(e);
-                    logger.error(msg);
+                int qcCode = NONE_READY;
+                IndividualQualityControl individualQc = individual.getQualityControl();
+
+                // Check inferred sex analysis
+                if (!analysisParams.getSkip().contains(INFERRED_SEX_ANALYSIS_ID)) {
+                    File qcFile = qcPath.resolve(INFERRED_SEX_ANALYSIS_ID).resolve(individual.getId() + QC_JSON_EXTENSION).toFile();
+                    try {
+                        InferredSexReport inferredSexReport = inferredSexReportReader.readValue(qcFile);
+                        if (inferredSexReport != null) {
+                            qcCode |= INFERRED_SEX_READY;
+                            individualQc.getInferredSexReports().add(inferredSexReport);
+                        }
+                    } catch (IOException e) {
+                        msg = "Failure: error parsing inferred sex report (JSON file: " + qcFile.getName() + " )";
+                        logger.error(msg, e);
+                        addError(new ToolException(msg, e));
+                    }
+                }
+
+                // Check Mendelian error analysis
+                if (!analysisParams.getSkip().contains(MENDELIAN_ERROR_ANALYSIS_ID)) {
+                    File qcFile = qcPath.resolve(MENDELIAN_ERROR_ANALYSIS_ID).resolve(individual.getId() + QC_JSON_EXTENSION).toFile();
+                    try {
+                        MendelianErrorReport mendelianErrorReport = mendelianErrorReportReader.readValue(qcFile);
+                        if (mendelianErrorReport != null) {
+                            qcCode |= MENDELIAN_ERROR_READY;
+                            individualQc.getMendelianErrorReports().add(mendelianErrorReport);
+                        }
+                    } catch (IOException e) {
+                        msg = "Failure: error parsing Mendelian error report (JSON file: " + qcFile.getName() + " )";
+                        logger.error(msg, e);
+                        addError(new ToolException(msg, e));
+                    }
+                }
+
+                // Check relatedness analysis
+                if (!analysisParams.getSkip().contains(RELATEDNESS_ANALYSIS_ID)) {
+                    File qcFile = qcPath.resolve(RELATEDNESS_ANALYSIS_ID).resolve(individual.getId() + QC_JSON_EXTENSION).toFile();
+                    try {
+                        RelatednessReport relatednessReport = relatednessReportReader.readValue(qcFile);
+                        if (relatednessReport != null) {
+                            qcCode |= RELATEDNESS_READY;
+                            // individualQc.getRelatednessReports().add(relatednessReport);
+                            individualQc.setSampleRelatednessReport(null);
+                        }
+                    } catch (IOException e) {
+                        msg = "Failure: error parsing relatedness report (JSON file: " + qcFile.getName() + " )";
+                        logger.error(msg, e);
+                        addError(new ToolException(msg, e));
+                    }
                 }
-            }
 
-            try {
                 // Update catalog: quality control and status
-                IndividualUpdateParams updateParams = new IndividualUpdateParams()
-                        .setQualityControl(individualQc)
-                        .setQualityControlStatus(qcStatus);
-                catalogManager.getIndividualManager().update(getStudy(), individual.getId(), updateParams, null, token);
-            } catch (CatalogException e) {
-                logger.error("Could not update quality control in OpenCGA catalog for individual {}: {}", individual.getId(),
-                        e.getMessage());
-                addError(e);
+                if (qcCode != NONE_READY) {
+                    // Update the individual QC code with the current one
+                    IndividualQualityControlStatus qcStatus = new IndividualQualityControlStatus(
+                            qcCode | individual.getInternal().getQualityControlStatus().getCode(), "");
+                    try {
+                        IndividualUpdateParams updateParams = new IndividualUpdateParams()
+                                .setQualityControl(individualQc)
+                                .setQualityControlStatus(qcStatus);
+                        catalogManager.getIndividualManager().update(getStudy(), individual.getId(), updateParams, null, token);
+                    } catch (CatalogException e) {
+                        logger.error("Could not update quality control in OpenCGA catalog for individual " + individual.getId(), e);
+                        addError(e);
+                    }
+                }
             }
         }
     }
@@ -272,9 +321,8 @@ public static void checkParameters(IndividualQcAnalysisParams params, String stu
 
                     // Check number of samples
                     List<String> sampleIds = getNoSomaticSampleIds(individual);
-                    if (sampleIds.size() < 1) {
-                        errors.put(individualId, "Too few samples found (" + sampleIds.size() + ") for that individual; minimum is 1"
-                                + " sample");
+                    if (CollectionUtils.isEmpty(sampleIds)) {
+                        errors.put(individualId, "No samples found");
                     }
                 }
             } catch (CatalogException e) {

opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java

@@ -6,7 +6,6 @@
 import org.opencb.biodata.models.core.SexOntologyTermAnnotation;
 import org.opencb.biodata.models.pedigree.IndividualProperty;
 import org.opencb.opencga.core.models.common.AnnotationSet;
-import org.opencb.opencga.core.models.common.QualityControlStatus;
 import org.opencb.opencga.core.models.common.StatusParams;
 import org.opencb.opencga.core.models.individual.*;
 import org.opencb.opencga.core.models.sample.SampleReferenceParam;
@@ -28,7 +27,7 @@ public IndividualPrivateUpdateParams(String id, String name, IndividualReference
                                          IndividualProperty.LifeStatus lifeStatus, List<SampleReferenceParam> samples,
                                          List<AnnotationSet> annotationSets, List<Phenotype> phenotypes, List<Disorder> disorders,
                                          StatusParams status, IndividualQualityControl qualityControl,
-                                         QualityControlStatus qualityControlStatus, Map<String, Object> attributes,
+                                         IndividualQualityControlStatus qualityControlStatus, Map<String, Object> attributes,
                                          IndividualInternal internal) {
         super(id, name, father, mother, creationDate, modificationDate, parentalConsanguinity, location, sex, ethnicity, population,
                 dateOfBirth, karyotypicSex, lifeStatus, samples, annotationSets, phenotypes, disorders, status, qualityControl,

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java

@@ -35,6 +35,7 @@
 import org.opencb.opencga.core.models.family.FamilyUpdateParams;
 import org.opencb.opencga.core.models.file.File;
 import org.opencb.opencga.core.models.individual.Individual;
+import org.opencb.opencga.core.models.individual.IndividualQualityControlStatus;
 import org.opencb.opencga.core.models.individual.IndividualUpdateParams;
 import org.opencb.opencga.core.models.sample.Sample;
 import org.opencb.opencga.core.models.study.Study;
@@ -66,6 +67,8 @@ public class VariantQcAnalysis extends OpenCgaToolScopeStudy {
     public static final String RESOURCES_FOLDER = "resources/";
     public static final String QC_RESOURCES_FOLDER = QC_FOLDER + RESOURCES_FOLDER;
 
+    public static final String QC_JSON_EXTENSION = ".qc.json";
+
     // Data type
     public static final String FAMILY_QC_TYPE = "family";
     public static final String INDIVIDUAL_QC_TYPE = "individual";
@@ -86,7 +89,7 @@ public class VariantQcAnalysis extends OpenCgaToolScopeStudy {
     protected static final String INFERRED_SEX_THRESHOLDS_FILE_MSG = "Karyotypic sex thresholds file";
 
     // For mendelian errors sex analysis
-    public static final String MENDELIAN_ERRORS_ANALYSIS_ID = "mendelian-errors";
+    public static final String MENDELIAN_ERROR_ANALYSIS_ID = "mendelian-errors";
 
     @Override
     protected void check() throws Exception {
@@ -231,17 +234,17 @@ protected static Path checkFileParameter(String fileId, String msg, String study
         return path;
     }
 
-    protected boolean setComputingStatus(String id, String qcType) throws ToolException {
+    protected boolean setQualityControlStatus(QualityControlStatus qcStatus, String id, String qcType) throws ToolException {
         try {
-            QualityControlStatus qcStatus = new QualityControlStatus(COMPUTING, "Performing " + qcType + " QC");
             switch (qcType) {
                 case FAMILY_QC_TYPE: {
                     FamilyUpdateParams updateParams = new FamilyUpdateParams().setQualityControlStatus(qcStatus);
                     catalogManager.getFamilyManager().update(getStudy(), id, updateParams, null, token);
                     break;
                 }
                 case INDIVIDUAL_QC_TYPE: {
-                    IndividualUpdateParams updateParams = new IndividualUpdateParams().setQualityControlStatus(qcStatus);
+                    IndividualUpdateParams updateParams = new IndividualUpdateParams()
+                            .setQualityControlStatus((IndividualQualityControlStatus) qcStatus);
                     catalogManager.getIndividualManager().update(getStudy(), id, updateParams, null, token);
                     break;
                 }

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java

@@ -20,7 +20,6 @@
 import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction;
 import org.opencb.opencga.client.exceptions.ClientException;
 import org.opencb.opencga.core.common.JacksonUtils;
-import org.opencb.opencga.core.models.common.QualityControlStatus;
 import org.opencb.opencga.core.models.common.StatusParams;
 import org.opencb.opencga.core.models.common.TsvAnnotationParams;
 import org.opencb.opencga.core.models.individual.Individual;
@@ -29,6 +28,7 @@
 import org.opencb.opencga.core.models.individual.IndividualCreateParams;
 import org.opencb.opencga.core.models.individual.IndividualPopulation;
 import org.opencb.opencga.core.models.individual.IndividualQualityControl;
+import org.opencb.opencga.core.models.individual.IndividualQualityControlStatus;
 import org.opencb.opencga.core.models.individual.IndividualReferenceParam;
 import org.opencb.opencga.core.models.individual.IndividualUpdateParams;
 import org.opencb.opencga.core.models.individual.Location;
@@ -454,6 +454,7 @@ private RestResponse<Individual> update() throws Exception {
             putNestedIfNotEmpty(beanParams, "qualityControlStatus.date",commandOptions.qualityControlStatusDate, true);
             putNestedIfNotEmpty(beanParams, "qualityControlStatus.version",commandOptions.qualityControlStatusVersion, true);
             putNestedIfNotEmpty(beanParams, "qualityControlStatus.commit",commandOptions.qualityControlStatusCommit, true);
+            putNestedIfNotNull(beanParams, "qualityControlStatus.code",commandOptions.qualityControlStatusCode, true);
             putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true);
 
             individualUpdateParams = JacksonUtils.getDefaultObjectMapper().copy()