diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java index b368fe4a56..64a3e6f7d1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java @@ -17,12 +17,14 @@ package org.opencb.opencga.analysis.family.qc; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.clinical.qc.Relatedness; import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -34,6 +36,7 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.family.Family; +import org.opencb.opencga.core.models.family.FamilyInternal; import org.opencb.opencga.core.models.family.FamilyQualityControl; import org.opencb.opencga.core.models.family.FamilyUpdateParams; import org.opencb.opencga.core.models.variant.FamilyQcAnalysisParams; @@ -52,8 +55,7 @@ import java.util.stream.Collectors; import static org.opencb.opencga.core.models.common.InternalStatus.READY; -import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING; -import static org.opencb.opencga.core.models.common.QualityControlStatus.NONE; +import static org.opencb.opencga.core.models.common.QualityControlStatus.ERROR; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_FAMILIES; import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.JSON; import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.VCF_GZ; @@ -68,23 +70,43 @@ public class FamilyVariantQcAnalysis extends VariantQcAnalysis { @ToolParams protected final FamilyQcAnalysisParams analysisParams = new FamilyQcAnalysisParams(); + private List families = new ArrayList<>(); + @Override protected void check() throws Exception { + setUpStorageEngineExecutor(study); + + // Check parameters super.check(); checkParameters(analysisParams, getStudy(), catalogManager, token); - // Prepare relatedness resource files - prepareRelatednessResources(analysisParams.getResourcesDir()); + // Check custom resources path + userResourcesPath = checkResourcesDir(analysisParams.getResourcesDir(), getStudy(), catalogManager, token); + } + + @Override + protected List getSteps() { + List steps = Arrays.asList(PREPARE_QC_STEP, ID); + if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { + steps.add(INDEX_QC_STEP); + } + return steps; } @Override protected void run() throws ToolException { - setUpStorageEngineExecutor(study); + // Main steps + step(PREPARE_QC_STEP, this::prepareQualityControl); + step(ID, this::runFamilyQc); + if (getSteps().contains(INDEX_QC_STEP)) { + step(INDEX_QC_STEP, this::indexQualityControl); + } - List families = new ArrayList<>(); - LinkedList familyVcfPaths = new LinkedList<>(); - LinkedList familyJsonPaths = new LinkedList<>(); + // Clean execution + clean(); + } + protected void prepareQualityControl() throws ToolException { try { ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(Family.class); for (String familyId : analysisParams.getFamilies()) { @@ -92,92 +114,133 @@ protected void run() throws ToolException { OpenCGAResult familyResult = catalogManager.getFamilyManager().get(study, familyId, QueryOptions.empty(), token); Family family = familyResult.first(); - // Decide if quality control has to be performed - // - by checking the quality control status, if it is READY means it has been computed previously, and - // - by checking the flag overwrite - if (family.getInternal() == null || mustPerformQualityControl(family.getInternal().getQualityControlStatus(), - analysisParams.getOverwrite())) { - // Set quality control status to COMPUTING to prevent multiple family QCs from running simultaneously - // for the same family - QualityControlStatus qcStatus = new QualityControlStatus(COMPUTING, "Performing " + FAMILY_QC_TYPE + " QC"); - if (!setQualityControlStatus(qcStatus, family.getId(), FAMILY_QC_TYPE)) { - continue; - } - - // Create directory to save variants and family - Path famOutPath = Files.createDirectories(getOutDir().resolve(familyId)); - if (!Files.exists(famOutPath)) { - throw new ToolException("Error creating directory: " + famOutPath); - } - - // Export family variants (VCF format) - // Create variant query - String gt = getNoSomaticSampleIds(family, study, catalogManager, token).stream().map(s -> s + ":0/0,0/1,1/1") - .collect(Collectors.joining(";")); - Query query = new Query() - .append(VariantQueryParam.STUDY.key(), study) - .append(VariantQueryParam.TYPE.key(), VariantType.SNV) - .append(VariantQueryParam.GENOTYPE.key(), gt) - .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" - .split(","))); - - // Create query options - QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); - - // Export to VCF.GZ format - String basename = famOutPath.resolve(familyId).toAbsolutePath().toString(); - getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); - - // Check VCF file - Path familyVcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); - if (!Files.exists(familyVcfPath)) { - throw new ToolException("Something wrong happened when exporting VCF file for family ID " + familyId + ". VCF file " - + familyVcfPath + " was not created. Export query = " + query.toJson() + "; export query options = " - + queryOptions.toJson()); - } - familyVcfPaths.add(familyVcfPath); + // Add family to the list + families.add(family); + + // Export family variants (VCF format) + // Create variant query + String gt = getNoSomaticSampleIds(family, study, catalogManager, token).stream().map(s -> s + ":0/0,0/1,1/1") + .collect(Collectors.joining(";")); + Query query = new Query() + .append(VariantQueryParam.STUDY.key(), study) + .append(VariantQueryParam.TYPE.key(), VariantType.SNV) + .append(VariantQueryParam.GENOTYPE.key(), gt) + .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" + .split(","))); + + // Create query options + QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); + + // Export to VCF.GZ format + String basename = getOutDir().resolve(familyId).toAbsolutePath().toString(); + getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); + + // Check VCF file + Path vcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); + if (!Files.exists(vcfPath)) { + throw new ToolException("Something wrong happened when exporting VCF file for family ID " + familyId + ". VCF file " + + vcfPath + " was not created. Export query = " + query.toJson() + "; export query options = " + + queryOptions.toJson()); + } + vcfPaths.add(vcfPath); - // Export family (JSON format) - Path familyJsonPath = Paths.get(basename + "." + JSON.getExtension()); - objectWriter.writeValue(familyJsonPath.toFile(), family); + // Export family (JSON format) + Path jsonPath = Paths.get(basename + "." + JSON.getExtension()); + objectWriter.writeValue(jsonPath.toFile(), family); - // Check VCF file - if (!Files.exists(familyJsonPath)) { - throw new ToolException("Something wrong happened when saving JSON file for family ID " + familyId + ". JSON file " - + familyJsonPath + " was not created."); - } - familyJsonPaths.add(familyJsonPath); - - // Add family to the list - families.add(family); + // Check VCF file + if (!Files.exists(jsonPath)) { + throw new ToolException("Something wrong happened when saving JSON file for family ID " + familyId + ". JSON file " + + jsonPath + " was not created."); } + jsonPaths.add(jsonPath); } } catch (CatalogException | IOException | StorageEngineException e) { + // Clean execution + clean(); throw new ToolException(e); } + // Prepare resource files + prepareResources(); + } + + protected void runFamilyQc() throws ToolException { // Get executor to execute Python script that computes the family QC FamilyVariantQcAnalysisExecutor executor = getToolExecutor(FamilyVariantQcAnalysisExecutor.class); - executor.setVcfPaths(familyVcfPaths) - .setJsonPaths(familyJsonPaths) - .setQcParams(analysisParams); + executor.setVcfPaths(vcfPaths) + .setJsonPaths(jsonPaths) + .setQcParams(analysisParams) + .execute(); + } - // Step by step - step(executor::execute); + protected void indexQualityControl() throws ToolException { + ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ObjectReader relatednessReader = JacksonUtils.getDefaultObjectMapper().readerFor(Relatedness.class); + ObjectReader relatednessListReader = JacksonUtils.getDefaultObjectMapper().readerFor(new TypeReference>() {}); - // Parse Python script results - if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { - updateFamilyQualityControl(families); + FamilyQualityControl familyQc; + QualityControlStatus qcStatus; + String logMsg; + + failedQcSet = new HashSet<>(); + for (Family family : families) { + familyQc = new FamilyQualityControl(); + + // Check and parse the relatedness output file + Path qcPath = getOutDir().resolve(family.getId()).resolve(RELATEDNESS_ANALYSIS_ID + QC_JSON_EXTENSION); + if (!Files.exists(qcPath)) { + failedQcSet.add(family.getId()); + qcStatus = new QualityControlStatus(ERROR, FAILURE_FILE + qcPath.getFileName() + NOT_FOUND); + + logMsg = qcStatus.getDescription() + getIdLogMessage(family.getId(), FAMILY_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } else { + try { + List relatedness = isQcArray(qcPath) + ? relatednessListReader.readValue(qcPath.toFile()) + : Collections.singletonList(relatednessReader.readValue(qcPath.toFile())); + + familyQc.setRelatedness(relatedness); + qcStatus = new QualityControlStatus(READY, SUCCESS); + } catch (IOException e) { + failedQcSet.add(family.getId()); + qcStatus = new QualityControlStatus(ERROR, FAILURE_ERROR_PARSING_QC_JSON_FILE + qcPath.getFileName() + "'"); + familyQc = new FamilyQualityControl(); + + logMsg = qcStatus.getDescription() + getIdLogMessage(family.getId(), FAMILY_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } + } + + try { + // Update catalog: quality control and status + FamilyUpdateParams updateParams = new FamilyUpdateParams() + .setQualityControl(familyQc) + .setQualityControlStatus(qcStatus); + catalogManager.getFamilyManager().update(getStudy(), family.getId(), updateParams, null, token); + } catch (CatalogException e) { + failedQcSet.add(family.getId()); + logMsg = FAILURE_COULD_NOT_UPDATE_QUALITY_CONTROL_IN_OPEN_CGA_CATALOG + getIdLogMessage(family.getId(), FAMILY_QC_TYPE); + addError(e); + logger.error(logMsg, e); + } } + + checkFailedQcCounter(families.size(), FAMILY_QC_TYPE); } + public static void checkParameters(FamilyQcAnalysisParams params, String studyId, CatalogManager catalogManager, String token) throws ToolException { // Check study checkStudy(studyId, catalogManager, token); // Check permissions - checkPermissions(WRITE_FAMILIES, studyId, catalogManager, token); + checkPermissions(WRITE_FAMILIES, params.getSkipIndex(), studyId, catalogManager, token); // Sanity check if (CollectionUtils.isEmpty(params.getFamilies())) { @@ -189,7 +252,7 @@ public static void checkParameters(FamilyQcAnalysisParams params, String studyId Map errors = new HashMap<>(); for (String familyId : params.getFamilies()) { // Get family from catalog - Family family; + Family family = null; try { OpenCGAResult familyResult = catalogManager.getFamilyManager().get(studyId, familyId, QueryOptions.empty(), token); if (familyResult.getNumResults() == 0) { @@ -206,6 +269,18 @@ public static void checkParameters(FamilyQcAnalysisParams params, String studyId + " samples"); } } + + // Check compatibility between QC status (READY) and overwrite + if (!Boolean.TRUE.equals(params.getSkipIndex()) && !Boolean.TRUE.equals(params.getOverwrite()) + && Optional.ofNullable(family) + .map(Family::getInternal) + .map(FamilyInternal::getQualityControlStatus) + .map(QualityControlStatus::getId) + .filter(READY::equals) + .isPresent()) { + errors.put(familyId, "It is mandatory to set 'overwrite' to TRUE when QC has already been computed" + + " (status: READY)"); + } } catch (CatalogException e) { errors.put(familyId, Arrays.toString(e.getStackTrace())); } @@ -220,50 +295,4 @@ public static void checkParameters(FamilyQcAnalysisParams params, String studyId // Check resources dir checkResourcesDir(params.getResourcesDir(), studyId, catalogManager, token); } - - private void updateFamilyQualityControl(List families) throws ToolException { - ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper(); - objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - ObjectReader objectReader = JacksonUtils.getDefaultObjectMapper().readerFor(FamilyQualityControl.class); - - for (Family family : families) { - FamilyQualityControl familyQc; - QualityControlStatus qcStatus; - - // Check output file - String msg; - Path qcPath = getOutDir().resolve(family.getId()).resolve(family.getId() + QC_JSON_EXTENSION); - if (!Files.exists(qcPath)) { - msg = "Failure: file " + qcPath.getFileName() + " not found"; - familyQc = new FamilyQualityControl(); - qcStatus = new QualityControlStatus(NONE, msg); - addError(new ToolException(msg)); - logger.error(msg); - } else { - try { - msg = "Success"; - familyQc = objectReader.readValue(qcPath.toFile()); - qcStatus = new QualityControlStatus(READY, msg); - logger.info(msg); - } catch (IOException e) { - msg = "Failure: error parsing JSON file " + qcPath.getFileName(); - familyQc = new FamilyQualityControl(); - qcStatus = new QualityControlStatus(NONE, msg); - addError(e); - logger.error(msg); - } - } - - try { - // Update catalog: quality control and status - FamilyUpdateParams updateParams = new FamilyUpdateParams() - .setQualityControl(familyQc) - .setQualityControlStatus(qcStatus); - catalogManager.getFamilyManager().update(getStudy(), family.getId(), updateParams, null, token); - } catch (CatalogException e) { - logger.error("Could not update quality control in OpenCGA catalog for family {}: {}", family.getId(), e.getMessage()); - addError(e); - } - } - } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java index 81566a4e2c..55f98dcdce 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.individual.qc; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; @@ -35,9 +36,10 @@ import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.IndividualQualityControl; -import org.opencb.opencga.core.models.individual.IndividualQualityControlStatus; import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; @@ -54,8 +56,8 @@ import java.util.*; import java.util.stream.Collectors; -import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING; -import static org.opencb.opencga.core.models.individual.IndividualQualityControlStatus.*; +import static org.opencb.opencga.core.models.common.InternalStatus.READY; +import static org.opencb.opencga.core.models.common.QualityControlStatus.ERROR; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_INDIVIDUALS; import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.JSON; import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.VCF_GZ; @@ -67,14 +69,20 @@ public class IndividualVariantQcAnalysis extends VariantQcAnalysis { public static final String DESCRIPTION = "Run quality control (QC) for a given individual. This includes inferred sex, Mendelian" + " errors (UDP), and, if parents are present, a relatedness analysis is also performed"; + @ToolParams + protected final IndividualQcAnalysisParams analysisParams = new IndividualQcAnalysisParams(); + + // Individuals to perform QC and VCF and JSON files + private List individuals = new ArrayList<>(); + // Set of individuals with known parents, i.e., trios private Set trios = new HashSet<>(); - @ToolParams - protected final IndividualQcAnalysisParams analysisParams = new IndividualQcAnalysisParams(); @Override protected void check() throws Exception { + setUpStorageEngineExecutor(study); + super.check(); checkParameters(analysisParams, getStudy(), catalogManager, token); @@ -86,189 +94,247 @@ protected void check() throws Exception { trios.add(individual); } } - if (CollectionUtils.isNotEmpty(trios)) { - prepareRelatednessResources(analysisParams.getResourcesDir()); - } - // Prepare inferred sex resource files - prepareInferredSexResources(analysisParams.getResourcesDir()); + // Check custom resources path + userResourcesPath = checkResourcesDir(analysisParams.getResourcesDir(), getStudy(), catalogManager, token); + } + + @Override + protected List getSteps() { + List steps = Arrays.asList(PREPARE_QC_STEP, ID); + if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { + steps.add(INDEX_QC_STEP); + } + return steps; } @Override protected void run() throws ToolException { - setUpStorageEngineExecutor(study); + // Main steps + step(PREPARE_QC_STEP, this::prepareQualityControl); + step(ID, this::runIndividualQc); + if (getSteps().contains(INDEX_QC_STEP)) { + step(INDEX_QC_STEP, this::indexQualityControl); + } - List individuals = new ArrayList<>(); - LinkedList individualVcfPaths = new LinkedList<>(); - LinkedList individualJsonPaths = new LinkedList<>(); + // Clean execution + clean(); + } + protected void prepareQualityControl() throws ToolException { try { ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(Individual.class); for (String individualId : analysisParams.getIndividuals()) { // Get individual Individual individual = catalogManager.getIndividualManager().get(study, individualId, QueryOptions.empty(), token).first(); - // Decide if quality control has to be performed - // - by checking the quality control status, if it is READY means it has been computed previously, and - // - by checking the flag overwrite - if (individual.getInternal() == null || mustPerformQualityControl(individual.getInternal().getQualityControlStatus(), - analysisParams.getOverwrite())) { - - // Set quality control status to COMPUTING to prevent multiple individual QCs from running simultaneously - // for the same individual - IndividualQualityControlStatus qcStatus = new IndividualQualityControlStatus(COMPUTING, - "Performing " + INDIVIDUAL_QC_TYPE + " QC"); - if (!setQualityControlStatus(qcStatus, individual.getId(), INDIVIDUAL_QC_TYPE)) { - continue; - } - - // Create directory to save variants and individual files - Path individualOutPath = Files.createDirectories(getOutDir().resolve(individualId)); - if (!Files.exists(individualOutPath)) { - throw new ToolException("Error creating directory: " + individualOutPath); - } - - // Export individual variants (VCF format) - // Create the query based on whether a trio is present or not - Query query; - if (trios.contains(individual)) { - String child = individual.getSamples().get(0).getId(); - String father = individual.getFather().getSamples().get(0).getId(); - String mother = individual.getMother().getSamples().get(0).getId(); - query = new Query() - .append(VariantQueryParam.SAMPLE.key(), child + ":0/1,1/1") - .append(VariantQueryParam.INCLUDE_SAMPLE.key(), child + "," + father + "," + mother) - .append(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), "GT"); - } else { - // Create variant query - String gt = getNoSomaticSampleIds(individual).stream().map(s -> s + ":0/0,0/1,1/1") - .collect(Collectors.joining(";")); - query = new Query() - .append(VariantQueryParam.GENOTYPE.key(), gt); - } - query.append(VariantQueryParam.STUDY.key(), study) - .append(VariantQueryParam.TYPE.key(), VariantType.SNV) - .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" - .split(","))); - - // Create query options - QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); - - // Export to VCF.GZ format - String basename = individualOutPath.resolve(individualId).toAbsolutePath().toString(); - getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); - - // Check VCF file - Path individualVcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); - if (!Files.exists(individualVcfPath)) { - throw new ToolException("Something wrong happened when exporting VCF file for individual ID '" + individualId - + "'. VCF file " + individualVcfPath + " was not created. Export query = " + query.toJson() + "; export query" - + " options = " + queryOptions.toJson()); - } - individualVcfPaths.add(individualVcfPath); - - // Export individual (JSON format) - Path individualJsonPath = Paths.get(basename + "." + JSON.getExtension()); - objectWriter.writeValue(individualJsonPath.toFile(), individual); + // Add family to the list + individuals.add(individual); + + // Export individual variants (VCF format) + // Create the query based on whether a trio is present or not + Query query; + if (trios.contains(individual)) { + String child = individual.getSamples().get(0).getId(); + String father = individual.getFather().getSamples().get(0).getId(); + String mother = individual.getMother().getSamples().get(0).getId(); + query = new Query() + .append(VariantQueryParam.SAMPLE.key(), child + ":0/1,1/1") + .append(VariantQueryParam.INCLUDE_SAMPLE.key(), child + "," + father + "," + mother) + .append(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), "GT"); + } else { + // Create variant query + String gt = getNoSomaticSampleIds(individual).stream().map(s -> s + ":0/0,0/1,1/1") + .collect(Collectors.joining(";")); + query = new Query() + .append(VariantQueryParam.GENOTYPE.key(), gt); + } + query.append(VariantQueryParam.STUDY.key(), study) + .append(VariantQueryParam.TYPE.key(), VariantType.SNV) + .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" + .split(","))); + + // Create query options + QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); + + // Export to VCF.GZ format + String basename = getOutDir().resolve(individualId).toAbsolutePath().toString(); + getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); + + // Check VCF file + Path vcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); + if (!Files.exists(vcfPath)) { + throw new ToolException("Something wrong happened when exporting VCF file for individual ID '" + individualId + + "'. VCF file " + vcfPath + " was not created. Export query = " + query.toJson() + "; export query" + + " options = " + queryOptions.toJson()); + } + vcfPaths.add(vcfPath); - // Check VCF file - if (!Files.exists(individualJsonPath)) { - throw new ToolException("Something wrong happened when saving JSON file for individual ID '" + individualId - + "'. JSON file " + individualJsonPath + " was not created."); - } - individualJsonPaths.add(individualJsonPath); + // Export individual (JSON format) + Path jsonPath = Paths.get(basename + "." + JSON.getExtension()); + objectWriter.writeValue(jsonPath.toFile(), individual); - // Add family to the list - individuals.add(individual); + // Check VCF file + if (!Files.exists(jsonPath)) { + throw new ToolException("Something wrong happened when saving JSON file for individual ID '" + individualId + + "'. JSON file " + jsonPath + " was not created."); } + jsonPaths.add(jsonPath); } } catch (CatalogException | IOException | StorageEngineException e) { + // Clean execution + clean(); throw new ToolException(e); } + // Prepare resource files + prepareResources(); + } + + protected void runIndividualQc() throws ToolException { // Get executor to execute Python script that computes the family QC IndividualVariantQcAnalysisExecutor executor = getToolExecutor(IndividualVariantQcAnalysisExecutor.class); - executor.setVcfPaths(individualVcfPaths) - .setJsonPaths(individualJsonPaths) - .setQcParams(analysisParams); - - // Step by step - step(executor::execute); - - // Parse Python script results - if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { - updateIndividualQualityControl(individuals); - } + executor.setVcfPaths(vcfPaths) + .setJsonPaths(jsonPaths) + .setQcParams(analysisParams) + .execute(); } - /** - * Update quality control for each individual by parsing the QC report for the inferred sex, Mendelian errors and relatedness analyses. - * - * @param individuals List of individuals - * @throws ToolException Tool exception - */ - private void updateIndividualQualityControl(List individuals) throws ToolException { + private void indexQualityControl() throws ToolException { // Create readers for each QC report ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper(); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); ObjectReader inferredSexReader = JacksonUtils.getDefaultObjectMapper().readerFor(InferredSex.class); + ObjectReader inferredSexListReader = JacksonUtils.getDefaultObjectMapper().readerFor(new TypeReference>() {}); ObjectReader mendelianErrorReader = JacksonUtils.getDefaultObjectMapper().readerFor(MendelianError.class); + ObjectReader mendelianErrorListReader = JacksonUtils.getDefaultObjectMapper().readerFor( + new TypeReference>() {}); ObjectReader relatednessReader = JacksonUtils.getDefaultObjectMapper().readerFor(Relatedness.class); + ObjectReader relatednessListReader = JacksonUtils.getDefaultObjectMapper().readerFor(new TypeReference>() {}); + + QualityControlStatus qcStatus; + IndividualQualityControl individualQc; + Set failedAnalysis = new HashSet<>(); + String logMsg; + failedQcSet = new HashSet<>(); for (Individual individual : individuals) { - // Check output files - Path qcPath = getOutDir().resolve(individual.getId()); + individualQc = new IndividualQualityControl(); + + // Check and parse the inferred sex results + Path qcPath = getOutDir().resolve(individual.getId()).resolve(INFERRED_SEX_ANALYSIS_ID) + .resolve(INFERRED_SEX_ANALYSIS_ID + QC_JSON_EXTENSION); if (!Files.exists(qcPath)) { - String msg = "Quality control error for individual " + individual.getId() + ": folder " + qcPath + " not found." - + " None quality control was performed."; - logger.error(msg); - addError(new ToolException(msg)); - continue; + failedQcSet.add(individual.getId()); + failedAnalysis.add(INFERRED_SEX_ANALYSIS_ID); + individualQc.setInferredSex(Collections.emptyList()); + + logMsg = FAILURE_FILE + qcPath.getFileName() + NOT_FOUND + getIdLogMessage(individual.getId(), INDIVIDUAL_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } else { + try { + List inferredSexList = isQcArray(qcPath) + ? inferredSexListReader.readValue(qcPath.toFile()) + : Collections.singletonList(inferredSexReader.readValue(qcPath.toFile())); + + individualQc.setInferredSex(inferredSexList); + } catch (IOException e) { + failedQcSet.add(individual.getId()); + failedAnalysis.add(INFERRED_SEX_ANALYSIS_ID); + individualQc.setInferredSex(Collections.emptyList()); + + logMsg = FAILURE_ERROR_PARSING_QC_JSON_FILE + qcPath.getFileName() + "'" + getIdLogMessage(individual.getId(), + INDIVIDUAL_QC_TYPE); + addError(e); + logger.error(logMsg, e); + } } - int qcCode = NONE_READY; - IndividualQualityControl individualQc = individual.getQualityControl(); - - // Check inferred sex report - InferredSex inferredSex = checkQcReport(individual.getId(), INFERRED_SEX_ANALYSIS_ID, analysisParams.getSkip(), qcPath, - INDIVIDUAL_QC_TYPE, inferredSexReader); - if (inferredSex != null) { - qcCode |= INFERRED_SEX_READY; - individualQc.getInferredSex().add(inferredSex); + // Check and parse the mendelian error results + qcPath = getOutDir().resolve(individual.getId()).resolve(MENDELIAN_ERROR_ANALYSIS_ID) + .resolve(MENDELIAN_ERROR_ANALYSIS_ID + QC_JSON_EXTENSION); + if (!Files.exists(qcPath)) { + failedQcSet.add(individual.getId()); + failedAnalysis.add(MENDELIAN_ERROR_ANALYSIS_ID); + individualQc.setInferredSex(Collections.emptyList()); + + logMsg = FAILURE_FILE + qcPath.getFileName() + NOT_FOUND + getIdLogMessage(individual.getId(), INDIVIDUAL_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } else { + try { + List mendelianErrorList = isQcArray(qcPath) + ? mendelianErrorListReader.readValue(qcPath.toFile()) + : Collections.singletonList(mendelianErrorReader.readValue(qcPath.toFile())); + + individualQc.setMendelianError(mendelianErrorList); + } catch (IOException e) { + failedQcSet.add(individual.getId()); + failedAnalysis.add(MENDELIAN_ERROR_ANALYSIS_ID); + individualQc.setMendelianError(Collections.emptyList()); + + logMsg = FAILURE_ERROR_PARSING_QC_JSON_FILE + qcPath.getFileName() + "'" + getIdLogMessage(individual.getId(), + INDIVIDUAL_QC_TYPE); + addError(e); + logger.error(logMsg, e); + } } - // Check Mendelian error report - MendelianError mendelianError = checkQcReport(individual.getId(), MENDELIAN_ERROR_ANALYSIS_ID, analysisParams.getSkip(), qcPath, - INDIVIDUAL_QC_TYPE, mendelianErrorReader); - if (mendelianError != null) { - qcCode |= MENDELIAN_ERROR_READY; - individualQc.getMendelianError().add(mendelianError); + // Check and parse the relatedness results, if trio is present + if (CollectionUtils.isNotEmpty(trios) && trios.contains(individual)) { + qcPath = getOutDir().resolve(individual.getId()).resolve(RELATEDNESS_ANALYSIS_ID) + .resolve(RELATEDNESS_ANALYSIS_ID + QC_JSON_EXTENSION); + if (!Files.exists(qcPath)) { + failedQcSet.add(individual.getId()); + failedAnalysis.add(RELATEDNESS_ANALYSIS_ID); + individualQc.setRelatedness(Collections.emptyList()); + + logMsg = FAILURE_FILE + qcPath.getFileName() + NOT_FOUND + getIdLogMessage(individual.getId(), INDIVIDUAL_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } else { + try { + List relatednessList = isQcArray(qcPath) + ? relatednessListReader.readValue(qcPath.toFile()) + : Collections.singletonList(relatednessReader.readValue(qcPath.toFile())); + + individualQc.setRelatedness(relatednessList); + } catch (IOException e) { + failedQcSet.add(individual.getId()); + failedAnalysis.add(RELATEDNESS_ANALYSIS_ID); + individualQc.setRelatedness(Collections.emptyList()); + + logMsg = FAILURE_ERROR_PARSING_QC_JSON_FILE + qcPath.getFileName() + "'" + getIdLogMessage(individual.getId(), + INDIVIDUAL_QC_TYPE); + addError(e); + logger.error(logMsg, e); + } + } } - // Check relatedness report - Relatedness relatedness = checkQcReport(individual.getId(), RELATEDNESS_ANALYSIS_ID, analysisParams.getSkip(), qcPath, - INDIVIDUAL_QC_TYPE, relatednessReader); - if (relatedness != null) { - qcCode |= RELATEDNESS_READY; - individualQc.getRelatedness().add(relatedness); + if (CollectionUtils.isEmpty(failedAnalysis)) { + qcStatus = new QualityControlStatus(READY, SUCCESS); + } else { + qcStatus = new QualityControlStatus(ERROR, "Failed analysis: " + StringUtils.join(failedAnalysis, ",")); } - // Update catalog (quality control and status) if necessary - if (qcCode != NONE_READY) { - // Update the individual QC code with the current one - IndividualQualityControlStatus qcStatus = new IndividualQualityControlStatus( - qcCode | individual.getInternal().getQualityControlStatus().getCode(), ""); - try { - IndividualUpdateParams updateParams = new IndividualUpdateParams() - .setQualityControl(individualQc) - .setQualityControlStatus(qcStatus); - catalogManager.getIndividualManager().update(getStudy(), individual.getId(), updateParams, null, token); - } catch (CatalogException e) { - logger.error("Could not update quality control in OpenCGA catalog for individual " + individual.getId(), e); - addError(e); - } + try { + // Update catalog: quality control and status + IndividualUpdateParams updateParams = new IndividualUpdateParams() + .setQualityControl(individualQc) + .setQualityControlStatus(qcStatus); + catalogManager.getIndividualManager().update(getStudy(), individual.getId(), updateParams, null, token); + } catch (CatalogException e) { + failedQcSet.add(individual.getId()); + logMsg = FAILURE_COULD_NOT_UPDATE_QUALITY_CONTROL_IN_OPEN_CGA_CATALOG + getIdLogMessage(individual.getId(), + INDIVIDUAL_QC_TYPE); + addError(e); + logger.error(logMsg, e); } } + + checkFailedQcCounter(individuals.size(), INDIVIDUAL_QC_TYPE); } /** @@ -287,14 +353,18 @@ public static void checkParameters(IndividualQcAnalysisParams params, String stu checkStudy(studyId, catalogManager, token); // Check permissions - checkPermissions(WRITE_INDIVIDUALS, studyId, catalogManager, token); + checkPermissions(WRITE_INDIVIDUALS, params.getSkipIndex(), studyId, catalogManager, token); + + if (!Boolean.TRUE.equals(params.getSkipIndex()) && CollectionUtils.isNotEmpty(params.getSkipAnalysis())) { + throw new ToolException("To index QC in the OpenCGA catalog, the 'skip analysis' parameter must be left empty"); + } // Check individuals in catalog // Collect possible errors in a map where key is the family ID and value is the error Map errors = new HashMap<>(); for (String individualId : params.getIndividuals()) { // Get Individual from catalog - Individual individual; + Individual individual = null; try { OpenCGAResult individualResult = catalogManager.getIndividualManager().get(studyId, individualId, QueryOptions.empty(), token); @@ -311,6 +381,18 @@ public static void checkParameters(IndividualQcAnalysisParams params, String stu errors.put(individualId, "No samples found"); } } + + // Check compatibility between QC status (READY) and overwrite + if (!Boolean.TRUE.equals(params.getSkipIndex()) && !Boolean.TRUE.equals(params.getOverwrite()) + && Optional.ofNullable(individual) + .map(Individual::getInternal) + .map(IndividualInternal::getQualityControlStatus) + .map(QualityControlStatus::getId) + .filter(READY::equals) + .isPresent()) { + errors.put(individualId, "It is mandatory to set 'overwrite' to TRUE when QC has already been computed" + + " (status: READY)"); + } } catch (CatalogException e) { errors.put(individualId, Arrays.toString(e.getStackTrace())); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java index fc75802d45..d40725f610 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java @@ -6,6 +6,7 @@ import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.sample.SampleReferenceParam; @@ -27,7 +28,7 @@ public IndividualPrivateUpdateParams(String id, String name, IndividualReference IndividualProperty.LifeStatus lifeStatus, List samples, List annotationSets, List phenotypes, List disorders, StatusParams status, IndividualQualityControl qualityControl, - IndividualQualityControlStatus qualityControlStatus, Map attributes, + QualityControlStatus qualityControlStatus, Map attributes, IndividualInternal internal) { super(id, name, father, mother, creationDate, modificationDate, parentalConsanguinity, location, sex, ethnicity, population, dateOfBirth, karyotypicSex, lifeStatus, samples, annotationSets, phenotypes, disorders, status, qualityControl, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/SamplePrivateUpdateParams.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/SamplePrivateUpdateParams.java index 2940ca1091..80814e5c7f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/SamplePrivateUpdateParams.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/SamplePrivateUpdateParams.java @@ -3,6 +3,7 @@ import org.opencb.biodata.models.clinical.Phenotype; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.common.ExternalSource; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.sample.*; @@ -18,7 +19,7 @@ public SamplePrivateUpdateParams() { public SamplePrivateUpdateParams(String id, String description, String creationDate, String modificationDate, String individualId, ExternalSource source, SampleProcessing processing, SampleCollection collection, - SampleQualityControl qualityControl, SampleQualityControlStatus qualityControlStatus, Boolean somatic, + SampleQualityControl qualityControl, QualityControlStatus qualityControlStatus, Boolean somatic, List phenotypes, List annotationSets, Map attributes, StatusParams status, SampleInternal internal) { super(id, description, creationDate, modificationDate, individualId, source, processing, collection, qualityControl, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleVariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleVariantQcAnalysis.java index d1c5d69450..f52532a32f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleVariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleVariantQcAnalysis.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.sample.qc; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; @@ -24,7 +25,6 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.qc.GenomePlot; -import org.opencb.biodata.models.clinical.qc.HRDetect; import org.opencb.biodata.models.clinical.qc.Signature; import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.commons.datastore.core.Query; @@ -37,9 +37,9 @@ import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleQualityControl; -import org.opencb.opencga.core.models.sample.SampleQualityControlStatus; import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.variant.SampleQcAnalysisParams; import org.opencb.opencga.core.models.variant.SampleVariantStatsAnalysisParams; @@ -57,8 +57,8 @@ import java.util.*; import java.util.stream.Collectors; -import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING; -import static org.opencb.opencga.core.models.sample.SampleQualityControlStatus.*; +import static org.opencb.opencga.core.models.common.InternalStatus.READY; +import static org.opencb.opencga.core.models.common.QualityControlStatus.ERROR; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_SAMPLES; import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.JSON; import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.VCF_GZ; @@ -73,22 +73,22 @@ public class SampleVariantQcAnalysis extends VariantQcAnalysis { private static final String STATS_QC_STEP = SampleVariantStatsAnalysis.ID + "-qc"; private static final String SOMATIC_QC_STEP = "somatic-qc"; - // Samples to perform QC and VCF and JSON files for these samples - private List targetSamples = new ArrayList<>(); + @ToolParams + protected final SampleQcAnalysisParams analysisParams = new SampleQcAnalysisParams(); - // Set of somatic samples + // Samples + private List samples = new ArrayList<>(); + + // List of somatic samples, for these we need to create VCF and JSON files private LinkedList targetSomaticSamples = new LinkedList<>(); - private LinkedList sampleVcfPaths = new LinkedList<>(); - private LinkedList sampleJsonPaths = new LinkedList<>(); - @ToolParams - protected final SampleQcAnalysisParams analysisParams = new SampleQcAnalysisParams(); + Map> failedAnalysisPerSample = new HashMap<>(); @Override protected void check() throws Exception { - super.check(); setUpStorageEngineExecutor(study); + super.check(); checkParameters(analysisParams, getStudy(), catalogManager, token); // Save the somatic samples @@ -99,123 +99,128 @@ protected void check() throws Exception { targetSomaticSamples.add(sample); } } + + // Check custom resources path + userResourcesPath = checkResourcesDir(analysisParams.getResourcesDir(), getStudy(), catalogManager, token); } @Override protected List getSteps() { - if (Boolean.TRUE.equals(analysisParams.getSkipIndex())) { - return Arrays.asList(STATS_QC_STEP, SOMATIC_QC_STEP); + List steps = Arrays.asList(PREPARE_QC_STEP); + if (CollectionUtils.isEmpty(analysisParams.getSkipAnalysis())) { + steps.addAll(Arrays.asList(STATS_QC_STEP, SOMATIC_QC_STEP)); } else { - return Arrays.asList(PREPARE_QC_STEP, STATS_QC_STEP, SOMATIC_QC_STEP, INDEX_QC_STEP); + if (!analysisParams.getSkipAnalysis().contains(SampleVariantStatsAnalysis.ID)) { + steps.add(STATS_QC_STEP); + } + if (!analysisParams.getSkipAnalysis().contains(SIGNATURE_ANALYSIS_ID) + || !analysisParams.getSkipAnalysis().contains(GENOME_PLOT_ANALYSIS_ID)) { + steps.add(SOMATIC_QC_STEP); + } } + if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { + steps.add(INDEX_QC_STEP); + } + return steps; } @Override protected void run() throws ToolException { - if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { - step(STATS_QC_STEP, this::prepareQualityControl); + List steps = getSteps(); + + // Main steps + step(PREPARE_QC_STEP, this::prepareQualityControl); + if (steps.contains(STATS_QC_STEP)) { + step(STATS_QC_STEP, this::runSampleVariantStatsStep); } - step(STATS_QC_STEP, this::runSampleVariantStatsStep); - step(SOMATIC_QC_STEP, this::runSomaticStep); - if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { + if (steps.contains(SOMATIC_QC_STEP)) { + step(SOMATIC_QC_STEP, this::runSomaticStep); + } + if (steps.contains(INDEX_QC_STEP)) { step(STATS_QC_STEP, this::indexQualityControl); } + + // Clean execution + clean(); } protected void prepareQualityControl() throws ToolException { - // Prepare resource files - if (CollectionUtils.isNotEmpty(targetSomaticSamples)) { - // Prepare resource files for somatic samples - //prepareSignatureResources(analysisParams.getResourcesDir()); - //prepareHrResources(analysisParams.getResourcesDir()); - //prepareGenomePlotResources(analysisParams.getResourcesDir()); - } - try { ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(Sample.class); for (String sampleId : analysisParams.getSamples()) { // Get sample Sample sample = catalogManager.getSampleManager().get(study, sampleId, QueryOptions.empty(), token).first(); - // Decide if quality control has to be performed - // - by checking the quality control status, if it is READY means it has been computed previously, and - // - by checking the flag overwrite - if (sample.getInternal() == null || mustPerformQualityControl(sample.getInternal().getQualityControlStatus(), - analysisParams.getOverwrite())) { - - // Set quality control status to COMPUTING to prevent multiple individual QCs from running simultaneously - // for the same individual - SampleQualityControlStatus qcStatus = new SampleQualityControlStatus(COMPUTING, - "Performing " + SAMPLE_QC_TYPE + " QC"); - if (!setQualityControlStatus(qcStatus, sample.getId(), SAMPLE_QC_TYPE)) { - continue; + // Add sample to target sample list, i.e., for those samples, QC will be performed + samples.add(sample); + failedAnalysisPerSample.put(sample.getId(), new HashSet<>()); + + // Only somatic samples need VCF and JSON files to compute signature, genome plot,... + if (sample.isSomatic()) { + targetSomaticSamples.add(sample); + + // Export sample variants (VCF format) + // Create the query based on whether a trio is present or not + Query query = new Query(); + query.append(VariantQueryParam.SAMPLE.key(), sampleId + ":0/1,1/1") + .append(VariantQueryParam.INCLUDE_SAMPLE.key(), sampleId) + .append(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), "GT") + .append(VariantQueryParam.STUDY.key(), study) + .append(VariantQueryParam.TYPE.key(), VariantType.SNV) + .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" + .split(","))); + + // Create query options + QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); + + // Export to VCF.GZ format + String basename = getOutDir().resolve(sampleId).toAbsolutePath().toString(); + getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); + + // Check VCF file + Path vcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); + if (!Files.exists(vcfPath)) { + throw new ToolException("Something wrong happened when exporting VCF file for sample ID '" + sampleId + "'. VCF" + + " file " + vcfPath + " was not created. Export query = " + query.toJson() + "; export query" + + " options = " + queryOptions.toJson()); } + vcfPaths.add(vcfPath); + + // Export individual (JSON format) + Path jsonPath = Paths.get(basename + "." + JSON.getExtension()); + objectWriter.writeValue(jsonPath.toFile(), sample); - // Add sample to target sample list, i.e., for those samples, QC will be performed - targetSamples.add(sample); - - // Only somatic samples need VCF and JSON files to compute signature, genome plot,... - if (sample.isSomatic()) { - targetSomaticSamples.add(sample); - - // Create directory to save variants and sample files - Path sampleOutPath = checkDirectory(getOutDir().resolve(sampleId)); - - // Export sample variants (VCF format) - // Create the query based on whether a trio is present or not - Query query = new Query(); - query.append(VariantQueryParam.SAMPLE.key(), sampleId + ":0/1,1/1") - .append(VariantQueryParam.INCLUDE_SAMPLE.key(), sampleId) - .append(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), "GT") - .append(VariantQueryParam.STUDY.key(), study) - .append(VariantQueryParam.TYPE.key(), VariantType.SNV) - .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" - .split(","))); - - // Create query options - QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); - - // Export to VCF.GZ format - String basename = sampleOutPath.resolve(sampleId).toAbsolutePath().toString(); - getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); - - // Check VCF file - Path sampleVcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); - if (!Files.exists(sampleVcfPath)) { - throw new ToolException("Something wrong happened when exporting VCF file for sample ID '" + sampleId + "'. VCF" - + " file " + sampleVcfPath + " was not created. Export query = " + query.toJson() + "; export query" - + " options = " + queryOptions.toJson()); - } - sampleVcfPaths.add(sampleVcfPath); - - // Export individual (JSON format) - Path sampleJsonPath = Paths.get(basename + "." + JSON.getExtension()); - objectWriter.writeValue(sampleJsonPath.toFile(), sample); - - // Check sample JSON file - if (!Files.exists(sampleJsonPath)) { - throw new ToolException("Something wrong happened when saving JSON file for sample ID '" + sampleId + "'. JSON" - + " file " + sampleJsonPath + " was not created."); - } - sampleJsonPaths.add(sampleJsonPath); + // Check sample JSON file + if (!Files.exists(jsonPath)) { + throw new ToolException("Something wrong happened when saving JSON file for sample ID '" + sampleId + "'. JSON" + + " file " + jsonPath + " was not created."); } + jsonPaths.add(jsonPath); } } } catch (CatalogException | IOException | StorageEngineException e) { + // Clean execution + clean(); throw new ToolException(e); } + + // Prepare resource files + if (CollectionUtils.isNotEmpty(targetSomaticSamples)) { + prepareResources(); + } } protected void runSampleVariantStatsStep() throws ToolException { - if (CollectionUtils.isEmpty(targetSamples)) { + if (CollectionUtils.isEmpty(samples)) { addWarning("No samples available for quality control. Please check your input parameters."); return; } + boolean callToolRunner; ToolRunner toolRunner = new ToolRunner(getOpencgaHome().toAbsolutePath().toString(), getCatalogManager(), getVariantStorageManager()); - for (Sample sample : targetSamples) { + for (Sample sample : samples) { SampleVariantStatsAnalysisParams statsParams = new SampleVariantStatsAnalysisParams() .setSample(Collections.singletonList(sample.getId())) .setIndex(!Boolean.TRUE.equals(analysisParams.getSkipIndex())) @@ -224,8 +229,38 @@ protected void runSampleVariantStatsStep() throws ToolException { .setIndexOverwrite(analysisParams.getOverwrite()) .setVariantQuery(new SampleVariantStatsAnalysisParams.VariantQueryParams(analysisParams.getVsQuery().toQuery())); - Path outDir = checkDirectory(getOutDir().resolve(sample.getId()).resolve(SampleVariantStatsAnalysis.ID)); - toolRunner.execute(SampleVariantStatsAnalysis.class, getStudy(), statsParams, outDir, null, false, getToken()); + callToolRunner = true; + + // Check the sample variant stats folder, and create + Path statsOutDir = getOutDir().resolve(sample.getId()).resolve(SampleVariantStatsAnalysis.ID); + if (!Files.exists(statsOutDir)) { + try { + Files.createDirectories(statsOutDir); + if (!Files.exists(statsOutDir)) { + failedAnalysisPerSample.get(sample.getId()).add(SampleVariantStatsAnalysis.ID); + String msg = "Could not create the sample variant stats folder for sample '" + sample.getId() + "'"; + addError(new ToolException(msg)); + logger.error(msg); + callToolRunner = false; + } + } catch (IOException e) { + failedQcSet.add(sample.getId()); + failedAnalysisPerSample.get(sample.getId()).add(SampleVariantStatsAnalysis.ID); + addError(e); + logger.error("Creating sample variant stats folder for sample '" + sample.getId() + "'", e); + callToolRunner = false; + } + } + if (callToolRunner) { + try { + toolRunner.execute(SampleVariantStatsAnalysis.class, getStudy(), statsParams, statsOutDir, null, false, getToken()); + } catch (ToolException e) { + failedQcSet.add(sample.getId()); + failedAnalysisPerSample.get(sample.getId()).add(SampleVariantStatsAnalysis.ID); + addError(e); + logger.error("Error running sample variant stats for sample '" + sample.getId() + "'", e); + } + } } } @@ -240,8 +275,8 @@ protected void runSomaticStep() throws ToolException { // Get executor to execute Python script that computes the family QC SampleVariantQcAnalysisExecutor executor = getToolExecutor(SampleVariantQcAnalysisExecutor.class); - executor.setVcfPaths(sampleVcfPaths) - .setJsonPaths(sampleJsonPaths) + executor.setVcfPaths(vcfPaths) + .setJsonPaths(jsonPaths) .setQcParams(analysisParams) .execute(); } @@ -256,55 +291,100 @@ private void indexQualityControl() throws ToolException { ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper(); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); ObjectReader signatureReader = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class); - ObjectReader hrDetectReader = JacksonUtils.getDefaultObjectMapper().readerFor(HRDetect.class); + ObjectReader signatureListReader = JacksonUtils.getDefaultObjectMapper().readerFor(new TypeReference>() {}); ObjectReader genomePlotReader = JacksonUtils.getDefaultObjectMapper().readerFor(GenomePlot.class); + ObjectReader genomePlotListReader = JacksonUtils.getDefaultObjectMapper().readerFor(new TypeReference>() {}); - for (Sample sample : targetSamples) { - // Check output files - Path qcPath = getOutDir().resolve(sample.getId()); - if (!Files.exists(qcPath)) { - String msg = "Quality control error for sample " + sample.getId() + ": folder " + qcPath + " not found." - + " None quality control was performed."; - logger.error(msg); - addError(new ToolException(msg)); - continue; - } - - int qcCode = NONE_READY; - SampleQualityControl sampleQc = sample.getQualityControl(); + QualityControlStatus qcStatus; + SampleQualityControl sampleQc; + String logMsg; - // Check signature - Signature signature = checkQcReport(sample.getId(), SIGNATURE_ANALYSIS_ID, analysisParams.getSkip(), qcPath, SAMPLE_QC_TYPE, - signatureReader); - if (signature != null) { - qcCode |= SIGNATURE_READY; - sampleQc.getVariant().getSignatures().add(signature); - } + for (Sample sample : targetSomaticSamples) { + sampleQc = new SampleQualityControl(); - // Check genome plot - GenomePlot genomePlot = checkQcReport(sample.getId(), GENOME_PLOT_ANALYSIS_ID, analysisParams.getSkip(), qcPath, SAMPLE_QC_TYPE, - genomePlotReader); - if (genomePlot != null) { - qcCode |= GENOME_PLOT_READY; - sampleQc.getVariant().getGenomePlots().add(genomePlot); + // Check and parse the signature results + Path qcPath = getOutDir().resolve(sample.getId()).resolve(SIGNATURE_ANALYSIS_ID) + .resolve(SIGNATURE_ANALYSIS_ID + QC_JSON_EXTENSION); + if (!Files.exists(qcPath)) { + failedQcSet.add(sample.getId()); + failedAnalysisPerSample.get(sample.getId()).add(SIGNATURE_ANALYSIS_ID); + sampleQc.getVariant().setSignatures(Collections.emptyList()); + + logMsg = FAILURE_FILE + qcPath.getFileName() + NOT_FOUND + getIdLogMessage(sample.getId(), SAMPLE_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } else { + try { + List signatureList = isQcArray(qcPath) + ? signatureListReader.readValue(qcPath.toFile()) + : Collections.singletonList(signatureReader.readValue(qcPath.toFile())); + + sampleQc.getVariant().setSignatures(signatureList); + } catch (IOException e) { + failedQcSet.add(sample.getId()); + failedAnalysisPerSample.get(sample.getId()).add(SIGNATURE_ANALYSIS_ID); + sampleQc.getVariant().setSignatures(Collections.emptyList()); + + logMsg = FAILURE_ERROR_PARSING_QC_JSON_FILE + qcPath.getFileName() + "'" + getIdLogMessage(sample.getId(), + INDIVIDUAL_QC_TYPE); + addError(e); + logger.error(logMsg, e); + } } - // Update catalog (quality control and status) if necessary - if (qcCode != NONE_READY) { - // Update the sample QC code with the current one - SampleQualityControlStatus qcStatus = new SampleQualityControlStatus( - qcCode | sample.getInternal().getQualityControlStatus().getCode(), sample.isSomatic(), ""); + // Check and parse the genome plot results + qcPath = getOutDir().resolve(sample.getId()).resolve(GENOME_PLOT_ANALYSIS_ID) + .resolve(GENOME_PLOT_ANALYSIS_ID + QC_JSON_EXTENSION); + if (!Files.exists(qcPath)) { + failedQcSet.add(sample.getId()); + failedAnalysisPerSample.get(sample.getId()).add(GENOME_PLOT_ANALYSIS_ID); + sampleQc.getVariant().setGenomePlots(Collections.emptyList()); + + logMsg = FAILURE_FILE + qcPath.getFileName() + NOT_FOUND + getIdLogMessage(sample.getId(), SAMPLE_QC_TYPE); + addError(new ToolException(logMsg)); + logger.error(logMsg); + } else { try { - SampleUpdateParams updateParams = new SampleUpdateParams() - .setQualityControl(sampleQc) - .setQualityControlStatus(qcStatus); - catalogManager.getSampleManager().update(getStudy(), sample.getId(), updateParams, null, token); - } catch (CatalogException e) { - logger.error("Could not update quality control in OpenCGA catalog for sample " + sample.getId(), e); + List genomePlotList = isQcArray(qcPath) + ? genomePlotListReader.readValue(qcPath.toFile()) + : Collections.singletonList(genomePlotReader.readValue(qcPath.toFile())); + + sampleQc.getVariant().setGenomePlots(genomePlotList); + } catch (IOException e) { + failedQcSet.add(sample.getId()); + failedAnalysisPerSample.get(sample.getId()).add(GENOME_PLOT_ANALYSIS_ID); + sampleQc.getVariant().setGenomePlots(Collections.emptyList()); + + logMsg = FAILURE_ERROR_PARSING_QC_JSON_FILE + qcPath.getFileName() + "'" + getIdLogMessage(sample.getId(), + SAMPLE_QC_TYPE); addError(e); + logger.error(logMsg, e); } } + + if (CollectionUtils.isEmpty(failedAnalysisPerSample.get(sample.getId()))) { + qcStatus = new QualityControlStatus(READY, SUCCESS); + } else { + qcStatus = new QualityControlStatus(ERROR, "Failed analysis: " + + StringUtils.join(failedAnalysisPerSample.get(sample.getId()), ",")); + } + + try { + // Update catalog: quality control and status + SampleUpdateParams updateParams = new SampleUpdateParams() + .setQualityControl(sampleQc) + .setQualityControlStatus(qcStatus); + catalogManager.getSampleManager().update(getStudy(), sample.getId(), updateParams, null, token); + } catch (CatalogException e) { + failedQcSet.add(sample.getId()); + logMsg = FAILURE_COULD_NOT_UPDATE_QUALITY_CONTROL_IN_OPEN_CGA_CATALOG + getIdLogMessage(sample.getId(), + INDIVIDUAL_QC_TYPE); + addError(e); + logger.error(logMsg, e); + } } + + checkFailedQcCounter(samples.size(), SAMPLE_QC_TYPE); } /** @@ -323,7 +403,7 @@ public static void checkParameters(SampleQcAnalysisParams params, String studyId checkStudy(studyId, catalogManager, token); // Check permissions - checkPermissions(WRITE_SAMPLES, studyId, catalogManager, token); + checkPermissions(WRITE_SAMPLES, params.getSkipIndex(), studyId, catalogManager, token); // Check samples in catalog if (CollectionUtils.isEmpty(params.getSamples())) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java index b3d29c598b..7c4cef3880 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java @@ -16,8 +16,11 @@ package org.opencb.opencga.analysis.variant.qc; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -30,34 +33,28 @@ import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.JwtPayload; -import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.family.Family; -import org.opencb.opencga.core.models.family.FamilyUpdateParams; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualQualityControlStatus; -import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.sample.SampleQualityControlStatus; -import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; import org.opencb.opencga.core.response.OpenCGAResult; import java.io.IOException; import java.net.URL; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; -import java.util.Arrays; +import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.opencb.opencga.analysis.AnalysisUtils.ANALYSIS_FOLDER; -import static org.opencb.opencga.analysis.AnalysisUtils.ANALYSIS_RESOURCES_FOLDER; -import static org.opencb.opencga.core.models.common.InternalStatus.READY; -import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_SAMPLES; public class VariantQcAnalysis extends OpenCgaToolScopeStudy { @@ -66,8 +63,10 @@ public class VariantQcAnalysis extends OpenCgaToolScopeStudy { public static final String QC_FOLDER = "qc/"; public static final String RESOURCES_FOLDER = "resources/"; public static final String QC_RESOURCES_FOLDER = QC_FOLDER + RESOURCES_FOLDER; + // File that contains the list of resource filenames (located at resources.opencb.org) to be downloaded + public static final String QC_RESOURCE_LIST_FILENAME = "qc_resource_list.txt"; - public static final String QC_JSON_EXTENSION = ".qc.json"; + public static final String QC_JSON_EXTENSION = ".json"; // Data type public static final String FAMILY_QC_TYPE = "family"; @@ -76,20 +75,12 @@ public class VariantQcAnalysis extends OpenCgaToolScopeStudy { // For relatedness analysis public static final String RELATEDNESS_ANALYSIS_ID = "relatedness"; - protected static final String RELATEDNESS_POP_FREQ_FILENAME = "autosomes_1000G_QC_prune_in.frq"; - protected static final String RELATEDNESS_POP_FREQ_FILE_MSG = "Population frequency file"; - protected static final String RELATEDNESS_POP_EXCLUDE_VAR_FILENAME = "autosomes_1000G_QC.prune.out"; - protected static final String RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG = "Population exclude variant file"; - protected static final String RELATEDNESS_THRESHOLDS_FILENAME = "relatedness_thresholds.tsv"; - protected static final String RELATEDNESS_THRESHOLDS_FILE_MSG = "Relatedness thresholds file"; // For inferred sex analysis public static final String INFERRED_SEX_ANALYSIS_ID = "inferred-sex"; - protected static final String INFERRED_SEX_THRESHOLDS_FILENAME = "karyotypic_sex_thresholds.json"; - protected static final String INFERRED_SEX_THRESHOLDS_FILE_MSG = "Karyotypic sex thresholds file"; // For mendelian errors sex analysis - public static final String MENDELIAN_ERROR_ANALYSIS_ID = "mendelian-errors"; + public static final String MENDELIAN_ERROR_ANALYSIS_ID = "mendelian-error"; // For signature analysis public static final String SIGNATURE_ANALYSIS_ID = "signature"; @@ -98,8 +89,22 @@ public class VariantQcAnalysis extends OpenCgaToolScopeStudy { public static final String GENOME_PLOT_ANALYSIS_ID = "genome-plot"; // Tool QC steps - public static final String PREPARE_QC_STEP = "prepare-qc"; - public static final String INDEX_QC_STEP = "index-qc"; + protected static final String PREPARE_QC_STEP = "prepare-qc"; + protected static final String INDEX_QC_STEP = "index-qc"; + + // Messages + protected static final String FAILURE_ERROR_PARSING_QC_JSON_FILE = "Failure: error parsing QC JSON file '"; + protected static final String FAILURE_FILE = "Failure: file '"; + protected static final String NOT_FOUND = "' not found"; + protected static final String FAILURE_COULD_NOT_UPDATE_QUALITY_CONTROL_IN_OPEN_CGA_CATALOG = "Failure: Could not update quality control" + + " in OpenCGA catalog"; + protected static final String SUCCESS = "Success"; + + protected LinkedList vcfPaths = new LinkedList<>(); + protected LinkedList jsonPaths = new LinkedList<>(); + + protected Path userResourcesPath; + protected Set failedQcSet; @Override protected void check() throws Exception { @@ -128,6 +133,31 @@ protected void run() throws Exception { // Nothing to do } + //------------------------------------------------------------------------- + + protected void clean() { + deleteFiles(vcfPaths); + deleteFiles(jsonPaths); + } + + private void deleteFiles(List paths) { + for (Path path : paths) { + try { + Files.delete(path); + } catch (IOException e) { + try { + addWarning("Could not delete file '" + path + "'"); + } catch (ToolException ex) { + logger.warn("When deleting file '" + path + "'", e); + } + } + } + } + + //------------------------------------------------------------------------- + // CHECKS MANAGEMENT + //------------------------------------------------------------------------- + protected static void checkStudy(String studyId, CatalogManager catalogManager, String token) throws ToolException { if (StringUtils.isEmpty(studyId)) { throw new ToolException("Missing study"); @@ -140,8 +170,8 @@ protected static void checkStudy(String studyId, CatalogManager catalogManager, } } - protected static void checkPermissions(StudyPermissions.Permissions permissions, String studyId, CatalogManager catalogManager, - String token) throws ToolException { + protected static void checkPermissions(StudyPermissions.Permissions permissions, Boolean skipIndex, String studyId, + CatalogManager catalogManager, String token) throws ToolException { checkStudy(studyId, catalogManager, token); try { @@ -150,8 +180,14 @@ protected static void checkPermissions(StudyPermissions.Permissions permissions, String organizationId = studyFqn.getOrganizationId(); String userId = jwtPayload.getUserId(organizationId); + // Check acess permissions Study study = catalogManager.getStudyManager().get(studyId, QueryOptions.empty(), token).first(); catalogManager.getAuthorizationManager().checkStudyPermission(organizationId, study.getUid(), userId, permissions); + + // Check admin permissions to Catalog index + if (!Boolean.TRUE.equals(skipIndex)) { + catalogManager.getAuthorizationManager().checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + } } catch (CatalogException e) { throw new ToolException(e); } @@ -175,7 +211,8 @@ protected static Path checkResourcesDir(String resourcesDir, String studyId, Cat if (!Files.exists(path)) { throw new ToolException("Resources path '" + path + "' does not exist (OpenCGA path: " + resourcesDir + ")"); } - return path; + + // TODO: Check permissions to read } catch (CatalogException e) { throw new ToolException("Error searching the OpenCGA catalog path '" + resourcesDir + "'", e); } @@ -183,62 +220,34 @@ protected static Path checkResourcesDir(String resourcesDir, String studyId, Cat return path; } - protected void prepareRelatednessResources(String resourcesDir) throws ToolException { - Path path = checkResourcesDir(resourcesDir, getStudy(), getCatalogManager(), getToken()); - - // Copy relatedness population frequency file - copyQcResourceFile(path, RELATEDNESS_POP_FREQ_FILENAME); - - // Copy relatedness population exclude variant file - copyQcResourceFile(path, RELATEDNESS_POP_FREQ_FILENAME); - - // Copy relatedness thresholds file - copyQcResourceFile(path, RELATEDNESS_THRESHOLDS_FILENAME); + protected void checkFailedQcCounter(int size, String individualQcType) throws ToolException { + if (CollectionUtils.isNotEmpty(failedQcSet) && failedQcSet.size() == size) { + // If all QC fail, then the job fails + clean(); + throw new ToolException("All " + individualQcType + " QCs fail. Please, check job results and logs for more details."); + } } - protected void prepareInferredSexResources(String resourcesDir) throws ToolException { - Path path = checkResourcesDir(resourcesDir, getStudy(), getCatalogManager(), getToken()); - - // Copy inferred sex thresholds file - copyQcResourceFile(path, INFERRED_SEX_THRESHOLDS_FILENAME); - } + //------------------------------------------------------------------------- + // QC file result management + //------------------------------------------------------------------------- - protected boolean setQualityControlStatus(QualityControlStatus qcStatus, String id, String qcType) throws ToolException { + protected boolean isQcArray(Path qcPath) throws ToolException { try { - switch (qcType) { - case FAMILY_QC_TYPE: { - FamilyUpdateParams updateParams = new FamilyUpdateParams().setQualityControlStatus(qcStatus); - catalogManager.getFamilyManager().update(getStudy(), id, updateParams, null, token); - break; - } - case INDIVIDUAL_QC_TYPE: { - IndividualUpdateParams updateParams = new IndividualUpdateParams() - .setQualityControlStatus((IndividualQualityControlStatus) qcStatus); - catalogManager.getIndividualManager().update(getStudy(), id, updateParams, null, token); - break; - } - case SAMPLE_QC_TYPE: { - SampleUpdateParams updateParams = new SampleUpdateParams() - .setQualityControlStatus((SampleQualityControlStatus) qcStatus); - catalogManager.getSampleManager().update(getStudy(), id, updateParams, null, token); - break; - } - default: { - String msg = "Internal error: unknown QC type '" + qcType + "' (valid values are: " + StringUtils.join( - Arrays.asList(FAMILY_QC_TYPE, INDIVIDUAL_QC_TYPE), ",") + ")"; - throw new ToolException(msg); - } - } - } catch (CatalogException e) { - String msg = "Could not set status to COMPUTING before performing QC for " + qcType + " ID '" + id + "': " + e.getMessage(); - logger.error(msg); - addError(new ToolException(msg, e)); - return false; + // Create an ObjectMapper instance + ObjectMapper objectMapper = new ObjectMapper(); + + // Read the JSON file into a JsonNode + JsonNode rootNode = objectMapper.readTree(qcPath.toFile()); + + // Check if the root element is an array + return rootNode.isArray(); + } catch (IOException e) { + throw new ToolException("Error checking QC file '" + qcPath + "'", e); } - return true; } - protected T checkQcReport(String id, String analysisId, List skip, Path qcPath, String qcType, ObjectReader reader) + protected T parseQcFile(String id, String analysisId, List skip, Path qcPath, String qcType, ObjectReader reader) throws ToolException { if (CollectionUtils.isEmpty(skip) || !skip.contains(analysisId)) { java.io.File qcFile = qcPath.resolve(analysisId).resolve(id + QC_JSON_EXTENSION).toFile(); @@ -255,6 +264,10 @@ protected T checkQcReport(String id, String analysisId, List skip, P return null; } + //------------------------------------------------------------------------- + // Catalog utils + //------------------------------------------------------------------------- + protected static List getNoSomaticSampleIds(Family family, String studyId, CatalogManager catalogManager, String token) throws CatalogException { // Get list of individual IDs @@ -287,60 +300,81 @@ protected static List getNoSomaticSampleIds(Individual individual) { return sampleIds; } - protected void copyQcResourceFile(Path path, String resourceName) throws ToolException { - // Copy resource file - if (path == null || !Files.exists(path.resolve(resourceName))) { - // Use the default resource file - copyQcResourceFile(resourceName); - } else { - // Use the custom resource file - copyQcResourceFile(path.resolve(resourceName)); + //------------------------------------------------------------------------- + // QC RESOURCES MANAGEMENT + //------------------------------------------------------------------------- + + protected void prepareResources() throws ToolException { + // First, download default resource files + downloadQcResourceFiles(); + if (userResourcesPath != null) { + // If necessary, copy the user resource files + copyUserResourceFiles(); } } - protected void copyQcResourceFile(String resourceName) throws ToolException { - Path srcResourcesPath = getOpencgaHome().resolve(ANALYSIS_RESOURCES_FOLDER).resolve(QC_FOLDER); + protected void downloadQcResourceFiles() throws ToolException { Path destResourcesPath = checkResourcesPath(getOutDir().resolve(RESOURCES_FOLDER)); - if (Files.exists(srcResourcesPath.resolve(resourceName))) { - // Copy resource file - copyQcResourceFile(srcResourcesPath.resolve(resourceName)); - } else { - // Download directly into the job dir - // It can be improved by downloading once (the first time) in the analysis resources folder - URL url = null; - try { - url = new URL(ResourceUtils.URL + ANALYSIS_FOLDER + QC_FOLDER + "/" + resourceName); - ResourceUtils.downloadThirdParty(url, destResourcesPath); - } catch (IOException e) { - throw new ToolException("Something wrong happened when downloading the resource '" + resourceName + "' from '" - + url + "'", e); - } - if (!Files.exists(destResourcesPath.resolve(resourceName))) { - throw new ToolException("Error downloading the resource '" + resourceName + "', it does not exist at " + destResourcesPath); - } + Path qcSourcesPath = getOutDir().resolve(RESOURCES_FOLDER).resolve(QC_RESOURCE_LIST_FILENAME); + downloadQcResourceFile(qcSourcesPath.getFileName().toString(), destResourcesPath); + if (!Files.exists(qcSourcesPath)) { + throw new ToolException("Unable to download QC resource list (file '" + qcSourcesPath.getFileName() + "')"); } - } - protected void copyQcResourceFile(Path srcResourcesPath) throws ToolException { - String resourceName = srcResourcesPath.getFileName().toString(); - Path destResourcesPath = checkResourcesPath(getOutDir().resolve(RESOURCES_FOLDER)); + List qcResourceList; + try { + qcResourceList = FileUtils.readLines(qcSourcesPath.toFile(), Charset.defaultCharset()); + } catch (IOException e) { + throw new ToolException("Error reading QC resource list (file '" + qcSourcesPath.getFileName() + "')", e); + } - String msg = "Error copying resource file '" + resourceName + "'"; + if (CollectionUtils.isEmpty(qcResourceList)) { + throw new ToolException("Something wrong happened: the QC resource list (file '" + qcSourcesPath.getFileName() + "') is empty"); + } - // Copy resource file + // Download all resource files + for (String resourceName : qcResourceList) { + downloadQcResourceFile(resourceName, destResourcesPath); + } + } + + protected void downloadQcResourceFile(String resourceName, Path destPath) throws ToolException { + Path destResourcesPath = checkResourcesPath(destPath); + URL url = null; try { - Files.copy(srcResourcesPath, destResourcesPath.resolve(resourceName)); + url = new URL(ResourceUtils.URL + ANALYSIS_FOLDER + QC_FOLDER + "/" + resourceName); + ResourceUtils.downloadThirdParty(url, destPath); } catch (IOException e) { - if (!Files.exists(destResourcesPath.resolve(resourceName)) - || srcResourcesPath.toFile().length() != destResourcesPath.resolve(resourceName).toFile().length()) { - throw new ToolException(msg, e); - } - logger.warn(msg, e); + throw new ToolException("Something wrong happened when downloading the resource '" + resourceName + "' from '" + url + "'", e); } if (!Files.exists(destResourcesPath.resolve(resourceName))) { - throw new ToolException(msg + ", it does not exist at " + destResourcesPath); + throw new ToolException("Error downloading the resource '" + resourceName + "', it does not exist at " + destResourcesPath); + } + } + + protected void copyUserResourceFiles() throws ToolException { + // Sanity check + if (userResourcesPath == null) { + // Nothing to do + return; + } + + Path destResourcesPath = checkResourcesPath(getOutDir().resolve(RESOURCES_FOLDER)); + + // Copy custom resource files to the job dir + for (java.io.File file : userResourcesPath.toFile().listFiles()) { + Path destPath = destResourcesPath.resolve(file.getName()); + if (file.isFile()) { + try { + Files.copy(file.toPath(), destPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + if (!Files.exists(destPath) || destPath.toFile().length() != file.length()) { + throw new ToolException("Error copying resource file '" + file + "'", e); + } + } + } } } @@ -358,37 +392,7 @@ protected Path checkResourcesPath(Path resourcesPath) throws ToolException { return resourcesPath; } - protected boolean mustPerformQualityControl(QualityControlStatus qcStatus, Boolean overwrite) { - boolean performQc; - if (Boolean.TRUE.equals(overwrite)) { - performQc = true; - } else if (qcStatus != null) { - String statusId = qcStatus.getId(); - performQc = !(statusId.equals(COMPUTING) || statusId.equals(READY)); - } else { - performQc = true; - } - return performQc; - } - - protected Path checkDirectory(Path dir) throws ToolException { - if (!Files.exists(dir)) { - try { - Files.createDirectories(dir); - } catch (IOException e) { - throw new ToolException("Error creating directory '" + dir + "'", e); - } - if (!Files.exists(dir)) { - throw new ToolException("Directory '" + dir + "' does not exist after creating directory"); - } - } - return dir; - } - - protected boolean mustSkip(String value, List skip) { - if (CollectionUtils.isEmpty(skip)) { - return false; - } - return skip.contains(value); + protected String getIdLogMessage(String id, String qcType) { + return " for " + qcType + " '" + id + "'"; } } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index d660328ca8..9873931c4b 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -71,6 +71,7 @@ import org.opencb.opencga.core.models.cohort.CohortCreateParams; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.family.FamilyQualityControl; import org.opencb.opencga.core.models.family.FamilyUpdateParams; @@ -83,7 +84,10 @@ import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; -import org.opencb.opencga.core.models.sample.*; +import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.sample.SampleQualityControl; +import org.opencb.opencga.core.models.sample.SampleReferenceParam; +import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.variant.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; @@ -775,7 +779,7 @@ public void testSampleQcStats() throws Exception { System.out.println("outDir = " + outDir); SampleQualityControl qc = new SampleQualityControl(); - SampleUpdateParams updateParams = new SampleUpdateParams().setQualityControl(qc).setQualityControlStatus(new SampleQualityControlStatus()); + SampleUpdateParams updateParams = new SampleUpdateParams().setQualityControl(qc).setQualityControlStatus(new QualityControlStatus()); catalogManager.getSampleManager().update(CANCER_STUDY, cancer_sample, updateParams, null, token); SampleQcAnalysisParams params = new SampleQcAnalysisParams(); @@ -783,7 +787,7 @@ public void testSampleQcStats() throws Exception { params.setVsId("test"); params.setVsDescription("Description test"); params.setVsQuery(new AnnotationVariantQueryParams()); - params.setSkip(Arrays.asList(SIGNATURE_ANALYSIS_ID, GENOME_PLOT_ANALYSIS_ID)); + params.setSkipAnalysis(Arrays.asList(SIGNATURE_ANALYSIS_ID, GENOME_PLOT_ANALYSIS_ID)); toolRunner.execute(SampleVariantQcAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), outDir, null, false, token); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 94d95f8ecb..0241e9e0dd 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -941,7 +941,7 @@ private RestResponse runIndividualQc() throws Exception { putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); putNestedIfNotEmpty(beanParams, "inferredSexMethod",commandOptions.inferredSexMethod, true); putNestedIfNotEmpty(beanParams, "resourcesDir",commandOptions.resourcesDir, true); - putNestedIfNotNull(beanParams, "skip",commandOptions.skip, true); + putNestedIfNotNull(beanParams, "skipAnalysis",commandOptions.skipAnalysis, true); putNestedIfNotNull(beanParams, "skipIndex",commandOptions.skipIndex, true); putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); @@ -1529,7 +1529,7 @@ private RestResponse runSampleQc() throws Exception { putNestedIfNotEmpty(beanParams, "gpId",commandOptions.gpId, true); putNestedIfNotEmpty(beanParams, "gpDescription",commandOptions.gpDescription, true); putNestedIfNotEmpty(beanParams, "gpConfigFile",commandOptions.gpConfigFile, true); - putNestedIfNotNull(beanParams, "skip",commandOptions.skip, true); + putNestedIfNotNull(beanParams, "skipAnalysis",commandOptions.skipAnalysis, true); putNestedIfNotNull(beanParams, "skipIndex",commandOptions.skipIndex, true); putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); putNestedIfNotEmpty(beanParams, "resourcesDir",commandOptions.resourcesDir, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 51bdbf0bb6..e49904e4d2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -19,6 +19,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.individual.Individual; @@ -27,7 +28,6 @@ import org.opencb.opencga.core.models.individual.IndividualCreateParams; import org.opencb.opencga.core.models.individual.IndividualPopulation; import org.opencb.opencga.core.models.individual.IndividualQualityControl; -import org.opencb.opencga.core.models.individual.IndividualQualityControlStatus; import org.opencb.opencga.core.models.individual.IndividualReferenceParam; import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.individual.Location; @@ -453,7 +453,6 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "qualityControlStatus.date",commandOptions.qualityControlStatusDate, true); putNestedIfNotEmpty(beanParams, "qualityControlStatus.version",commandOptions.qualityControlStatusVersion, true); putNestedIfNotEmpty(beanParams, "qualityControlStatus.commit",commandOptions.qualityControlStatusCommit, true); - putNestedIfNotNull(beanParams, "qualityControlStatus.code",commandOptions.qualityControlStatusCode, true); putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); individualUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java index ecf3c38281..0b0f3678d0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java @@ -18,6 +18,7 @@ import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.ExternalSource; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.RgaIndex.Status; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; @@ -29,7 +30,6 @@ import org.opencb.opencga.core.models.sample.SampleCreateParams; import org.opencb.opencga.core.models.sample.SampleProcessing; import org.opencb.opencga.core.models.sample.SampleQualityControl; -import org.opencb.opencga.core.models.sample.SampleQualityControlStatus; import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.sample.SampleVariantQualityControlMetrics; import org.opencb.opencga.core.response.QueryType; @@ -473,7 +473,6 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "qualityControlStatus.date",commandOptions.qualityControlStatusDate, true); putNestedIfNotEmpty(beanParams, "qualityControlStatus.version",commandOptions.qualityControlStatusVersion, true); putNestedIfNotEmpty(beanParams, "qualityControlStatus.commit",commandOptions.qualityControlStatusCommit, true); - putNestedIfNotNull(beanParams, "qualityControlStatus.code",commandOptions.qualityControlStatusCode, true); putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); putNestedIfNotEmpty(beanParams, "status.id",commandOptions.statusId, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index fd2206d917..42cd209a3d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -1301,8 +1301,8 @@ public class RunIndividualQcCommandOptions { @Parameter(names = {"--resources-dir"}, description = "Directory where the QC resource files are located", required = false, arity = 1) public String resourcesDir; - @Parameter(names = {"--skip"}, description = "Individual QC analysis to skip. Valid values are: inferred-sex, mendelian-errors", required = false, arity = 1) - public String skip; + @Parameter(names = {"--skip-analysis"}, description = "List of individual QC analysis to skip. Valid values are: inferred-sex, mendelian-errors and relatedness (if parents are present)", required = false, arity = 1) + public String skipAnalysis; @Parameter(names = {"--skip-index"}, description = "Do not save the computed quality control in catalog", required = false, arity = 1) public Boolean skipIndex; @@ -2307,8 +2307,8 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--gp-config-file"}, description = "Genome plot configuration file.", required = false, arity = 1) public String gpConfigFile; - @Parameter(names = {"--skip"}, description = "Quality control metrics to skip. Valid values are: variant-stats, signature, signature-catalogue, signature-fitting, genome-plot", required = false, arity = 1) - public String skip; + @Parameter(names = {"--skip-analysis"}, description = "Quality control metrics to skip. Valid values are: variant-stats, signature, signature-catalogue, signature-fitting, genome-plot", required = false, arity = 1) + public String skipAnalysis; @Parameter(names = {"--skip-index"}, description = "Do not save the computed quality control in catalog", required = false, arity = 1) public Boolean skipIndex; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java index cba0c05faf..0a49adc560 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java @@ -680,9 +680,6 @@ public class UpdateCommandOptions { @Parameter(names = {"--quality-control-status-commit"}, description = "The body web service commit parameter", required = false, arity = 1) public String qualityControlStatusCommit; - @Parameter(names = {"--quality-control-status-code"}, description = "The body web service code parameter", required = false, arity = 1) - public Integer qualityControlStatusCode; - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java index 724121e144..10544b79d0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/SamplesCommandOptions.java @@ -712,9 +712,6 @@ public class UpdateCommandOptions { @Parameter(names = {"--quality-control-status-commit"}, description = "The body web service commit parameter", required = false, arity = 1) public String qualityControlStatusCommit; - @Parameter(names = {"--quality-control-status-code"}, description = "The body web service code parameter", required = false, arity = 1) - public Integer qualityControlStatusCode; - @Parameter(names = {"--somatic"}, description = "The body web service somatic parameter", required = false, arity = 1) public Boolean somatic; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index a3ca6b7b50..73974a435e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -204,8 +204,8 @@ public class FieldConstants { public static final String INDIVIDUAL_QC_SAMPLE_ID_DESCRIPTION = "Sample ID (required when the individual has multiple samples)"; @Deprecated public static final String INFERRED_SEX_METHOD_DESCRIPTION = "Inferred sex method."; - public static final String INDIVIDUAL_QC_SKIP_DESCRIPTION = "Individual QC analysis to skip. Valid values are: " + INFERRED_SEX_ID - + ", " + MENDELIAN_ERRORS_ID; + public static final String INDIVIDUAL_QC_SKIP_ANALYSIS_DESCRIPTION = "List of individual QC analysis to skip. Valid values are: " + + INFERRED_SEX_ID + ", " + MENDELIAN_ERRORS_ID + " and " + RELATEDNESS_ID + " (if parents are present)"; public static final String INDIVIDUAL_QC_INFERRED_SEX_DESCRIPTION = "Inferred sex results for the individual"; public static final String INDIVIDUAL_QC_RELATEDNESS_DESCRIPTION = "Relatedness results for the individual"; public static final String INDIVIDUAL_QC_MENDELIAN_ERROR_DESCRIPTION = "Mendelian error results for the individual"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java index 1072d75e09..f4cbc48847 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java @@ -24,14 +24,13 @@ public class QualityControlStatus extends InternalStatus { /* * States * - * NONE --> COMPUTING --> READY - * --> INCOMPLETE + * NONE --> READY + * ERROR */ public static final String NONE = "NONE"; - public static final String COMPUTING = "COMPUTING"; - public static final String INCOMPLETE = "INCOMPLETE"; + public static final String ERROR = "ERROR"; - protected static final List STATUS_LIST = Arrays.asList(READY, DELETED, NONE, COMPUTING, INCOMPLETE); + protected static final List STATUS_LIST = Arrays.asList(READY, DELETED, NONE, ERROR); public QualityControlStatus(String status, String message) { if (isValid(status)) { @@ -57,8 +56,7 @@ public static boolean isValid(String status) { return status != null && (status.equals(READY) || status.equals(DELETED) - || status.equals(INCOMPLETE) || status.equals(NONE) - || status.equals(COMPUTING)); + || status.equals(ERROR)); } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java index 2c6c1f7f72..02565eaa8d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java @@ -23,20 +23,20 @@ public class IndividualInternal extends Internal { - private IndividualQualityControlStatus qualityControlStatus; + private QualityControlStatus qualityControlStatus; public IndividualInternal() { } public IndividualInternal(InternalStatus status, String registrationDate, String modificationDate, - IndividualQualityControlStatus qualityControlStatus) { + QualityControlStatus qualityControlStatus) { super(status, registrationDate, modificationDate); this.qualityControlStatus = qualityControlStatus; } public static IndividualInternal init() { String time = TimeUtils.getTime(); - return new IndividualInternal(new InternalStatus(InternalStatus.READY), time, time, new IndividualQualityControlStatus()); + return new IndividualInternal(new InternalStatus(InternalStatus.READY), time, time, new QualityControlStatus()); } @Override @@ -77,11 +77,11 @@ public IndividualInternal setLastModified(String lastModified) { return this; } - public IndividualQualityControlStatus getQualityControlStatus() { + public QualityControlStatus getQualityControlStatus() { return qualityControlStatus; } - public IndividualInternal setQualityControlStatus(IndividualQualityControlStatus qualityControlStatus) { + public IndividualInternal setQualityControlStatus(QualityControlStatus qualityControlStatus) { this.qualityControlStatus = qualityControlStatus; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualQualityControlStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualQualityControlStatus.java deleted file mode 100644 index a1668dfdb9..0000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualQualityControlStatus.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.individual; - -import org.opencb.opencga.core.models.common.QualityControlStatus; - -import java.util.Objects; - -public class IndividualQualityControlStatus extends QualityControlStatus { - - private int code; - - public static final int NONE_READY = 0; - public static final int INFERRED_SEX_READY = 1; - public static final int MENDELIAN_ERROR_READY = 2; - public static final int RELATEDNESS_READY = 4; - - public IndividualQualityControlStatus(int code, String message) { - String status; - switch (code) { - case NONE_READY: - status = NONE; - break; - case INFERRED_SEX_READY + MENDELIAN_ERROR_READY + RELATEDNESS_READY: - status = READY; - break; - default: - status = INCOMPLETE; - break; - } - this.code = code; - init(status, status, message); - } - - public IndividualQualityControlStatus(int code) { - this(code, ""); - } - - public IndividualQualityControlStatus() { - this(NONE_READY, ""); - } - - public IndividualQualityControlStatus(String status, String message) { - super(status, message); - } - - public static IndividualQualityControlStatus init() { - return new IndividualQualityControlStatus(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("IndividualQualityControlStatus{"); - sb.append("code=").append(code); - sb.append(", id='").append(id).append('\''); - sb.append(", name='").append(name).append('\''); - sb.append(", description='").append(description).append('\''); - sb.append(", date='").append(date).append('\''); - sb.append('}'); - return sb.toString(); - } - - public int getCode() { - return code; - } - - public IndividualQualityControlStatus setCode(int code) { - this.code = code; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof IndividualQualityControlStatus)) return false; - if (!super.equals(o)) return false; - - IndividualQualityControlStatus that = (IndividualQualityControlStatus) o; - - return Objects.equals(code, that.code); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + code; - return result; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java index c35a2373f7..c25b5f0070 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java @@ -25,6 +25,7 @@ import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleReferenceParam; @@ -59,7 +60,7 @@ public class IndividualUpdateParams { private List disorders; private StatusParams status; private IndividualQualityControl qualityControl; - private IndividualQualityControlStatus qualityControlStatus; + private QualityControlStatus qualityControlStatus; private Map attributes; private static final String QUALITY_CONTROL_STATUS_KEY = "qualityControlStatus"; @@ -74,8 +75,8 @@ public IndividualUpdateParams(String id, String name, IndividualReferenceParam f String dateOfBirth, IndividualProperty.KaryotypicSex karyotypicSex, IndividualProperty.LifeStatus lifeStatus, List samples, List annotationSets, List phenotypes, List disorders, - StatusParams status, IndividualQualityControl qualityControl, - IndividualQualityControlStatus qualityControlStatus, Map attributes) { + StatusParams status, IndividualQualityControl qualityControl, QualityControlStatus qualityControlStatus, + Map attributes) { this.id = id; this.name = name; this.father = father; @@ -343,11 +344,11 @@ public IndividualUpdateParams setQualityControl(IndividualQualityControl quality return this; } - public IndividualQualityControlStatus getQualityControlStatus() { + public QualityControlStatus getQualityControlStatus() { return qualityControlStatus; } - public IndividualUpdateParams setQualityControlStatus(IndividualQualityControlStatus qualityControlStatus) { + public IndividualUpdateParams setQualityControlStatus(QualityControlStatus qualityControlStatus) { this.qualityControlStatus = qualityControlStatus; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java index 76b3a88cdc..8786c8d6d1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java @@ -21,6 +21,7 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.RgaIndex; import java.util.Objects; @@ -35,13 +36,13 @@ public class SampleInternal extends Internal { description = FieldConstants.SAMPLE_INTERNAL_RGA_DESCRIPTION) private RgaIndex rga; private SampleInternalVariant variant; - private SampleQualityControlStatus qualityControlStatus; + private QualityControlStatus qualityControlStatus; public SampleInternal() { } public SampleInternal(String registrationDate, String modificationDate, InternalStatus status, RgaIndex rga, - SampleInternalVariant variant, SampleQualityControlStatus qualityControlStatus) { + SampleInternalVariant variant, QualityControlStatus qualityControlStatus) { super(status, registrationDate, modificationDate); this.rga = rga; this.variant = variant; @@ -51,7 +52,7 @@ public SampleInternal(String registrationDate, String modificationDate, Internal public static SampleInternal init() { String time = TimeUtils.getTime(); return new SampleInternal(time, time, new InternalStatus(InternalStatus.READY), RgaIndex.init(), SampleInternalVariant.init(), - new SampleQualityControlStatus()); + new QualityControlStatus()); } @Override @@ -112,11 +113,11 @@ public SampleInternal setLastModified(String lastModified) { return this; } - public SampleQualityControlStatus getQualityControlStatus() { + public QualityControlStatus getQualityControlStatus() { return qualityControlStatus; } - public SampleInternal setQualityControlStatus(SampleQualityControlStatus qualityControlStatus) { + public SampleInternal setQualityControlStatus(QualityControlStatus qualityControlStatus) { this.qualityControlStatus = qualityControlStatus; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleQualityControlStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleQualityControlStatus.java deleted file mode 100644 index 9ed4133ba8..0000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleQualityControlStatus.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2015-2020 OpenCB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.opencb.opencga.core.models.sample; - -import org.opencb.opencga.core.models.common.QualityControlStatus; - -import java.util.Objects; - -public class SampleQualityControlStatus extends QualityControlStatus { - - private int code; - - public static final int NONE_READY = 0; - public static final int STATS_READY = 1; - public static final int SIGNATURE_READY = 2; - public static final int HR_DETECT_READY = 4; - public static final int GENOME_PLOT_READY = 8; - - public SampleQualityControlStatus(int code, boolean isSomatic, String message) { - String status; - switch (code) { - case NONE_READY: - status = NONE; - break; - case STATS_READY + SIGNATURE_READY + HR_DETECT_READY + GENOME_PLOT_READY: - status = READY; - break; - default: - if (!isSomatic && (code | STATS_READY) > 0) { - status = READY; - } else { - status = INCOMPLETE; - } - break; - } - this.code = code; - init(status, status, message); - } - - public SampleQualityControlStatus(int code, boolean isSomatic) { - this(code, isSomatic, ""); - } - - public SampleQualityControlStatus() { - this(NONE_READY, false, ""); - } - - public SampleQualityControlStatus(String status, String message) { - super(status, message); - } - - public static SampleQualityControlStatus init() { - return new SampleQualityControlStatus(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("SampleQualityControlStatus{"); - sb.append("code=").append(code); - sb.append(", id='").append(id).append('\''); - sb.append(", name='").append(name).append('\''); - sb.append(", description='").append(description).append('\''); - sb.append(", date='").append(date).append('\''); - sb.append('}'); - return sb.toString(); - } - - public int getCode() { - return code; - } - - public SampleQualityControlStatus setCode(int code) { - this.code = code; - return this; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof SampleQualityControlStatus)) return false; - if (!super.equals(o)) return false; - - SampleQualityControlStatus that = (SampleQualityControlStatus) o; - - return Objects.equals(code, that.code); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + code; - return result; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleUpdateParams.java index 13f49a4769..5bb822c59c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleUpdateParams.java @@ -22,6 +22,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.common.ExternalSource; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import java.util.Collections; @@ -41,7 +42,7 @@ public class SampleUpdateParams { private SampleProcessing processing; private SampleCollection collection; private SampleQualityControl qualityControl; - private SampleQualityControlStatus qualityControlStatus; + private QualityControlStatus qualityControlStatus; private Boolean somatic; private List phenotypes; private List annotationSets; @@ -53,9 +54,9 @@ public SampleUpdateParams() { public SampleUpdateParams(String id, String description, String creationDate, String modificationDate, String individualId, ExternalSource source, SampleProcessing processing, SampleCollection collection, - SampleQualityControl qualityControl, SampleQualityControlStatus qualityControlStatus, Boolean somatic, - List phenotypes, - List annotationSets, Map attributes, StatusParams status) { + SampleQualityControl qualityControl, QualityControlStatus qualityControlStatus, Boolean somatic, + List phenotypes, List annotationSets, Map attributes, + StatusParams status) { this.id = id; this.description = description; this.creationDate = creationDate; @@ -205,11 +206,11 @@ public SampleUpdateParams setQualityControl(SampleQualityControl qualityControl) return this; } - public SampleQualityControlStatus getQualityControlStatus() { + public QualityControlStatus getQualityControlStatus() { return qualityControlStatus; } - public SampleUpdateParams setQualityControlStatus(SampleQualityControlStatus qualityControlStatus) { + public SampleUpdateParams setQualityControlStatus(QualityControlStatus qualityControlStatus) { this.qualityControlStatus = qualityControlStatus; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java index 0a4c2ff1ef..75397d2a58 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java @@ -44,8 +44,8 @@ public class IndividualQcAnalysisParams extends ToolParams { @DataField(id = "resourcesDir", description = FieldConstants.QC_RESOURCES_DIR_DESCRIPTION) private String resourcesDir; - @DataField(id = "skip", description = FieldConstants.INDIVIDUAL_QC_SKIP_DESCRIPTION) - private List skip; + @DataField(id = "skipAnalysis", description = FieldConstants.INDIVIDUAL_QC_SKIP_ANALYSIS_DESCRIPTION) + private List skipAnalysis; @DataField(id = "skipIndex", description = FieldConstants.QC_SKIP_INDEX_DESCRIPTION) private Boolean skipIndex; @@ -60,13 +60,13 @@ public IndividualQcAnalysisParams() { } public IndividualQcAnalysisParams(List individuals, String individual, String sample, String inferredSexMethod, - String resourcesDir, List skip, Boolean skipIndex, Boolean overwrite, String outdir) { + String resourcesDir, List skipAnalysis, Boolean skipIndex, Boolean overwrite, String outdir) { this.individuals = individuals; this.individual = individual; this.sample = sample; this.inferredSexMethod = inferredSexMethod; this.resourcesDir = resourcesDir; - this.skip = skip; + this.skipAnalysis = skipAnalysis; this.skipIndex = skipIndex; this.overwrite = overwrite; this.outdir = outdir; @@ -80,7 +80,7 @@ public String toString() { sb.append(", sample='").append(sample).append('\''); sb.append(", inferredSexMethod='").append(inferredSexMethod).append('\''); sb.append(", resourcesDir='").append(resourcesDir).append('\''); - sb.append(", skip=").append(skip); + sb.append(", skipAnalysis=").append(skipAnalysis); sb.append(", skipIndex=").append(skipIndex); sb.append(", overwrite=").append(overwrite); sb.append(", outdir='").append(outdir).append('\''); @@ -133,12 +133,12 @@ public IndividualQcAnalysisParams setResourcesDir(String resourcesDir) { return this; } - public List getSkip() { - return skip; + public List getSkipAnalysis() { + return skipAnalysis; } - public IndividualQcAnalysisParams setSkip(List skip) { - this.skip = skip; + public IndividualQcAnalysisParams setSkipAnalysis(List skipAnalysis) { + this.skipAnalysis = skipAnalysis; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java index b3b651acfa..c091a7b56b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java @@ -107,8 +107,8 @@ public class SampleQcAnalysisParams extends ToolParams { private String gpConfigFile; // Other - @DataField(id = "skip", description = FieldConstants.SAMPLE_QUALITY_CONTROL_SKIP_DESCRIPTION) - private List skip; + @DataField(id = "skipAnalysis", description = FieldConstants.SAMPLE_QUALITY_CONTROL_SKIP_DESCRIPTION) + private List skipAnalysis; @DataField(id = "skipIndex", description = FieldConstants.QC_SKIP_INDEX_DESCRIPTION) private Boolean skipIndex; @@ -129,8 +129,8 @@ public SampleQcAnalysisParams(String sample, List samples, String vsId, AnnotationVariantQueryParams vsQuery, String msId, String msDescription, String msQuery, String msFitId, String msFitMethod, Integer msFitNBoot, String msFitSigVersion, String msFitOrgan, Float msFitThresholdPerc, Float msFitThresholdPval, Integer msFitMaxRareSigs, String msFitSignaturesFile, - String msFitRareSignaturesFile, String gpId, String gpDescription, String gpConfigFile, List skip, - Boolean skipIndex, Boolean overwrite, String resourcesDir, String outdir) { + String msFitRareSignaturesFile, String gpId, String gpDescription, String gpConfigFile, + List skipAnalysis, Boolean skipIndex, Boolean overwrite, String resourcesDir, String outdir) { this.sample = sample; this.samples = samples; this.vsId = vsId; @@ -152,7 +152,7 @@ public SampleQcAnalysisParams(String sample, List samples, String vsId, this.gpId = gpId; this.gpDescription = gpDescription; this.gpConfigFile = gpConfigFile; - this.skip = skip; + this.skipAnalysis = skipAnalysis; this.skipIndex = skipIndex; this.overwrite = overwrite; this.resourcesDir = resourcesDir; @@ -183,7 +183,7 @@ public String toString() { sb.append(", gpId='").append(gpId).append('\''); sb.append(", gpDescription='").append(gpDescription).append('\''); sb.append(", gpConfigFile='").append(gpConfigFile).append('\''); - sb.append(", skip=").append(skip); + sb.append(", skipAnalysis=").append(skipAnalysis); sb.append(", skipIndex=").append(skipIndex); sb.append(", overwrite=").append(overwrite); sb.append(", resourcesDir='").append(resourcesDir).append('\''); @@ -381,12 +381,12 @@ public SampleQcAnalysisParams setGpConfigFile(String gpConfigFile) { return this; } - public List getSkip() { - return skip; + public List getSkipAnalysis() { + return skipAnalysis; } - public SampleQcAnalysisParams setSkip(List skip) { - this.skip = skip; + public SampleQcAnalysisParams setSkipAnalysis(List skipAnalysis) { + this.skipAnalysis = skipAnalysis; return this; }