From f4eefb7dd027b3acec4082504ffce69aa6767ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Mon, 9 Sep 2024 16:45:22 +0200 Subject: [PATCH] analysis: update and improve invidual QC analysis according to the latest changes, #TASK-6772, #TASK-6766 --- .../family/qc/FamilyVariantQcAnalysis.java | 113 +------- .../FamilyVariantQcLocalAnalysisExecutor.java | 41 +-- .../qc/IndividualVariantQcAnalysis.java | 255 +++++++++++++++++- ...ividualVariantQcLocalAnalysisExecutor.java | 49 ++++ .../models/IndividualPrivateUpdateParams.java | 7 +- .../utils/VariantQcAnalysisExecutorUtils.java | 76 ++++++ .../variant/qc/VariantQcAnalysis.java | 115 ++++++++ .../AnalysisVariantCommandExecutor.java | 9 +- .../executors/IndividualsCommandExecutor.java | 7 + .../AnalysisVariantCommandOptions.java | 21 ++ .../options/IndividualsCommandOptions.java | 18 ++ .../models/family/FamilyUpdateParams.java | 3 +- .../individual/IndividualUpdateParams.java | 23 +- .../variant/FamilyQcAnalysisParams.java | 8 +- .../variant/IndividualQcAnalysisParams.java | 62 ++++- ....java => QcRelatednessAnalysisParams.java} | 12 +- .../IndividualVariantQcAnalysisExecutor.java | 61 +++++ 17 files changed, 715 insertions(+), 165 deletions(-) create mode 100644 opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcLocalAnalysisExecutor.java create mode 100644 opencga-analysis/src/main/java/org/opencb/opencga/analysis/utils/VariantQcAnalysisExecutorUtils.java rename opencga-core/src/main/java/org/opencb/opencga/core/models/variant/{FamilyQcRelatednessAnalysisParams.java => QcRelatednessAnalysisParams.java} (81%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/IndividualVariantQcAnalysisExecutor.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java index 18997a3eb01..0deaa0443d7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcAnalysis.java @@ -28,7 +28,6 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.qc.VariantQcAnalysis; import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; -import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.JacksonUtils; @@ -38,10 +37,8 @@ import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.family.FamilyQualityControl; import org.opencb.opencga.core.models.family.FamilyUpdateParams; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.variant.FamilyQcAnalysisParams; -import org.opencb.opencga.core.models.variant.FamilyQcRelatednessAnalysisParams; +import org.opencb.opencga.core.models.variant.QcRelatednessAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @@ -70,14 +67,6 @@ public class FamilyVariantQcAnalysis extends VariantQcAnalysis { public static final String DESCRIPTION = "Run quality control (QC) for a given family. It computes the relatedness scores among the" + " family members"; - public static final String RELATEDNESS_POP_FREQ_FILENAME = "autosomes_1000G_QC_prune_in.frq"; - public static final String RELATEDNESS_POP_EXCLUDE_VAR_FILENAME = "autosomes_1000G_QC.prune.out"; - public static final String RELATEDNESS_THRESHOLDS_FILENAME = "relatedness_thresholds.tsv"; - - private static final String RELATEDNESS_POP_FREQ_FILE_MSG = "Population frequency file"; - private static final String RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG = "Population exclude variant file"; - private static final String RELATEDNESS_THRESHOLDS_FILE_MSG = "Thresholds file"; - @ToolParams protected final FamilyQcAnalysisParams analysisParams = new FamilyQcAnalysisParams(); @@ -86,38 +75,8 @@ protected void check() throws Exception { super.check(); checkParameters(analysisParams, getStudy(), catalogManager, token); - // Get paths from external files - FamilyQcRelatednessAnalysisParams relatednessParams = analysisParams.getRelatednessParams(); - - // Get relatedness population frequency - if (relatednessParams != null && StringUtils.isNotEmpty(relatednessParams.getPopulationFrequencyFile())) { - Path path = checkFileParameter(relatednessParams.getPopulationFrequencyFile(), RELATEDNESS_POP_FREQ_FILE_MSG, getStudy(), - catalogManager, getToken()); - analysisParams.getRelatednessParams().setPopulationFrequencyFile(path.toAbsolutePath().toString()); - } else { - Path path = getExternalFilePath(RelatednessAnalysis.ID, RELATEDNESS_POP_FREQ_FILENAME); - analysisParams.getRelatednessParams().setPopulationFrequencyFile(path.toAbsolutePath().toString()); - } - - // Get relatedness population exclude variant - if (relatednessParams != null && StringUtils.isNotEmpty(relatednessParams.getPopulationExcludeVariantsFile())) { - Path path = checkFileParameter(relatednessParams.getPopulationExcludeVariantsFile(), RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG, - getStudy(), catalogManager, getToken()); - analysisParams.getRelatednessParams().setPopulationExcludeVariantsFile(path.toAbsolutePath().toString()); - } else { - Path path = getExternalFilePath(RelatednessAnalysis.ID, RELATEDNESS_POP_EXCLUDE_VAR_FILENAME); - analysisParams.getRelatednessParams().setPopulationExcludeVariantsFile(path.toAbsolutePath().toString()); - } - - // Get relatedness thresholds - if (relatednessParams != null && StringUtils.isNotEmpty(relatednessParams.getPopulationFrequencyFile())) { - Path path = checkFileParameter(relatednessParams.getThresholdsFile(), RELATEDNESS_THRESHOLDS_FILE_MSG, getStudy(), - catalogManager, getToken()); - analysisParams.getRelatednessParams().setThresholdsFile(path.toAbsolutePath().toString()); - } else { - Path path = getExternalFilePath(RelatednessAnalysis.ID, RELATEDNESS_THRESHOLDS_FILENAME); - analysisParams.getRelatednessParams().setThresholdsFile(path.toAbsolutePath().toString()); - } + // Update paths from relatedness external files + updateRelatednessFilePaths(analysisParams.getRelatednessParams()); } @Override @@ -138,7 +97,8 @@ protected void run() throws ToolException { // Decide if quality control has to be performed // - by checking the quality control status, if it is READY means it has been computed previously, and // - by checking the flag overwrite - if (!performQualityControl(family, analysisParams.getOverwrite())) { + if (family.getInternal() != null + && !performQualityControl(family.getInternal().getQualityControlStatus(), analysisParams.getOverwrite())) { // Quality control does not have to be performed for this family continue; } @@ -157,7 +117,7 @@ protected void run() throws ToolException { // Export family variants (VCF format) // Create variant query - String gt = getSampleIds(family, study, catalogManager, token).stream().map(s -> s + ":0/0,0/1,1/1") + String gt = getNoSomaticSampleIds(family, study, catalogManager, token).stream().map(s -> s + ":0/0,0/1,1/1") .collect(Collectors.joining(";")); Query query = new Query() .append(VariantQueryParam.STUDY.key(), study) @@ -259,7 +219,7 @@ public static void checkParameters(FamilyQcAnalysisParams params, String studyId family = familyResult.first(); // Check number of samples - List sampleIds = getSampleIds(family, studyId, catalogManager, token); + List sampleIds = getNoSomaticSampleIds(family, studyId, catalogManager, token); if (sampleIds.size() < 2) { errors.put(familyId, "Too few samples found (" + sampleIds.size() + ") for that family members; minimum is 2 member" + " samples"); @@ -276,63 +236,8 @@ public static void checkParameters(FamilyQcAnalysisParams params, String studyId e -> "Family ID " + e.getKey() + ": " + e.getValue()).collect(Collectors.toList()), ",")); } - // Check external files: pop. freq. file, pop. exclude var. file and threadshold file - if (params.getRelatednessParams() != null) { - FamilyQcRelatednessAnalysisParams relatednessParams = params.getRelatednessParams(); - if (StringUtils.isNotEmpty(relatednessParams.getPopulationFrequencyFile())) { - checkFileParameter(relatednessParams.getPopulationFrequencyFile(), RELATEDNESS_POP_FREQ_FILE_MSG, studyId, catalogManager, - token); - } - if (StringUtils.isNotEmpty(relatednessParams.getPopulationExcludeVariantsFile())) { - checkFileParameter(relatednessParams.getPopulationExcludeVariantsFile(), RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG, studyId, - catalogManager, token); - } - if (StringUtils.isNotEmpty(relatednessParams.getThresholdsFile())) { - checkFileParameter(relatednessParams.getThresholdsFile(), RELATEDNESS_THRESHOLDS_FILE_MSG, studyId, catalogManager, token); - } - } - } - - private static List getSampleIds(Family family, String studyId, CatalogManager catalogManager, String token) - throws CatalogException { - // Get list of individual IDs - List individualIds = family.getMembers().stream().map(m -> m.getId()).collect(Collectors.toList()); - - Query query = new Query(IndividualDBAdaptor.QueryParams.ID.key(), individualIds); - QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "samples"); - - List sampleIds = new ArrayList<>(); - OpenCGAResult individualResult = catalogManager.getIndividualManager().search(studyId, query, queryOptions, token); - for (Individual individual : individualResult.getResults()) { - if (CollectionUtils.isNotEmpty(individual.getSamples())) { - for (Sample sample : individual.getSamples()) { - if (!sample.isSomatic()) { - // We take the first no somatic sample for each individual - sampleIds.add(sample.getId()); - break; - } - } - } - } - return sampleIds; - } - - private boolean performQualityControl(Family family, Boolean overwrite) { - boolean performQc; - if (Boolean.TRUE.equals(overwrite)) { - performQc = true; - } else if (family.getInternal() != null && family.getInternal().getQualityControlStatus() != null) { - String statusId = family.getInternal().getQualityControlStatus().getId(); - performQc = !(statusId.equals(COMPUTING) || statusId.equals(READY)); - } else { - performQc = true; - } - - if (performQc) { - // Second, set status to COMPUTING - } - - return performQc; + // Check relatedness files: pop. freq. file, pop. exclude var. file and threshold file + checkRelatednessParameters(params.getRelatednessParams(), studyId, catalogManager, token); } private void updateFamilyQualityControl(List families) throws ToolException { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcLocalAnalysisExecutor.java index ab0a8d092d5..8f2ff864f2b 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/FamilyVariantQcLocalAnalysisExecutor.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.DockerUtils; import org.opencb.opencga.analysis.StorageToolExecutor; +import org.opencb.opencga.analysis.utils.VariantQcAnalysisExecutorUtils; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureLocalAnalysisExecutor; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; @@ -50,48 +51,22 @@ import java.util.List; import java.util.stream.Collectors; +import static org.opencb.opencga.analysis.utils.VariantQcAnalysisExecutorUtils.CONFIG_FILENAME; + @ToolExecutor(id="opencga-local", tool = FamilyVariantQcAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class FamilyVariantQcLocalAnalysisExecutor extends FamilyVariantQcAnalysisExecutor implements StorageToolExecutor { @Override public void run() throws ToolExecutorException { - // Run the Python script responsible for performing the family QC analyses - // variant_qc.main.py --vcf-file xxx --info-json xxx --bam-file xxx --qc-type xxx --config xxx --output-dir xxx - - // Build command line to run Python script via docker image - Path opencgaHome = getOpencgaHome(); - + Path configPath = getOutDir().resolve(CONFIG_FILENAME); + ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(FamilyQcAnalysisParams.class); try { - // Input binding - List> inputBindings = new ArrayList<>(); - inputBindings.add(new AbstractMap.SimpleEntry<>(opencgaHome.resolve("analysis/variant-qc").toAbsolutePath().toString(), - "/script")); - - // Output binding - AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(getOutDir().toAbsolutePath().toString(), - "/jobdir"); - - String configFilename = "config.json"; - ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(FamilyQcAnalysisParams.class); - objectWriter.writeValue(getOutDir().resolve(configFilename).toFile(), getQcParams()); - - String params = "python3 /script/variant_qc.main.py" - + " --vcf-file " + StringUtils.join(getVcfPaths().stream().map(p -> p.toAbsolutePath().toString().replace( - getOutDir().toAbsolutePath().toString(), "/jobdir")).collect(Collectors.toList()), ",") - + " --info-json " + StringUtils.join(getJsonPaths().stream().map(p -> p.toAbsolutePath().toString().replace( - getOutDir().toAbsolutePath().toString(), "/jobdir")).collect(Collectors.toList()), ",") - + " --qc-type family" - + " --config /jobdir/" + configFilename - + " --output-dir /jobdir"; - - - // Execute Pythong script in docker - String dockerImage = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion(); - - DockerUtils.run(dockerImage, inputBindings, outputBinding, params, null); + objectWriter.writeValue(configPath.toFile(), getQcParams()); } catch (IOException e) { throw new ToolExecutorException(e); } + + VariantQcAnalysisExecutorUtils.run(getVcfPaths(), getJsonPaths(), configPath, getOutDir(), getOpencgaHome()); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java index 5259874e600..0850c6524d7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcAnalysis.java @@ -16,18 +16,49 @@ package org.opencb.opencga.analysis.individual.qc; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.variant.avro.VariantType; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.variant.qc.VariantQcAnalysis; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.FamilyQcAnalysisParams; +import org.opencb.opencga.core.models.common.QualityControlStatus; +import org.opencb.opencga.core.models.family.FamilyQualityControl; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.individual.IndividualQualityControl; +import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; +import org.opencb.opencga.core.tools.variant.IndividualVariantQcAnalysisExecutor; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_FAMILIES; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +import static org.opencb.opencga.analysis.utils.VariantQcAnalysisExecutorUtils.QC_JSON_EXTENSION; +import static org.opencb.opencga.core.models.common.InternalStatus.READY; +import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING; +import static org.opencb.opencga.core.models.common.QualityControlStatus.NONE; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_INDIVIDUALS; +import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.JSON; +import static org.opencb.opencga.storage.core.variant.io.VariantWriterFactory.VariantOutputFormat.VCF_GZ; @Tool(id = IndividualVariantQcAnalysis.ID, resource = Enums.Resource.SAMPLE, description = IndividualVariantQcAnalysis.DESCRIPTION) public class IndividualVariantQcAnalysis extends VariantQcAnalysis { @@ -36,24 +67,197 @@ public class IndividualVariantQcAnalysis extends VariantQcAnalysis { public static final String DESCRIPTION = "Run quality control (QC) for a given individual. It includes inferred sex and " + " mendelian errors (UDP)"; + // Set of individuals with known parents, i.e., trios + private Set trios = new HashSet<>(); + @ToolParams protected final IndividualQcAnalysisParams analysisParams = new IndividualQcAnalysisParams(); @Override protected void check() throws Exception { super.check(); + checkParameters(analysisParams, getStudy(), catalogManager, token); + + // Check for the presence of trios to compute relatedness + for (String individualId : analysisParams.getIndividuals()) { + // Get individual + Individual individual = catalogManager.getIndividualManager().get(study, individualId, QueryOptions.empty(), token).first(); + if (individual.getFather() != null && individual.getMother() != null) { + trios.add(individual); + } + } + if (CollectionUtils.isNotEmpty(trios)) { + updateRelatednessFilePaths(analysisParams.getRelatednessParams()); + } } @Override protected void run() throws ToolException { - // Export variants (VCF format) + setUpStorageEngineExecutor(study); + + List individuals = new ArrayList<>(); + LinkedList individualVcfPaths = new LinkedList<>(); + LinkedList individualJsonPaths = new LinkedList<>(); + + try { + ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(Individual.class); + for (String individualId : analysisParams.getIndividuals()) { + // Get individual + Individual individual = catalogManager.getIndividualManager().get(study, individualId, QueryOptions.empty(), token).first(); + + // Decide if quality control has to be performed + // - by checking the quality control status, if it is READY means it has been computed previously, and + // - by checking the flag overwrite + if (individual.getInternal() != null + && !performQualityControl(individual.getInternal().getQualityControlStatus(), analysisParams.getOverwrite())) { + // Quality control does not have to be performed for this individual + continue; + } + + // Set quality control status to COMPUTING to prevent multiple individual QCs from running simultaneously + // for the same individual + if (!setComputingStatus(individual)) { + continue; + } + + // Create directory to save variants and individual files + Path individualOutPath = Files.createDirectories(getOutDir().resolve(individualId)); + if (!Files.exists(individualOutPath)) { + throw new ToolException("Error creating directory: " + individualOutPath); + } + + // Export individual variants (VCF format) + // Create the query based on whether a trio is present or not + Query query; + if (trios.contains(individual)) { + String child = individual.getSamples().get(0).getId(); + String father = individual.getFather().getSamples().get(0).getId(); + String mother = individual.getMother().getSamples().get(0).getId(); + query = new Query() + .append(VariantQueryParam.SAMPLE.key(), child + ":0/1,1/1") + .append(VariantQueryParam.INCLUDE_SAMPLE.key(), child + "," + father + "," + mother) + .append(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), "GT"); + } else { + // Create variant query + String gt = getNoSomaticSampleIds(individual).stream().map(s -> s + ":0/0,0/1,1/1") + .collect(Collectors.joining(";")); + query = new Query() + .append(VariantQueryParam.GENOTYPE.key(), gt); + } + query.append(VariantQueryParam.STUDY.key(), study) + .append(VariantQueryParam.TYPE.key(), VariantType.SNV) + .append(VariantQueryParam.REGION.key(), Arrays.asList("1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22" + .split(","))); + + // Create query options + QueryOptions queryOptions = new QueryOptions().append(QueryOptions.INCLUDE, "id,studies.samples"); + + // Export to VCF.GZ format + String basename = individualOutPath.resolve(individualId).toAbsolutePath().toString(); + getVariantStorageManager().exportData(basename, VCF_GZ, null, query, queryOptions, token); + + // Check VCF file + Path individualVcfPath = Paths.get(basename + "." + VCF_GZ.getExtension()); + if (!Files.exists(individualVcfPath)) { + throw new ToolException("Something wrong happened when exporting VCF file for individual ID '" + individualId + + "'. VCF file " + individualVcfPath + " was not created. Export query = " + query.toJson() + "; export query" + + " options = " + queryOptions.toJson()); + } + individualVcfPaths.add(individualVcfPath); + + // Export individual (JSON format) + Path individualJsonPath = Paths.get(basename + "." + JSON.getExtension()); + objectWriter.writeValue(individualJsonPath.toFile(), individual); - // Export catalog info (JSON format) + // Check VCF file + if (!Files.exists(individualJsonPath)) { + throw new ToolException("Something wrong happened when saving JSON file for individual ID '" + individualId + + "'. JSON file " + individualJsonPath + " was not created."); + } + individualJsonPaths.add(individualJsonPath); - // Execute Python script: - // variant_qc.main.py --vcf-file xxx --info-json xxx --bam-file xxx --qc-type xxx --config xxx --output-dir xxx + // Add family to the list + individuals.add(individual); + } + } catch (CatalogException | IOException | StorageEngineException e) { + throw new ToolException(e); + } - // Parse results + // Get executor to execute Python script that computes the family QC + IndividualVariantQcAnalysisExecutor executor = getToolExecutor(IndividualVariantQcAnalysisExecutor.class); + executor.setVcfPaths(individualVcfPaths) + .setJsonPaths(individualJsonPaths) + .setQcParams(analysisParams); + + // Step by step + step(executor::execute); + + // Parse Python script results + if (!Boolean.TRUE.equals(analysisParams.getSkipIndex())) { + updateIndividualQualityControl(individuals); + } + } + + private boolean setComputingStatus(Individual individual) throws ToolException { + try { + QualityControlStatus qcStatus = new QualityControlStatus(COMPUTING, "Performing individual QC"); + IndividualUpdateParams updateParams = new IndividualUpdateParams().setQualityControlStatus(qcStatus); + catalogManager.getIndividualManager().update(getStudy(), individual.getId(), updateParams, null, token); + } catch (CatalogException e) { + String msg = "Could not set status to COMPUTING before performing the QC for the individual '" + individual.getId() + "': " + + e.getMessage(); + logger.error(msg); + addError(new ToolException(msg, e)); + return false; + } + return true; + } + + private void updateIndividualQualityControl(List individuals) throws ToolException { + ObjectMapper objectMapper = JacksonUtils.getDefaultObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + ObjectReader objectReader = JacksonUtils.getDefaultObjectMapper().readerFor(FamilyQualityControl.class); + + for (Individual individual : individuals) { + IndividualQualityControl individualQc; + QualityControlStatus qcStatus; + + // Check output file + String msg; + Path qcPath = getOutDir().resolve(individual.getId()).resolve(individual.getId() + QC_JSON_EXTENSION); + if (!Files.exists(qcPath)) { + msg = "Quality control error for individual " + individual.getId() + ": file " + qcPath.getFileName() + " not found"; + individualQc = new IndividualQualityControl(); + qcStatus = new QualityControlStatus(NONE, msg); + addError(new ToolException(msg)); + logger.error(msg); + } else { + try { + msg = "Computed successfully for individual " + individual.getId(); + individualQc = objectReader.readValue(qcPath.toFile()); + qcStatus = new QualityControlStatus(READY, msg); + logger.info(msg); + } catch (IOException e) { + msg = "Quality control error for individual " + individual.getId() + ": error parsing JSON file " + qcPath.getFileName(); + individualQc = new IndividualQualityControl(); + qcStatus = new QualityControlStatus(NONE, msg); + addError(e); + logger.error(msg); + } + } + + try { + // Update catalog: quality control and status + IndividualUpdateParams updateParams = new IndividualUpdateParams() + .setQualityControl(individualQc) + .setQualityControlStatus(qcStatus); + catalogManager.getIndividualManager().update(getStudy(), individual.getId(), updateParams, null, token); + } catch (CatalogException e) { + logger.error("Could not update quality control in OpenCGA catalog for individual {}: {}", individual.getId(), + e.getMessage()); + addError(e); + } + } } public static void checkParameters(IndividualQcAnalysisParams params, String studyId, CatalogManager catalogManager, String token) @@ -63,5 +267,42 @@ public static void checkParameters(IndividualQcAnalysisParams params, String stu // Check permissions checkPermissions(WRITE_INDIVIDUALS, studyId, catalogManager, token); + + // Check individuals in catalog + // Collect possible errors in a map where key is the family ID and value is the error + Map errors = new HashMap<>(); + for (String individualId : params.getIndividuals()) { + // Get Individual from catalog + Individual individual; + try { + OpenCGAResult individualResult = catalogManager.getIndividualManager().get(studyId, individualId, + QueryOptions.empty(), token); + if (individualResult.getNumResults() == 0) { + errors.put(individualId, "Individual not found"); + } else if (individualResult.getNumResults() > 1) { + errors.put(individualId, "More than one individual found"); + } else { + individual = individualResult.first(); + + // Check number of samples + List sampleIds = getNoSomaticSampleIds(individual); + if (sampleIds.size() < 1) { + errors.put(individualId, "Too few samples found (" + sampleIds.size() + ") for that individual; minimum is 1" + + " sample"); + } + } + } catch (CatalogException e) { + errors.put(individualId, Arrays.toString(e.getStackTrace())); + } + } + + // Check error + if (MapUtils.isNotEmpty(errors)) { + throw new ToolException("Found the following error for individual IDs: " + StringUtils.join(errors.entrySet().stream().map( + e -> "Individual ID " + e.getKey() + ": " + e.getValue()).collect(Collectors.toList()), ",")); + } + + // Check relatedness files: pop. freq. file, pop. exclude var. file and threshold file + checkRelatednessParameters(params.getRelatednessParams(), studyId, catalogManager, token); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcLocalAnalysisExecutor.java new file mode 100644 index 00000000000..ddf00c11cb9 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/qc/IndividualVariantQcLocalAnalysisExecutor.java @@ -0,0 +1,49 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.analysis.individual.qc; + +import com.fasterxml.jackson.databind.ObjectWriter; +import org.opencb.opencga.analysis.StorageToolExecutor; +import org.opencb.opencga.analysis.utils.VariantQcAnalysisExecutorUtils; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; +import org.opencb.opencga.core.tools.variant.IndividualVariantQcAnalysisExecutor; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.opencb.opencga.analysis.utils.VariantQcAnalysisExecutorUtils.CONFIG_FILENAME; + +@ToolExecutor(id="opencga-local", tool = IndividualVariantQcAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, + source = ToolExecutor.Source.STORAGE) +public class IndividualVariantQcLocalAnalysisExecutor extends IndividualVariantQcAnalysisExecutor implements StorageToolExecutor { + + @Override + public void run() throws ToolExecutorException { + Path configPath = getOutDir().resolve(CONFIG_FILENAME); + ObjectWriter objectWriter = JacksonUtils.getDefaultObjectMapper().writerFor(IndividualQcAnalysisParams.class); + try { + objectWriter.writeValue(configPath.toFile(), getQcParams()); + } catch (IOException e) { + throw new ToolExecutorException(e); + } + + VariantQcAnalysisExecutorUtils.run(getVcfPaths(), getJsonPaths(), configPath, getOutDir(), getOpencgaHome()); + } +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java index 15bd1383174..d40725f610a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/models/IndividualPrivateUpdateParams.java @@ -6,6 +6,7 @@ import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.sample.SampleReferenceParam; @@ -26,10 +27,12 @@ public IndividualPrivateUpdateParams(String id, String name, IndividualReference String dateOfBirth, IndividualProperty.KaryotypicSex karyotypicSex, IndividualProperty.LifeStatus lifeStatus, List samples, List annotationSets, List phenotypes, List disorders, - StatusParams status, IndividualQualityControl qualityControl, Map attributes, + StatusParams status, IndividualQualityControl qualityControl, + QualityControlStatus qualityControlStatus, Map attributes, IndividualInternal internal) { super(id, name, father, mother, creationDate, modificationDate, parentalConsanguinity, location, sex, ethnicity, population, - dateOfBirth, karyotypicSex, lifeStatus, samples, annotationSets, phenotypes, disorders, status, qualityControl, attributes); + dateOfBirth, karyotypicSex, lifeStatus, samples, annotationSets, phenotypes, disorders, status, qualityControl, + qualityControlStatus, attributes); this.internal = internal; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/utils/VariantQcAnalysisExecutorUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/utils/VariantQcAnalysisExecutorUtils.java new file mode 100644 index 00000000000..c0757693a77 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/utils/VariantQcAnalysisExecutorUtils.java @@ -0,0 +1,76 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.analysis.utils; + +import com.fasterxml.jackson.databind.ObjectWriter; +import org.apache.commons.lang.StringUtils; +import org.opencb.commons.utils.DockerUtils; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.models.variant.FamilyQcAnalysisParams; +import org.opencb.opencga.core.tools.ToolParams; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +public class VariantQcAnalysisExecutorUtils { + + public static String CONFIG_FILENAME = "config.json"; + public static String QC_JSON_EXTENSION = ".qc.json"; + + public static void run(LinkedList vcfPaths, LinkedList jsonPaths, Path configPath, Path outDir, Path opencgaHome) + throws ToolExecutorException { + // Run the Python script responsible for performing the family QC analyses + // variant_qc.main.py --vcf-file xxx --info-json xxx --bam-file xxx --qc-type xxx --config xxx --output-dir xxx + + // Build command line to run Python script via docker image + + try { + // Input binding + List> inputBindings = new ArrayList<>(); + inputBindings.add(new AbstractMap.SimpleEntry<>(opencgaHome.resolve("analysis/variant-qc").toAbsolutePath().toString(), + "/script")); + + // Output binding + AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(outDir.toAbsolutePath().toString(), + "/jobdir"); + + String params = "python3 /script/variant_qc.main.py" + + " --vcf-file " + StringUtils.join(vcfPaths.stream().map(p -> p.toAbsolutePath().toString().replace( + outDir.toAbsolutePath().toString(), "/jobdir")).collect(Collectors.toList()), ",") + + " --info-json " + StringUtils.join(jsonPaths.stream().map(p -> p.toAbsolutePath().toString().replace( + outDir.toAbsolutePath().toString(), "/jobdir")).collect(Collectors.toList()), ",") + + " --qc-type family" + + " --config /jobdir/" + configPath.getFileName() + + " --output-dir /jobdir"; + + + // Execute Pythong script in docker + String dockerImage = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion(); + + DockerUtils.run(dockerImage, inputBindings, outputBinding, params, null); + } catch (IOException e) { + throw new ToolExecutorException(e); + } + } +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java index 0fc97607fc3..1569880316c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/qc/VariantQcAnalysis.java @@ -16,30 +16,53 @@ package org.opencb.opencga.analysis.variant.qc; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; +import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis; +import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.common.QualityControlStatus; +import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; +import org.opencb.opencga.core.models.variant.QcRelatednessAnalysisParams; +import org.opencb.opencga.core.response.OpenCGAResult; import java.io.IOException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import static org.opencb.opencga.core.models.common.InternalStatus.READY; +import static org.opencb.opencga.core.models.common.QualityControlStatus.COMPUTING; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_SAMPLES; public class VariantQcAnalysis extends OpenCgaToolScopeStudy { + protected static final String RELATEDNESS_POP_FREQ_FILENAME = "autosomes_1000G_QC_prune_in.frq"; + protected static final String RELATEDNESS_POP_EXCLUDE_VAR_FILENAME = "autosomes_1000G_QC.prune.out"; + protected static final String RELATEDNESS_THRESHOLDS_FILENAME = "relatedness_thresholds.tsv"; + + protected static final String RELATEDNESS_POP_FREQ_FILE_MSG = "Population frequency file"; + protected static final String RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG = "Population exclude variant file"; + protected static final String RELATEDNESS_THRESHOLDS_FILE_MSG = "Thresholds file"; + @Override protected void check() throws Exception { super.check(); @@ -96,6 +119,53 @@ protected static void checkPermissions(StudyPermissions.Permissions permissions, } } + protected static void checkRelatednessParameters(QcRelatednessAnalysisParams relatednessParams, String studyId, + CatalogManager catalogManager, String token) throws ToolException { + if (StringUtils.isNotEmpty(relatednessParams.getPopulationFrequencyFile())) { + checkFileParameter(relatednessParams.getPopulationFrequencyFile(), RELATEDNESS_POP_FREQ_FILE_MSG, studyId, catalogManager, + token); + } + if (StringUtils.isNotEmpty(relatednessParams.getPopulationExcludeVariantsFile())) { + checkFileParameter(relatednessParams.getPopulationExcludeVariantsFile(), RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG, studyId, + catalogManager, token); + } + if (StringUtils.isNotEmpty(relatednessParams.getThresholdsFile())) { + checkFileParameter(relatednessParams.getThresholdsFile(), RELATEDNESS_THRESHOLDS_FILE_MSG, studyId, catalogManager, token); + } + } + + protected void updateRelatednessFilePaths(QcRelatednessAnalysisParams relatednessParams) throws ToolException { + // Get relatedness population frequency + if (relatednessParams != null && StringUtils.isNotEmpty(relatednessParams.getPopulationFrequencyFile())) { + Path path = checkFileParameter(relatednessParams.getPopulationFrequencyFile(), RELATEDNESS_POP_FREQ_FILE_MSG, getStudy(), + catalogManager, getToken()); + relatednessParams.setPopulationFrequencyFile(path.toAbsolutePath().toString()); + } else { + Path path = getExternalFilePath(RelatednessAnalysis.ID, RELATEDNESS_POP_FREQ_FILENAME); + relatednessParams.setPopulationFrequencyFile(path.toAbsolutePath().toString()); + } + + // Get relatedness population exclude variant + if (relatednessParams != null && StringUtils.isNotEmpty(relatednessParams.getPopulationExcludeVariantsFile())) { + Path path = checkFileParameter(relatednessParams.getPopulationExcludeVariantsFile(), RELATEDNESS_POP_EXCLUDE_VAR_FILE_MSG, + getStudy(), catalogManager, getToken()); + relatednessParams.setPopulationExcludeVariantsFile(path.toAbsolutePath().toString()); + } else { + Path path = getExternalFilePath(RelatednessAnalysis.ID, RELATEDNESS_POP_EXCLUDE_VAR_FILENAME); + relatednessParams.setPopulationExcludeVariantsFile(path.toAbsolutePath().toString()); + } + + // Get relatedness thresholds + if (relatednessParams != null && StringUtils.isNotEmpty(relatednessParams.getPopulationFrequencyFile())) { + Path path = checkFileParameter(relatednessParams.getThresholdsFile(), RELATEDNESS_THRESHOLDS_FILE_MSG, getStudy(), + catalogManager, getToken()); + relatednessParams.setThresholdsFile(path.toAbsolutePath().toString()); + } else { + Path path = getExternalFilePath(RelatednessAnalysis.ID, RELATEDNESS_THRESHOLDS_FILENAME); + relatednessParams.setThresholdsFile(path.toAbsolutePath().toString()); + } + } + protected static Path checkFileParameter(String fileId, String msg, String studyId, CatalogManager catalogManager, String token) throws ToolException { if (StringUtils.isEmpty(fileId)) { @@ -114,6 +184,38 @@ protected static Path checkFileParameter(String fileId, String msg, String study return path; } + protected static List getNoSomaticSampleIds(Family family, String studyId, CatalogManager catalogManager, String token) + throws CatalogException { + // Get list of individual IDs + List individualIds = family.getMembers().stream().map(m -> m.getId()).collect(Collectors.toList()); + + Query query = new Query(IndividualDBAdaptor.QueryParams.ID.key(), individualIds); + QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "samples"); + + List sampleIds = new ArrayList<>(); + OpenCGAResult individualResult = catalogManager.getIndividualManager().search(studyId, query, queryOptions, token); + for (Individual individual : individualResult.getResults()) { + if (CollectionUtils.isNotEmpty(individual.getSamples())) { + sampleIds.addAll(getNoSomaticSampleIds(individual)); + } + } + return sampleIds; + } + + protected static List getNoSomaticSampleIds(Individual individual) { + List sampleIds = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(individual.getSamples())) { + for (Sample sample : individual.getSamples()) { + if (!sample.isSomatic()) { + // We take the first no somatic sample for each individual + sampleIds.add(sample.getId()); + break; + } + } + } + return sampleIds; + } + protected Path getExternalFilePath(String analysisId, String resourceName) throws ToolException { URL url = null; try { @@ -156,4 +258,17 @@ protected Path downloadExternalFileAtResources(String analysisId, String resourc } return resourcePath.resolve(resourceName); } + + protected boolean performQualityControl(QualityControlStatus qcStatus, Boolean overwrite) { + boolean performQc; + if (Boolean.TRUE.equals(overwrite)) { + performQc = true; + } else if (qcStatus != null) { + String statusId = qcStatus.getId(); + performQc = !(statusId.equals(COMPUTING) || statusId.equals(READY)); + } else { + performQc = true; + } + return performQc; + } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 2f7f34589c9..64ea16645d2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -34,7 +34,6 @@ import org.opencb.opencga.core.models.variant.CircosAnalysisParams; import org.opencb.opencga.core.models.variant.CohortVariantStatsAnalysisParams; import org.opencb.opencga.core.models.variant.FamilyQcAnalysisParams; -import org.opencb.opencga.core.models.variant.FamilyQcRelatednessAnalysisParams; import org.opencb.opencga.core.models.variant.GatkWrapperParams; import org.opencb.opencga.core.models.variant.GenomePlotAnalysisParams; import org.opencb.opencga.core.models.variant.GwasAnalysisParams; @@ -45,6 +44,7 @@ import org.opencb.opencga.core.models.variant.MendelianErrorAnalysisParams; import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.models.variant.PlinkWrapperParams; +import org.opencb.opencga.core.models.variant.QcRelatednessAnalysisParams; import org.opencb.opencga.core.models.variant.RelatednessAnalysisParams; import org.opencb.opencga.core.models.variant.RvtestsWrapperParams; import org.opencb.opencga.core.models.variant.SampleEligibilityAnalysisParams; @@ -944,9 +944,16 @@ private RestResponse runIndividualQc() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), IndividualQcAnalysisParams.class); } else { ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "individuals",commandOptions.individuals, true); putNestedIfNotEmpty(beanParams, "individual",commandOptions.individual, true); putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "relatednessParams.populationFrequencyFile",commandOptions.relatednessParamsPopulationFrequencyFile, true); + putNestedIfNotEmpty(beanParams, "relatednessParams.populationExcludeVariantsFile",commandOptions.relatednessParamsPopulationExcludeVariantsFile, true); + putNestedIfNotEmpty(beanParams, "relatednessParams.thresholdsFile",commandOptions.relatednessParamsThresholdsFile, true); putNestedIfNotEmpty(beanParams, "inferredSexMethod",commandOptions.inferredSexMethod, true); + putNestedIfNotNull(beanParams, "skip",commandOptions.skip, true); + putNestedIfNotNull(beanParams, "skipIndex",commandOptions.skipIndex, true); + putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); individualQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 495341543d6..ab3e85c66e3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -20,6 +20,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.individual.Individual; @@ -447,6 +448,12 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "status.name",commandOptions.statusName, true); putNestedIfNotEmpty(beanParams, "status.description",commandOptions.statusDescription, true); putNestedIfNotNull(beanParams, "qualityControl.files",commandOptions.qualityControlFiles, true); + putNestedIfNotEmpty(beanParams, "qualityControlStatus.id",commandOptions.qualityControlStatusId, true); + putNestedIfNotEmpty(beanParams, "qualityControlStatus.name",commandOptions.qualityControlStatusName, true); + putNestedIfNotEmpty(beanParams, "qualityControlStatus.description",commandOptions.qualityControlStatusDescription, true); + putNestedIfNotEmpty(beanParams, "qualityControlStatus.date",commandOptions.qualityControlStatusDate, true); + putNestedIfNotEmpty(beanParams, "qualityControlStatus.version",commandOptions.qualityControlStatusVersion, true); + putNestedIfNotEmpty(beanParams, "qualityControlStatus.commit",commandOptions.qualityControlStatusCommit, true); putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); individualUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 235b48755d6..db9e7b72063 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -1294,15 +1294,36 @@ public class RunIndividualQcCommandOptions { @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) public Boolean jobDryRun; + @Parameter(names = {"--individuals"}, description = "List of individual IDs", required = false, arity = 1) + public String individuals; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; @Parameter(names = {"--sample"}, description = "Sample ID (required when the individual has multiple samples)", required = false, arity = 1) public String sample; + @Parameter(names = {"--relatedness-params-population-frequency-file"}, description = "Population frequencies file ID for relatedness analysis", required = false, arity = 1) + public String relatednessParamsPopulationFrequencyFile; + + @Parameter(names = {"--relatedness-params-population-exclude-variants-file"}, description = "Population exclude variants file ID for relatedness analysis", required = false, arity = 1) + public String relatednessParamsPopulationExcludeVariantsFile; + + @Parameter(names = {"--relatedness-params-thresholds-file"}, description = "Threshold file ID for relatedness analysis", required = false, arity = 1) + public String relatednessParamsThresholdsFile; + @Parameter(names = {"--inferred-sex-method"}, description = "Inferred sex method. Valid values: CoverageRatio", required = false, arity = 1) public String inferredSexMethod = "CoverageRatio"; + @Parameter(names = {"--skip"}, description = "Individual QC analysis to skip. Valid values are: inferred-sex, mendelian-errors", required = false, arity = 1) + public String skip; + + @Parameter(names = {"--skip-index"}, description = "Do not save the computed quality control in catalog", required = false, arity = 1) + public Boolean skipIndex; + + @Parameter(names = {"--overwrite"}, description = "Overwrite quality control in catalog", required = false, arity = 1) + public Boolean overwrite; + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java index 8fdefbd9e91..0a49adc5604 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/IndividualsCommandOptions.java @@ -662,6 +662,24 @@ public class UpdateCommandOptions { @Parameter(names = {"--quality-control-files"}, description = "File IDs related to the quality control", required = false, arity = 1) public String qualityControlFiles; + @Parameter(names = {"--quality-control-status-id"}, description = "The body web service id parameter", required = false, arity = 1) + public String qualityControlStatusId; + + @Parameter(names = {"--quality-control-status-name"}, description = "The body web service name parameter", required = false, arity = 1) + public String qualityControlStatusName; + + @Parameter(names = {"--quality-control-status-description"}, description = "The body web service description parameter", required = false, arity = 1) + public String qualityControlStatusDescription; + + @Parameter(names = {"--quality-control-status-date"}, description = "The body web service date parameter", required = false, arity = 1) + public String qualityControlStatusDate; + + @Parameter(names = {"--quality-control-status-version"}, description = "The body web service version parameter", required = false, arity = 1) + public String qualityControlStatusVersion; + + @Parameter(names = {"--quality-control-status-commit"}, description = "The body web service commit parameter", required = false, arity = 1) + public String qualityControlStatusCommit; + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyUpdateParams.java index 07c51dead21..ba95c50132a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyUpdateParams.java @@ -94,7 +94,8 @@ public Family toFamily() { ? members.stream().map(m -> new Individual().setId(m.getId()).setUuid(m.getUuid())).collect(Collectors.toList()) : null, creationDate, modificationDate, description, members != null ? members.size() : 0, 1, 1, annotationSets, - status != null ? status.toStatus() : null, new FamilyInternal(), Collections.emptyMap(), null, attributes); + status != null ? status.toStatus() : null, new FamilyInternal().setQualityControlStatus(qualityControlStatus), + Collections.emptyMap(), null, attributes); } @Override diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java index 71ad0edf262..b8ba37f0f49 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualUpdateParams.java @@ -25,6 +25,7 @@ import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleReferenceParam; @@ -59,6 +60,7 @@ public class IndividualUpdateParams { private List disorders; private StatusParams status; private IndividualQualityControl qualityControl; + private QualityControlStatus qualityControlStatus; private Map attributes; public IndividualUpdateParams() { @@ -70,7 +72,8 @@ public IndividualUpdateParams(String id, String name, IndividualReferenceParam f String dateOfBirth, IndividualProperty.KaryotypicSex karyotypicSex, IndividualProperty.LifeStatus lifeStatus, List samples, List annotationSets, List phenotypes, List disorders, - StatusParams status, IndividualQualityControl qualityControl, Map attributes) { + StatusParams status, IndividualQualityControl qualityControl, QualityControlStatus qualityControlStatus, + Map attributes) { this.id = id; this.name = name; this.father = father; @@ -91,6 +94,7 @@ public IndividualUpdateParams(String id, String name, IndividualReferenceParam f this.disorders = disorders; this.status = status; this.qualityControl = qualityControl; + this.qualityControlStatus = qualityControlStatus; this.attributes = attributes; } @@ -100,6 +104,10 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { this.annotationSets = null; ObjectMap params = new ObjectMap(getUpdateObjectMapper().writeValueAsString(this)); + if (params.containsKey("qualityControlStatus")) { + params.put("internal.qualityControlStatus", params.get("qualityControlStatus")); + params.remove("qualityControlStatus"); + } this.annotationSets = annotationSetList; if (this.annotationSets != null) { @@ -120,7 +128,8 @@ public Individual toIndividual() { samples != null ? samples.stream().map(s -> new Sample().setId(s.getId()).setUuid(s.getUuid())).collect(Collectors.toList()) : null, parentalConsanguinity != null && parentalConsanguinity, annotationSets, - status != null ? status.toStatus() : null, new IndividualInternal(), attributes); + status != null ? status.toStatus() : null, new IndividualInternal().setQualityControlStatus(qualityControlStatus), + attributes); } @Override @@ -146,6 +155,7 @@ public String toString() { sb.append(", disorders=").append(disorders); sb.append(", status=").append(status); sb.append(", qualityControl=").append(qualityControl); + sb.append(", qualityControlStatus=").append(qualityControlStatus); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); @@ -331,6 +341,15 @@ public IndividualUpdateParams setQualityControl(IndividualQualityControl quality return this; } + public QualityControlStatus getQualityControlStatus() { + return qualityControlStatus; + } + + public IndividualUpdateParams setQualityControlStatus(QualityControlStatus qualityControlStatus) { + this.qualityControlStatus = qualityControlStatus; + return this; + } + public Map getAttributes() { return attributes; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcAnalysisParams.java index 20427787d0f..3c174ff24e8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcAnalysisParams.java @@ -40,7 +40,7 @@ public class FamilyQcAnalysisParams extends ToolParams { private String relatednessMaf; @DataField(id = "relatednessParams", description = FieldConstants.FAMILY_QC_RELATEDNESS_DESCRIPTION) - private FamilyQcRelatednessAnalysisParams relatednessParams; + private QcRelatednessAnalysisParams relatednessParams; @DataField(id = "skipIndex", description = FieldConstants.QC_SKIP_INDEX_DESCRIPTION) private Boolean skipIndex; @@ -63,7 +63,7 @@ public FamilyQcAnalysisParams(String family, String relatednessMethod, String re } public FamilyQcAnalysisParams(List families, String family, String relatednessMethod, String relatednessMaf, - FamilyQcRelatednessAnalysisParams relatednessParams, Boolean skipIndex, Boolean overwrite, String outdir) { + QcRelatednessAnalysisParams relatednessParams, Boolean skipIndex, Boolean overwrite, String outdir) { this.families = families; this.family = family; this.relatednessMethod = relatednessMethod; @@ -125,11 +125,11 @@ public FamilyQcAnalysisParams setRelatednessMaf(String relatednessMaf) { return this; } - public FamilyQcRelatednessAnalysisParams getRelatednessParams() { + public QcRelatednessAnalysisParams getRelatednessParams() { return relatednessParams; } - public FamilyQcAnalysisParams setRelatednessParams(FamilyQcRelatednessAnalysisParams relatednessParams) { + public FamilyQcAnalysisParams setRelatednessParams(QcRelatednessAnalysisParams relatednessParams) { this.relatednessParams = relatednessParams; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java index 07e79d727e7..dc4e9136624 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/IndividualQcAnalysisParams.java @@ -35,6 +35,9 @@ public class IndividualQcAnalysisParams extends ToolParams { @DataField(id = "sample", description = FieldConstants.INDIVIDUAL_QC_SAMPLE_ID_DESCRIPTION, deprecated = true) private String sample; + @DataField(id = "relatednessParams", description = FieldConstants.FAMILY_QC_RELATEDNESS_DESCRIPTION) + private QcRelatednessAnalysisParams relatednessParams; + @Deprecated @DataField(id = "inferredSexMethod", description = FieldConstants.INFERRED_SEX_METHOD_DESCRIPTION, defaultValue = IndividualQcAnalysisExecutor.COVERAGE_RATIO_INFERRED_SEX_METHOD, deprecated = true) @@ -63,10 +66,14 @@ public IndividualQcAnalysisParams(String individual, String sample, String infer this.outdir = outdir; } - public IndividualQcAnalysisParams(String individual, String sample, List skip, Boolean skipIndex, Boolean overwrite, - String outdir) { + public IndividualQcAnalysisParams(List individuals, String individual, String sample, + QcRelatednessAnalysisParams relatednessParams, String inferredSexMethod, List skip, + Boolean skipIndex, Boolean overwrite, String outdir) { + this.individuals = individuals; this.individual = individual; this.sample = sample; + this.relatednessParams = relatednessParams; + this.inferredSexMethod = inferredSexMethod; this.skip = skip; this.skipIndex = skipIndex; this.overwrite = overwrite; @@ -76,8 +83,10 @@ public IndividualQcAnalysisParams(String individual, String sample, List @Override public String toString() { final StringBuilder sb = new StringBuilder("IndividualQcAnalysisParams{"); - sb.append("individual='").append(individual).append('\''); + sb.append("individuals=").append(individuals); + sb.append(", individual='").append(individual).append('\''); sb.append(", sample='").append(sample).append('\''); + sb.append(", relatednessParams=").append(relatednessParams); sb.append(", inferredSexMethod='").append(inferredSexMethod).append('\''); sb.append(", skip=").append(skip); sb.append(", skipIndex=").append(skipIndex); @@ -87,6 +96,15 @@ public String toString() { return sb.toString(); } + public List getIndividuals() { + return individuals; + } + + public IndividualQcAnalysisParams setIndividuals(List individuals) { + this.individuals = individuals; + return this; + } + public String getIndividual() { return individual; } @@ -105,17 +123,51 @@ public IndividualQcAnalysisParams setSample(String sample) { return this; } - @Deprecated + public QcRelatednessAnalysisParams getRelatednessParams() { + return relatednessParams; + } + + public IndividualQcAnalysisParams setRelatednessParams(QcRelatednessAnalysisParams relatednessParams) { + this.relatednessParams = relatednessParams; + return this; + } + public String getInferredSexMethod() { return inferredSexMethod; } - @Deprecated public IndividualQcAnalysisParams setInferredSexMethod(String inferredSexMethod) { this.inferredSexMethod = inferredSexMethod; return this; } + public List getSkip() { + return skip; + } + + public IndividualQcAnalysisParams setSkip(List skip) { + this.skip = skip; + return this; + } + + public Boolean getSkipIndex() { + return skipIndex; + } + + public IndividualQcAnalysisParams setSkipIndex(Boolean skipIndex) { + this.skipIndex = skipIndex; + return this; + } + + public Boolean getOverwrite() { + return overwrite; + } + + public IndividualQcAnalysisParams setOverwrite(Boolean overwrite) { + this.overwrite = overwrite; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcRelatednessAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/QcRelatednessAnalysisParams.java similarity index 81% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcRelatednessAnalysisParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/variant/QcRelatednessAnalysisParams.java index be5c2c93242..0ed8eadae4b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/FamilyQcRelatednessAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/QcRelatednessAnalysisParams.java @@ -19,7 +19,7 @@ import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; -public class FamilyQcRelatednessAnalysisParams { +public class QcRelatednessAnalysisParams { @DataField(id = "populationFrequencyFile", description = FieldConstants.FAMILY_QC_RELATEDNESS_POP_FREQ_FILE_DESCRIPTION) private String populationFrequencyFile; @@ -30,10 +30,10 @@ public class FamilyQcRelatednessAnalysisParams { @DataField(id = "thresholdsFile", description = FieldConstants.FAMILY_QC_RELATEDNESS_THRESHOLD_FILE_DESCRIPTION) private String thresholdsFile; - public FamilyQcRelatednessAnalysisParams() { + public QcRelatednessAnalysisParams() { } - public FamilyQcRelatednessAnalysisParams(String populationFrequencyFile, String populationExcludeVariantsFile, String thresholdsFile) { + public QcRelatednessAnalysisParams(String populationFrequencyFile, String populationExcludeVariantsFile, String thresholdsFile) { this.populationFrequencyFile = populationFrequencyFile; this.populationExcludeVariantsFile = populationExcludeVariantsFile; this.thresholdsFile = thresholdsFile; @@ -53,7 +53,7 @@ public String getPopulationFrequencyFile() { return populationFrequencyFile; } - public FamilyQcRelatednessAnalysisParams setPopulationFrequencyFile(String populationFrequencyFile) { + public QcRelatednessAnalysisParams setPopulationFrequencyFile(String populationFrequencyFile) { this.populationFrequencyFile = populationFrequencyFile; return this; } @@ -62,7 +62,7 @@ public String getPopulationExcludeVariantsFile() { return populationExcludeVariantsFile; } - public FamilyQcRelatednessAnalysisParams setPopulationExcludeVariantsFile(String populationExcludeVariantsFile) { + public QcRelatednessAnalysisParams setPopulationExcludeVariantsFile(String populationExcludeVariantsFile) { this.populationExcludeVariantsFile = populationExcludeVariantsFile; return this; } @@ -71,7 +71,7 @@ public String getThresholdsFile() { return thresholdsFile; } - public FamilyQcRelatednessAnalysisParams setThresholdsFile(String thresholdsFile) { + public QcRelatednessAnalysisParams setThresholdsFile(String thresholdsFile) { this.thresholdsFile = thresholdsFile; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/IndividualVariantQcAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/IndividualVariantQcAnalysisExecutor.java new file mode 100644 index 00000000000..c06b1c9b359 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/IndividualVariantQcAnalysisExecutor.java @@ -0,0 +1,61 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.tools.variant; + +import org.opencb.opencga.core.models.variant.FamilyQcAnalysisParams; +import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams; +import org.opencb.opencga.core.tools.OpenCgaToolExecutor; + +import java.nio.file.Path; +import java.util.LinkedList; + +public abstract class IndividualVariantQcAnalysisExecutor extends OpenCgaToolExecutor { + + protected LinkedList vcfPaths; + protected LinkedList jsonPaths; + protected IndividualQcAnalysisParams qcParams; + + public IndividualVariantQcAnalysisExecutor() { + } + + public LinkedList getVcfPaths() { + return vcfPaths; + } + + public IndividualVariantQcAnalysisExecutor setVcfPaths(LinkedList vcfPaths) { + this.vcfPaths = vcfPaths; + return this; + } + + public LinkedList getJsonPaths() { + return jsonPaths; + } + + public IndividualVariantQcAnalysisExecutor setJsonPaths(LinkedList jsonPaths) { + this.jsonPaths = jsonPaths; + return this; + } + + public IndividualQcAnalysisParams getQcParams() { + return qcParams; + } + + public IndividualVariantQcAnalysisExecutor setQcParams(IndividualQcAnalysisParams qcParams) { + this.qcParams = qcParams; + return this; + } +}