diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java index bc6e1dc674b..c494f8db7ca 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java @@ -16,9 +16,13 @@ package org.opencb.opencga.analysis; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.opencb.commons.utils.FileUtils; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +32,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; public class ConfigurationUtils { private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class); + private ConfigurationUtils() { + throw new IllegalStateException("Utility class"); + } /** * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml. * @@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome) .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml")); } } + + public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException { + List tools = new ArrayList<>(); + for (AnalysisTool tool : configuration.getAnalysis().getTools()) { + if (tool.getId().equals(toolId)) { + tools.add(tool); + } + } + if (CollectionUtils.isEmpty(tools)) { + throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file"); + } + if (tools.size() == 1) { + return tools.get(0).getVersion(); + } + String defaultVersion = null; + for (AnalysisTool tool : tools) { + if (tool.isDefaultVersion()) { + if (!StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("More than one default version found for tool ID '" + toolId + "'"); + } else { + defaultVersion = tool.getVersion(); + } + } + } + if (StringUtils.isEmpty(defaultVersion)) { + throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true"); + } + return defaultVersion; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 06e1c7f4838..bbf685d1a03 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -30,8 +30,10 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.clinical.InterpretationAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -44,10 +46,10 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import java.io.BufferedReader; import java.io.File; @@ -69,6 +71,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { private String clinicalAnalysisId; private String sampleId; private ClinicalAnalysis.Type clinicalAnalysisType; + private String exomiserVersion; private ClinicalAnalysis clinicalAnalysis; @@ -97,8 +100,7 @@ protected void check() throws Exception { try { clinicalAnalysisQueryResult = catalogManager.getClinicalAnalysisManager().get(studyId, clinicalAnalysisId, QueryOptions.empty(), token); - } catch ( - CatalogException e) { + } catch (CatalogException e) { throw new ToolException(e); } if (clinicalAnalysisQueryResult.getNumResults() != 1) { @@ -117,6 +119,7 @@ protected void check() throws Exception { } sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId(); + // Check clinical analysis type if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) { clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY; } else { @@ -125,6 +128,13 @@ protected void check() throws Exception { logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(), clinicalAnalysisType); + // Check exomiser version + if (StringUtils.isEmpty(exomiserVersion)) { + // Missing exomiser version use the default one + exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + } + // Update executor params with OpenCGA home and session ID setUpStorageEngineExecutor(studyId); } @@ -134,25 +144,28 @@ protected void run() throws ToolException { step(() -> { executorParams.put(EXECUTOR_ID, ExomiserWrapperAnalysisExecutor.ID); - getToolExecutor(ExomiserWrapperAnalysisExecutor.class) + ExomiserWrapperAnalysisExecutor exomiserExecutor = getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(studyId) .setSampleId(sampleId) .setClinicalAnalysisType(clinicalAnalysisType) - .execute(); + .setExomiserVersion(exomiserVersion); + + exomiserExecutor.execute(); - saveInterpretation(studyId, clinicalAnalysis); + saveInterpretation(studyId, clinicalAnalysis, exomiserExecutor.getDockerImageName(), exomiserExecutor.getDockerImageVersion()); }); } - protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException, + protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis, String dockerImage, String dockerImageVersion) + throws ToolException, StorageEngineException, CatalogException, IOException { // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( new Software() .setName("Exomiser") - .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME) - .setVersion(ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_VERSION))); + .setRepository("Docker: " + dockerImage) + .setVersion(dockerImageVersion))); // Analyst ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(studyId, token); @@ -274,8 +287,17 @@ private List getPrimaryFindings() throws IOException, StorageEn // Convert variants to clinical variants for (Variant variant : variantResults.getResults()) { ClinicalVariant clinicalVariant = clinicalVariantCreator.create(variant); - List exomiserTranscripts = new ArrayList<>(variantTranscriptMap.get(normalizedToTsv - .get(variant.toStringSimple()))); + List exomiserTranscripts = new ArrayList<>(); + if (normalizedToTsv.containsKey(variant.toStringSimple())) { + if (variantTranscriptMap.containsKey(normalizedToTsv.get(variant.toStringSimple()))) { + exomiserTranscripts.addAll(variantTranscriptMap.get(normalizedToTsv.get(variant.toStringSimple()))); + } else { + logger.warn("Variant {} (normalizedToTsv {}), not found in map variantTranscriptMap", variant.toStringSimple(), + normalizedToTsv.get(variant.toStringSimple())); + } + } else { + logger.warn("Variant {} not found in map normalizedToTsv", variant.toStringSimple()); + } for (String[] fields : variantTsvMap.get(variant.toStringSimple())) { ClinicalProperty.ModeOfInheritance moi = getModeOfInheritance(fields[4]); Map attributes = getAttributesFromTsv(fields); @@ -463,4 +485,13 @@ public ExomiserInterpretationAnalysis setClinicalAnalysisId(String clinicalAnaly this.clinicalAnalysisId = clinicalAnalysisId; return this; } + + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserInterpretationAnalysis setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index 5a88635e45c..6024761d596 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -552,7 +552,7 @@ protected final T getToolExecutor(Class clazz toolExecutor.getSource(), toolExecutor.getFramework())); - toolExecutor.setUp(erm, executorParams, outDir); + toolExecutor.setUp(erm, executorParams, outDir, configuration); return toolExecutor; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java index a5d2484b2d7..270e9e0b213 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/gwas/GwasAnalysis.java @@ -295,7 +295,7 @@ protected void run() throws ToolException { step("gwas", () -> { GwasAnalysisExecutor gwasExecutor = getToolExecutor(GwasAnalysisExecutor.class); - gwasExecutor.setConfiguration(gwasConfiguration) + gwasExecutor.setGwasConfiguration(gwasConfiguration) .setStudy(study) .setSampleList1(caseCohortSamples) .setSampleList2(controlCohortSamples) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java index 04606492bff..6121cf1e247 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java @@ -134,7 +134,6 @@ private void rebuildSampleFileIds(VariantStorageMetadataManager metadataManager, for (Map.Entry> entry : batch.entrySet()) { Integer sampleId = entry.getKey(); List fileIds = entry.getValue(); - List actualFiles = metadataManager.getSampleMetadata(studyId, sampleId).getFiles(); if (actualFiles.size() != fileIds.size() || !actualFiles.containsAll(fileIds)) { fixedSamples++; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index d68f363b36d..0f1fc2bec67 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -7,8 +7,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; -import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.config.AnalysisTool; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.slf4j.Logger; @@ -18,12 +18,9 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; -public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { +public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor { public final static String DOCKER_INPUT_PATH = "/data/input"; public final static String DOCKER_OUTPUT_PATH = "/data/output"; @@ -31,14 +28,51 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDOUT_FILENAME = "stdout.txt"; public static final String STDERR_FILENAME = "stderr.txt"; + public String getDockerImageName() throws ToolException { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[0]; + } + public static final String DOCKER_CLI_MSG = "Docker CLI: "; - public String getDockerImageName() { - return "opencb/opencga-ext-tools"; + public String getDockerImageVersion() throws ToolException { + if (getConfiguration().getAnalysis().getOpencgaExtTools().contains(":")) { + return getConfiguration().getAnalysis().getOpencgaExtTools().split(":")[1]; + } else { + return GitRepositoryState.getInstance().getBuildVersion(); + } + } + + protected AnalysisTool getAnalysisTool(String toolId, String version) throws ToolException { + for (AnalysisTool tool : getConfiguration().getAnalysis().getTools()) { + if (toolId.equals(tool.getId()) && version.equals(tool.getVersion())) { + return tool; + } + } + throw new ToolException("Missing analyis tool (ID = " + toolId + ", version = " + version + ") in configuration file"); + } + + public String getDockerImageName(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + return tool.getDockerId().split(":")[0]; + } + + public String getDockerImageVersion(String toolId, String version) throws ToolException { + AnalysisTool tool = getAnalysisTool(toolId, version); + if (tool.getDockerId().contains(":")) { + return tool.getDockerId().split(":")[1]; + } else { + return null; + } } - public String getDockerImageVersion() { - return GitRepositoryState.getInstance().getBuildVersion(); + protected String getToolResource(String toolId, String version, String resourceKey) throws ToolException { + // Get resources from the configuration file + AnalysisTool tool = getAnalysisTool(toolId, version); + if (!tool.getResources().containsKey(resourceKey)) { + throw new ToolException("Error getting resource " + resourceKey + " of analysis tool (ID = " + toolId + ", version = " + + version + "): it does not exist in the configuration file"); + } + return tool.getResources().get(resourceKey); } private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); @@ -59,6 +93,14 @@ protected StringBuilder initCommandLine() { return new StringBuilder("docker run --log-driver=none -a stdin -a stdout -a stderr "); } + protected StringBuilder initCommandLine(String user) { + StringBuilder sb = initCommandLine(); + if (StringUtils.isNotEmpty(user)) { + sb.append("--user ").append(user); + } + return sb; + } + protected Map appendMounts(List> inputFilenames, StringBuilder sb) { Map mountMap = new HashMap<>(); @@ -84,7 +126,7 @@ protected Map appendMounts(List> inputFilen return mountMap; } - protected void appendCommand(String command, StringBuilder sb) { + protected void appendCommand(String command, StringBuilder sb) throws ToolException { // Docker image and version sb.append(getDockerImageName()); if (StringUtils.isNotEmpty(getDockerImageVersion())) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index c4a9d87d4f7..a6d34af3f37 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -17,6 +17,7 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.analysis.ConfigurationUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; @@ -34,6 +35,14 @@ public class ExomiserWrapperAnalysis extends OpenCgaToolScopeStudy { public final static String DESCRIPTION = "The Exomiser is a Java program that finds potential disease-causing variants" + " from whole-exome or whole-genome sequencing data."; + // It must match the tool prefix in the tool keys for exomiser in the configuration file + public final static String EXOMISER_PREFIX = "exomiser-"; + + // It must match the resources key in the exomiser/tool section in the configuration file + public final static String HG19_RESOURCE_KEY = "HG19"; + public final static String HG38_RESOURCE_KEY = "HG38"; + public final static String PHENOTYPE_RESOURCE_KEY = "PHENOTYPE"; + @ToolParams protected final ExomiserWrapperParams analysisParams = new ExomiserWrapperParams(); @@ -43,6 +52,14 @@ protected void check() throws Exception { if (StringUtils.isEmpty(getStudy())) { throw new ToolException("Missing study"); } + + // Check exomiser version + if (StringUtils.isEmpty(analysisParams.getExomiserVersion())) { + // Missing exomiser version use the default one + String exomiserVersion = ConfigurationUtils.getToolDefaultVersion(ExomiserWrapperAnalysis.ID, configuration); + logger.warn("Missing exomiser version, using the default {}", exomiserVersion); + analysisParams.setExomiserVersion(exomiserVersion); + } } @Override @@ -53,6 +70,7 @@ protected void run() throws Exception { getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(study) .setSampleId(analysisParams.getSample()) + .setExomiserVersion(analysisParams.getExomiserVersion()) .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType())) .execute(); }); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index a6325932fb9..2d0d0dfb0ab 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -1,7 +1,6 @@ package org.opencb.opencga.analysis.wrappers.exomiser; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; @@ -10,6 +9,7 @@ import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; @@ -30,10 +30,14 @@ import java.io.*; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import static org.opencb.commons.utils.FileUtils.copyFile; +import static org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis.*; + @ToolExecutor(id = ExomiserWrapperAnalysisExecutor.ID, tool = ExomiserWrapperAnalysis.ID, source = ToolExecutor.Source.STORAGE, @@ -46,24 +50,32 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private final static String EXOMISER_PROPERTIES_TEMPLATE_FILENAME = "application.properties"; private static final String EXOMISER_OUTPUT_OPTIONS_FILENAME = "output.yml"; - public final static String DOCKER_IMAGE_NAME = "exomiser/exomiser-cli"; - public final static String DOCKER_IMAGE_VERSION = "13.1.0"; - private String studyId; private String sampleId; + private String exomiserVersion; private ClinicalAnalysis.Type clinicalAnalysisType; private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void run() throws ToolException { + public void run() throws ToolException, IOException, CatalogException { // Check HPOs, it will use a set to avoid duplicate HPOs, // and it will check both phenotypes and disorders - logger.info("{}: Checking individual for sample {} in study {}", ID, sampleId, studyId); + logger.info("Checking individual for sample {} in study {}", sampleId, studyId); Set hpos = new HashSet<>(); Individual individual = IndividualQcUtils.getIndividualBySampleId(studyId, sampleId, getVariantStorageManager().getCatalogManager(), getToken()); + // Check assembly + String assembly = IndividualQcUtils.getAssembly(studyId, getVariantStorageManager().getCatalogManager(), getToken()); + if (assembly.equalsIgnoreCase("GRCh38")) { + assembly = "hg38"; +// } else if (assembly.equalsIgnoreCase("GRCh37")) { +// assembly = "hg19"; + } else { + throw new ToolException("Invalid assembly '" + assembly + "'. Supported assemblies are: GRCh38"); + } + // Set father and mother if necessary (family ?) if (individual.getFather() != null && StringUtils.isNotEmpty(individual.getFather().getId())) { individual.setFather(IndividualQcUtils.getIndividualById(studyId, individual.getFather().getId(), @@ -74,7 +86,7 @@ public void run() throws ToolException { getVariantStorageManager().getCatalogManager(), getToken())); } - logger.info("{}: Individual found: {}", ID, individual.getId()); + logger.info("Individual found: {}", individual.getId()); if (CollectionUtils.isNotEmpty(individual.getPhenotypes())) { for (Phenotype phenotype : individual.getPhenotypes()) { if (phenotype.getId().startsWith("HP:")) { @@ -94,7 +106,7 @@ public void run() throws ToolException { throw new ToolException("Missing phenotypes, i.e. HPO terms, for individual/sample (" + individual.getId() + "/" + sampleId + ")"); } - logger.info("{}: Getting HPO for individual {}: {}", ID, individual.getId(), StringUtils.join(hpos, ",")); + logger.info("Getting HPO for individual {}: {}", individual.getId(), StringUtils.join(hpos, ",")); List samples = new ArrayList<>(); samples.add(sampleId); @@ -137,8 +149,8 @@ public void run() throws ToolException { QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,studies.samples"); - logger.info("{}: Exomiser exports variants using the query: {}", ID, query.toJson()); - logger.info("{}: Exomiser exports variants using the query options: {}", ID, queryOptions.toJson()); + logger.info("Exomiser exports variants using the query: {}", query.toJson()); + logger.info("Exomiser exports variants using the query options: {}", queryOptions.toJson()); try { getVariantStorageManager().exportData(vcfPath.toString(), VariantWriterFactory.VariantOutputFormat.VCF_GZ, null, query, @@ -153,30 +165,34 @@ public void run() throws ToolException { // Copy the analysis try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME) + .toFile(), getOutDir().resolve(EXOMISER_ANALYSIS_TEMPLATE_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser analysis file", e); } - // Copy the application.properties + // Copy the application.properties and update data according to Exomiser version try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile(), - getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME).toFile()); + Path target = getOutDir().resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_PROPERTIES_TEMPLATE_FILENAME) + .toFile(), target.toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser properties file", e); } // Copy the output options try { - FileUtils.copyFile(openCgaHome.resolve("analysis/exomiser/" + EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), + copyFile(openCgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile(), getOutDir().resolve(EXOMISER_OUTPUT_OPTIONS_FILENAME).toFile()); } catch (IOException e) { throw new ToolException("Error copying Exomiser output options file", e); } // Build the docker command line to run Exomiser - StringBuilder sb = initCommandLine(); + String[] userAndGroup = FileUtils.getUserAndGroup(getOutDir(), true); + String dockerUser = userAndGroup[0] + ":" + userAndGroup[1]; + logger.info("Docker user: {}", dockerUser); + StringBuilder sb = initCommandLine(dockerUser); // Append mounts sb.append(" --mount type=bind,source=" + exomiserDataPath + ",target=/data") @@ -192,7 +208,8 @@ public void run() throws ToolException { sb.append(" --ped /jobdir/").append(pedigreeFile.getName()); } sb.append(" --vcf /jobdir/" + vcfPath.getFileName()) - .append(" --assembly hg38 --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) + .append(" --assembly ").append(assembly) + .append(" --output /jobdir/").append(EXOMISER_OUTPUT_OPTIONS_FILENAME) .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); // Execute command and redirect stdout and stderr to the files @@ -365,7 +382,7 @@ private Path getAnalysisDataPath(String analysisId) throws ToolException { } private Path getExomiserDataPath(Path openCgaHome) throws ToolException { - Path exomiserDataPath = openCgaHome.resolve("analysis/resources/exomiser"); + Path exomiserDataPath = openCgaHome.resolve("analysis/resources/" + ExomiserWrapperAnalysis.ID); if (!exomiserDataPath.toFile().exists()) { if (!exomiserDataPath.toFile().mkdirs()) { throw new ToolException("Error creating the Exomiser data directory"); @@ -374,33 +391,37 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { // Mutex management to avoid multiple downloadings at the same time // the first to come, download data, others have to wait for - File readyFile = exomiserDataPath.resolve("READY").toFile(); - File preparingFile = exomiserDataPath.resolve("PREPARING").toFile(); + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); + String resourceVersion = Paths.get(resource).getFileName().toString().split("[_]")[0]; + File readyFile = exomiserDataPath.resolve("READY-" + resourceVersion).toFile(); + File preparingFile = exomiserDataPath.resolve("PREPARING-" + resourceVersion).toFile(); // If all is ready, then return if (readyFile.exists()) { - logger.info("{}: Exomiser data is already downloaded, so Exomiser analysis is ready to be executed.", ID); + logger.info("Exomiser {} data {} is already downloaded, so Exomiser analysis is ready to be executed.", exomiserVersion, + resourceVersion); return exomiserDataPath; } // If it is preparing, then wait for ready and then return if (preparingFile.exists()) { long startTime = System.currentTimeMillis(); - logger.info("{}: Exomiser data is downloading, waiting for it...", ID); + logger.info("Exomiser {} data {} is downloading, waiting for it...", exomiserVersion, resourceVersion); while (!readyFile.exists()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // Nothing to do here + preparingFile.delete(); throw new ToolException(e); } long elapsedTime = System.currentTimeMillis() - startTime; if (elapsedTime > 18000000) { - throw new ToolException("Unable to run the Exomiser analysis because of Exomiser data is not ready yet: maximum" - + " waiting time exceeded"); + throw new ToolException("Unable to run the Exomiser analysis because of Exomiser " + exomiserVersion + " data " + + resourceVersion + " is not ready yet: maximum waiting time exceeded"); } } - logger.info("{}: Exomiser data is now downloaded: Exomiser analysis is ready to be executed", ID); + logger.info("Exomiser {} data is now downloaded: Exomiser analysis is ready to be executed", exomiserVersion); return exomiserDataPath; } @@ -408,38 +429,49 @@ private Path getExomiserDataPath(Path openCgaHome) throws ToolException { try { preparingFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error creating the Exomiser data directory"); + preparingFile.delete(); + throw new ToolException("Error creating the Exomiser " + exomiserVersion + " data " + resourceVersion + " directory"); } - // Download and unzip files + // Download resources and unzip files try { - downloadAndUnzip(exomiserDataPath, "2109_hg38.zip"); - downloadAndUnzip(exomiserDataPath, "2109_phenotype.zip"); + downloadAndUnzip(exomiserDataPath, HG38_RESOURCE_KEY); + downloadAndUnzip(exomiserDataPath, PHENOTYPE_RESOURCE_KEY); } catch (ToolException e) { // If something wrong happened, the preparing file has to be deleted preparingFile.delete(); - throw new ToolException("Something wrong happened when preparing Exomiser data", e); + throw new ToolException("Something wrong happened when preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } // Mutex management, signal exomiser data is ready try { readyFile.createNewFile(); } catch (IOException e) { - throw new ToolException("Error preparing Exomiser data", e); + throw new ToolException("Error preparing Exomiser " + exomiserVersion + " data " + resourceVersion, e); } preparingFile.delete(); return exomiserDataPath; } + private String getHg38DataVersion() throws ToolException { + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, HG38_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + + private String getPhenotypeDataVersion() throws ToolException { + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, PHENOTYPE_RESOURCE_KEY); + return Paths.get(resource).getFileName().toString().split("_")[0]; + } + @Override - public String getDockerImageName() { - return DOCKER_IMAGE_NAME; + public String getDockerImageName() throws ToolException { + return getDockerImageName(ExomiserWrapperAnalysis.ID, exomiserVersion); } @Override - public String getDockerImageVersion() { - return DOCKER_IMAGE_VERSION; + public String getDockerImageVersion() throws ToolException { + return getDockerImageVersion(ExomiserWrapperAnalysis.ID, exomiserVersion); } public String getStudyId() { @@ -451,21 +483,38 @@ public ExomiserWrapperAnalysisExecutor setStudyId(String studyId) { return this; } - private void downloadAndUnzip(Path exomiserDataPath, String filename) throws ToolException { - URL url = null; - - // Download data - try { - url = new URL("http://resources.opencb.org/opencb/opencga/analysis/exomiser/" + filename); - logger.info("{}: Downloading Exomiser data: {} in {}", ID, url, exomiserDataPath); - ResourceUtils.downloadThirdParty(url, exomiserDataPath); - } catch (IOException e) { - throw new ToolException("Error downloading Exomiser data from " + url, e); + private void downloadAndUnzip(Path exomiserDataPath, String resourceKey) throws ToolException { + String filename; + String resource = getToolResource(ExomiserWrapperAnalysis.ID, exomiserVersion, resourceKey); + if (resource.startsWith("file://")) { + // Copy resouce + try { + Path sourcePath = Paths.get(resource); + filename = sourcePath.getFileName().toString(); + Files.copy(sourcePath, exomiserDataPath.resolve(filename)); + } catch (IOException e) { + throw new ToolException("Error copying Exomiser data from " + resource, e); + } + } else { + // Download resource + String url; + if (resource.startsWith("http://") || resource.startsWith("https://") || resource.startsWith("ftp://")) { + url = resource; + } else { + url = getConfiguration().getAnalysis().getResourceUrl() + resource; + } + logger.info("Downloading Exomiser data: {} in {}", url, exomiserDataPath); + try { + ResourceUtils.downloadThirdParty(new URL(url), exomiserDataPath); + filename = Paths.get(url).getFileName().toString(); + } catch (IOException e) { + throw new ToolException("Error downloading Exomiser data from " + url, e); + } } // Unzip try { - logger.info("{}: Decompressing Exomiser data: {}", ID, filename); + logger.info("Decompressing Exomiser {} data: {}", exomiserDataPath, filename); new Command("unzip -o -d " + exomiserDataPath + " " + exomiserDataPath + "/" + filename) .setOutputOutputStream(new DataOutputStream(new FileOutputStream(getOutDir().resolve("stdout_unzip_" + filename + ".txt").toFile()))) @@ -473,11 +522,11 @@ private void downloadAndUnzip(Path exomiserDataPath, String filename) throws Too + filename + ".txt").toFile()))) .run(); } catch (FileNotFoundException e) { - throw new ToolException("Error unzipping Exomiser data: " + filename, e); + throw new ToolException("Error unzipping Exomiser " + exomiserVersion + " data: " + filename, e); } // Free disk space - logger.info("{}: Deleting Exomiser data: {}", ID, filename); + logger.info("Deleting Exomiser data: {}", filename); exomiserDataPath.resolve(filename).toFile().delete(); } @@ -490,6 +539,15 @@ public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) { return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperAnalysisExecutor setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public ClinicalAnalysis.Type getClinicalAnalysisType() { return clinicalAnalysisType; } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java index f77a1025377..5e6033750b6 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java @@ -51,15 +51,6 @@ public static AbstractClinicalManagerTest getClinicalTest(OpenCGATestExternalRes clinicalTest.catalogManagerResource = opencga.getCatalogManagerExternalResource(); clinicalTest.setUp(); - // Exomiser analysis - Path exomiserDataPath = opencga.getOpencgaHome().resolve("analysis/exomiser"); - Files.createDirectories(exomiserDataPath); - Path parent = Paths.get(ClinicalAnalysisUtilsTest.class.getClassLoader().getResource("pheno").getPath()).getParent(); - Files.copy(parent.resolve("exomiser/application.properties"), exomiserDataPath.resolve("application.properties"), - StandardCopyOption.REPLACE_EXISTING); - Files.copy(parent.resolve("exomiser/output.yml"), exomiserDataPath.resolve("output.yml"), - StandardCopyOption.REPLACE_EXISTING); - // Storage ObjectMap storageOptions = new ObjectMap() .append(VariantStorageOptions.ANNOTATE.key(), true) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 441f03d06a3..46799bfab58 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -1,12 +1,15 @@ package org.opencb.opencga.analysis.clinical.exomiser; import org.apache.commons.lang3.StringUtils; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.tools.variant.VariantNormalizer; -import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; @@ -34,6 +37,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; @Category(MediumTests.class) public class ExomiserInterpretationAnalysisTest { @@ -72,8 +76,12 @@ public void testNormalization() throws NonStandardCompliantSampleField { } @Test - public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_single")); @@ -83,11 +91,14 @@ public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ClinicalAnalysis clinicalAnalysis = caResult.getResults().get(0); assertEquals(0, clinicalAnalysis.getSecondaryInterpretations().size()); - ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); + System.out.println("opencga.getOpencgaHome() = " + opencga.getOpencgaHome().toAbsolutePath()); + System.out.println("outDir = " + outDir); + ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID2); + .setClinicalAnalysisId(clinicalTest.CA_ID2) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -115,8 +126,12 @@ public void singleSingleExomiserAnalysis() throws IOException, CatalogException, } @Test - public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_family")); @@ -129,7 +144,8 @@ public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, T exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID3); + .setClinicalAnalysisId(clinicalTest.CA_ID3) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); @@ -160,7 +176,11 @@ public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, T @Test public void trioSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { - assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); +// String exomiserVersion = "13.1"; +// String resourceVersion = "2109"; + String exomiserVersion = "14.0"; + String resourceVersion = "2402"; + assumeTrue(Paths.get("/opt/opencga/analysis/resources/exomiser/READY-" + resourceVersion).toFile().exists()); prepareExomiserData(); outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_single")); @@ -173,7 +193,8 @@ public void trioSingleExomiserAnalysis() throws IOException, CatalogException, T exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); exomiser.setStudyId(clinicalTest.studyFqn) - .setClinicalAnalysisId(clinicalTest.CA_ID4); + .setClinicalAnalysisId(clinicalTest.CA_ID4) + .setExomiserVersion(exomiserVersion); ExecutionResult result = exomiser.start(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 5c5956d7cf7..b333a01b625 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -22,6 +22,7 @@ import org.opencb.commons.datastore.mongodb.MongoDataStore; import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.analysis.StorageManager; +import org.opencb.opencga.analysis.clinical.exomiser.ExomiserInterpretationAnalysisTest; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -261,11 +262,15 @@ public Path isolateOpenCGA() throws IOException { Files.copy(inputStream, analysisPath.resolve("ped.R"), StandardCopyOption.REPLACE_EXISTING); // Exomiser analysis files - analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser")).toAbsolutePath(); + List exomiserVersions = Arrays.asList("13.1", "14.0"); List exomiserFiles = Arrays.asList("application.properties", "exomiser-analysis.yml", "output.yml"); - for (String exomiserFile : exomiserFiles) { - inputStream = new FileInputStream("../opencga-app/app/analysis/exomiser/" + exomiserFile); - Files.copy(inputStream, analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + for (String exomiserVersion : exomiserVersions) { + analysisPath = Files.createDirectories(opencgaHome.resolve("analysis/exomiser").resolve(exomiserVersion).toAbsolutePath()); + Path exomiserPath = Paths.get(ExomiserInterpretationAnalysisTest.class.getClassLoader().getResource("exomiser").getPath()); + for (String exomiserFile : exomiserFiles) { + String resource = exomiserVersion + "/" + exomiserFile; + Files.copy(exomiserPath.resolve(resource).toAbsolutePath(), analysisPath.resolve(exomiserFile), StandardCopyOption.REPLACE_EXISTING); + } } return opencgaHome; diff --git a/opencga-analysis/src/test/resources/exomiser/application.properties b/opencga-analysis/src/test/resources/exomiser/13.1/application.properties similarity index 95% rename from opencga-analysis/src/test/resources/exomiser/application.properties rename to opencga-analysis/src/test/resources/exomiser/13.1/application.properties index e722c2fd89b..1775e32d14c 100644 --- a/opencga-analysis/src/test/resources/exomiser/application.properties +++ b/opencga-analysis/src/test/resources/exomiser/13.1/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=2109 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz exomiser.phenotype.data-version=2109 logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/exomiser-analysis.yml b/opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/exomiser-analysis.yml rename to opencga-analysis/src/test/resources/exomiser/13.1/exomiser-analysis.yml diff --git a/opencga-analysis/src/test/resources/exomiser/output.yml b/opencga-analysis/src/test/resources/exomiser/13.1/output.yml similarity index 88% rename from opencga-analysis/src/test/resources/exomiser/output.yml rename to opencga-analysis/src/test/resources/exomiser/13.1/output.yml index 6a70c778641..3203c545356 100644 --- a/opencga-analysis/src/test/resources/exomiser/output.yml +++ b/opencga-analysis/src/test/resources/exomiser/13.1/output.yml @@ -1,8 +1,8 @@ --- -#outputContributingVariantsOnly: true +outputContributingVariantsOnly: true #numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results -#numGenes: 20 -#minExomiserGeneScore: 0.7 +numGenes: 20 +minExomiserGeneScore: 0.0 #outputPrefix options: specify the path/filename without an extension and this will be added # according to the outputFormats option. If unspecified this will default to the following: # {exomiserDir}/results/input-vcf-name-exomiser-results.html diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/application.properties b/opencga-analysis/src/test/resources/exomiser/14.0/application.properties new file mode 100644 index 00000000000..d1658370370 --- /dev/null +++ b/opencga-analysis/src/test/resources/exomiser/14.0/application.properties @@ -0,0 +1,47 @@ +# +# The Exomiser - A tool to annotate and prioritize genomic variants +# +# Copyright (c) 2016-2021 Queen Mary University of London. +# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +## exomiser root data directory ## +# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed +# in this properties file will be found in the data directory, unless specifically overridden here. +#exomiser.data-directory=/exomiser-data +exomiser.data-directory=/data + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg38.data-version=1811 +#exomiser.hg38.data-version=2109 +#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +#exomiser.phenotype.data-version=2109 +#logging.file.name=/exomiser-data/logs/exomiser.log + + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg19.data-version=1811 +exomiser.hg38.data-version=2402 +#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz +#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2402 +exomiser.hg38.clin-var-data-version=2402 +exomiser.hg38.use-clinvar-white-list=true +logging.file.name=/jobdir/exomiser.log diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml b/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml new file mode 100644 index 00000000000..e3d064ba07b --- /dev/null +++ b/opencga-analysis/src/test/resources/exomiser/14.0/exomiser-analysis.yml @@ -0,0 +1,85 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + X_DOMINANT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + X_RECESSIVE_HOM_ALT: 0.1, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + UK10K, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + # GNOMAD_E_FIN, + GNOMAD_E_NFE, + # GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + # GNOMAD_G_FIN, + GNOMAD_G_NFE, + # GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +#all steps are optional +steps: [ + #hiPhivePrioritiser: {}, + #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, + #intervalFilter: {interval: 'chr10:123256200-123256300'}, + # or for multiple intervals: + #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, + # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF + #intervalFilter: {bed: /full/path/to/bed_file.bed}, + #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, + #geneBlacklistFilter: { }, + failedVariantFilter: { }, + #qualityFilter: {minQuality: 50.0}, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + UPSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + DOWNSTREAM_GENE_VARIANT + ] + }, + # removes variants represented in the database + #knownVariantFilter: {}, + frequencyFilter: {maxFrequency: 2.0}, + pathogenicityFilter: {keepNonPathogenic: true}, + # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed + # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. + inheritanceFilter: {}, + # omimPrioritiser isn't mandatory. + omimPrioritiser: {}, + #priorityScoreFilter: {minPriorityScore: 0.4}, + # Other prioritisers: Only combine omimPrioritiser with one of these. + # Don't include any if you only want to filter the variants. + hiPhivePrioritiser: {}, + # or run hiPhive in benchmarking mode: + #hiPhivePrioritiser: {runParams: 'mouse'}, + #phivePrioritiser: {} + #phenixPrioritiser: {} + #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} +] diff --git a/opencga-analysis/src/test/resources/exomiser/14.0/output.yml b/opencga-analysis/src/test/resources/exomiser/14.0/output.yml new file mode 100644 index 00000000000..695255a573b --- /dev/null +++ b/opencga-analysis/src/test/resources/exomiser/14.0/output.yml @@ -0,0 +1,11 @@ +--- +outputContributingVariantsOnly: true +#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results +numGenes: 20 +minExomiserGeneScore: 0.0 +# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory +outputDirectory: /jobdir/ +# Filename for the output files. Will default to {input-vcf-filename}-exomiser +outputFileName: exomiser_output +#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) +outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-app/app/analysis/exomiser/application.properties b/opencga-app/app/analysis/exomiser/13.1/application.properties similarity index 95% rename from opencga-app/app/analysis/exomiser/application.properties rename to opencga-app/app/analysis/exomiser/13.1/application.properties index e722c2fd89b..1775e32d14c 100644 --- a/opencga-app/app/analysis/exomiser/application.properties +++ b/opencga-app/app/analysis/exomiser/13.1/application.properties @@ -40,6 +40,6 @@ exomiser.data-directory=/data exomiser.hg38.data-version=2109 #exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz #exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz -exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.hg38.variant-white-list-path=2109_hg38_clinvar_whitelist.tsv.gz exomiser.phenotype.data-version=2109 logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml new file mode 100644 index 00000000000..5ec7d93d508 --- /dev/null +++ b/opencga-app/app/analysis/exomiser/13.1/exomiser-analysis.yml @@ -0,0 +1,70 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + X_DOMINANT: 0.1, + X_RECESSIVE_HOM_ALT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + THOUSAND_GENOMES, + TOPMED, + UK10K, + + ESP_AFRICAN_AMERICAN, ESP_EUROPEAN_AMERICAN, ESP_ALL, + + EXAC_AFRICAN_INC_AFRICAN_AMERICAN, EXAC_AMERICAN, + EXAC_SOUTH_ASIAN, EXAC_EAST_ASIAN, + EXAC_FINNISH, EXAC_NON_FINNISH_EUROPEAN, + EXAC_OTHER, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + GNOMAD_E_FIN, + GNOMAD_E_NFE, + GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + GNOMAD_G_FIN, + GNOMAD_G_NFE, + GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +steps: [ + failedVariantFilter: { }, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + UPSTREAM_GENE_VARIANT, + DOWNSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT + ] + }, + frequencyFilter: { maxFrequency: 2.0 }, + pathogenicityFilter: { keepNonPathogenic: true }, + inheritanceFilter: { }, + omimPrioritiser: { }, + hiPhivePrioritiser: { } +] diff --git a/opencga-app/app/analysis/exomiser/output.yml b/opencga-app/app/analysis/exomiser/13.1/output.yml similarity index 100% rename from opencga-app/app/analysis/exomiser/output.yml rename to opencga-app/app/analysis/exomiser/13.1/output.yml diff --git a/opencga-app/app/analysis/exomiser/14.0/application.properties b/opencga-app/app/analysis/exomiser/14.0/application.properties new file mode 100644 index 00000000000..d1658370370 --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/application.properties @@ -0,0 +1,47 @@ +# +# The Exomiser - A tool to annotate and prioritize genomic variants +# +# Copyright (c) 2016-2021 Queen Mary University of London. +# Copyright (c) 2012-2016 Charité Universitätsmedizin Berlin and Genome Research Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +## exomiser root data directory ## +# root path where data is to be downloaded and worked on it is assumed that all the files required by exomiser listed +# in this properties file will be found in the data directory, unless specifically overridden here. +#exomiser.data-directory=/exomiser-data +exomiser.data-directory=/data + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg38.data-version=1811 +#exomiser.hg38.data-version=2109 +#exomiser.hg38.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg38.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +#exomiser.phenotype.data-version=2109 +#logging.file.name=/exomiser-data/logs/exomiser.log + + +#remm.version=0.3.1.post1 +#cadd.version=1.4 +#exomiser.hg19.data-version=1811 +exomiser.hg38.data-version=2402 +#exomiser.hg19.remm-path=${exomiser.data-directory}/remm/ReMM.v${remm.version}.hg19.tsv.gz +#exomiser.hg19.variant-white-list-path=${exomiser.hg19.data-version}_hg19_clinvar_whitelist.tsv.gz +#exomiser.hg38.variant-white-list-path=${exomiser.hg38.data-version}_hg38_clinvar_whitelist.tsv.gz +exomiser.phenotype.data-version=2402 +exomiser.hg38.clin-var-data-version=2402 +exomiser.hg38.use-clinvar-white-list=true +logging.file.name=/jobdir/exomiser.log diff --git a/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml new file mode 100644 index 00000000000..e3d064ba07b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/exomiser-analysis.yml @@ -0,0 +1,85 @@ +--- +analysisMode: PASS_ONLY +inheritanceModes: { + AUTOSOMAL_DOMINANT: 0.1, + AUTOSOMAL_RECESSIVE_COMP_HET: 2.0, + AUTOSOMAL_RECESSIVE_HOM_ALT: 0.1, + X_DOMINANT: 0.1, + X_RECESSIVE_COMP_HET: 2.0, + X_RECESSIVE_HOM_ALT: 0.1, + MITOCHONDRIAL: 0.2 +} +frequencySources: [ + UK10K, + + GNOMAD_E_AFR, + GNOMAD_E_AMR, + # GNOMAD_E_ASJ, + GNOMAD_E_EAS, + # GNOMAD_E_FIN, + GNOMAD_E_NFE, + # GNOMAD_E_OTH, + GNOMAD_E_SAS, + + GNOMAD_G_AFR, + GNOMAD_G_AMR, + # GNOMAD_G_ASJ, + GNOMAD_G_EAS, + # GNOMAD_G_FIN, + GNOMAD_G_NFE, + # GNOMAD_G_OTH, + GNOMAD_G_SAS +] +# Possible pathogenicitySources: (POLYPHEN, MUTATION_TASTER, SIFT), (REVEL, MVP), CADD, REMM +# REMM is trained on non-coding regulatory regions +# *WARNING* if you enable CADD or REMM ensure that you have downloaded and installed the CADD/REMM tabix files +# and updated their location in the application.properties. Exomiser will not run without this. +pathogenicitySources: [ REVEL, MVP ] +#this is the standard exomiser order. +#all steps are optional +steps: [ + #hiPhivePrioritiser: {}, + #priorityScoreFilter: {priorityType: HIPHIVE_PRIORITY, minPriorityScore: 0.500}, + #intervalFilter: {interval: 'chr10:123256200-123256300'}, + # or for multiple intervals: + #intervalFilter: {intervals: ['chr10:123256200-123256300', 'chr10:123256290-123256350']}, + # or using a BED file - NOTE this should be 0-based, Exomiser otherwise uses 1-based coordinates in line with VCF + #intervalFilter: {bed: /full/path/to/bed_file.bed}, + #genePanelFilter: {geneSymbols: ['FGFR1','FGFR2']}, + #geneBlacklistFilter: { }, + failedVariantFilter: { }, + #qualityFilter: {minQuality: 50.0}, + variantEffectFilter: { + remove: [ + FIVE_PRIME_UTR_EXON_VARIANT, + FIVE_PRIME_UTR_INTRON_VARIANT, + THREE_PRIME_UTR_EXON_VARIANT, + THREE_PRIME_UTR_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_EXON_VARIANT, + UPSTREAM_GENE_VARIANT, + INTERGENIC_VARIANT, + REGULATORY_REGION_VARIANT, + CODING_TRANSCRIPT_INTRON_VARIANT, + NON_CODING_TRANSCRIPT_INTRON_VARIANT, + DOWNSTREAM_GENE_VARIANT + ] + }, + # removes variants represented in the database + #knownVariantFilter: {}, + frequencyFilter: {maxFrequency: 2.0}, + pathogenicityFilter: {keepNonPathogenic: true}, + # inheritanceFilter and omimPrioritiser should always run AFTER all other filters have completed + # they will analyse genes according to the specified modeOfInheritance above- UNDEFINED will not be analysed. + inheritanceFilter: {}, + # omimPrioritiser isn't mandatory. + omimPrioritiser: {}, + #priorityScoreFilter: {minPriorityScore: 0.4}, + # Other prioritisers: Only combine omimPrioritiser with one of these. + # Don't include any if you only want to filter the variants. + hiPhivePrioritiser: {}, + # or run hiPhive in benchmarking mode: + #hiPhivePrioritiser: {runParams: 'mouse'}, + #phivePrioritiser: {} + #phenixPrioritiser: {} + #exomeWalkerPrioritiser: {seedGeneIds: [11111, 22222, 33333]} +] diff --git a/opencga-app/app/analysis/exomiser/14.0/output.yml b/opencga-app/app/analysis/exomiser/14.0/output.yml new file mode 100644 index 00000000000..4fc4a4faa6b --- /dev/null +++ b/opencga-app/app/analysis/exomiser/14.0/output.yml @@ -0,0 +1,11 @@ +--- +outputContributingVariantsOnly: true +#numGenes options: 0 = all or specify a limit e.g. 500 for the first 500 results +numGenes: 20 +minExomiserGeneScore: 0.7 +# Path to the desired output directory. Will default to the 'results' subdirectory of the exomiser install directory +outputDirectory: /jobdir/ +# Filename for the output files. Will default to {input-vcf-filename}-exomiser +outputFileName: exomiser_output +#out-format options: HTML, JSON, TSV_GENE, TSV_VARIANT, VCF (default: HTML) +outputFormats: [TSV_VARIANT, JSON, HTML] diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java index ac18f769392..4800ec88c19 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java @@ -317,7 +317,8 @@ private void exomiserInterpretation() throws Exception { ExomiserInterpretationAnalysis exomiserInterpretationAnalysis = new ExomiserInterpretationAnalysis(); exomiserInterpretationAnalysis.setUp(opencgaHome.toString(), new ObjectMap(), outDir, token); exomiserInterpretationAnalysis.setStudyId(cliOptions.study) - .setClinicalAnalysisId(cliOptions.clinicalAnalysis); + .setClinicalAnalysisId(cliOptions.clinicalAnalysis) + .setExomiserVersion(cliOptions.exomiserVersion); // exomiserInterpretationAnalysis.setPrimary(cliOptions.primary); exomiserInterpretationAnalysis.start(); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index 11d18532218..7d20deccb09 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -62,7 +62,6 @@ import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.exceptions.AnalysisExecutionException; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.operations.variant.*; @@ -1021,6 +1020,7 @@ private void exomiser() throws Exception { ObjectMap params = new ExomiserWrapperParams( cliOptions.sample, cliOptions.clinicalAnalysisType, + cliOptions.exomiserVersion, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java index 853d280736d..c2bac1cd1aa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java @@ -23,6 +23,8 @@ import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.*; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @Parameters(commandNames = {"clinical"}, commandDescription = "Clinical analysis commands") @@ -335,9 +337,13 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud @ParametersDelegate public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions; - @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = "Clinical analysis", required = true, arity = 1) + @Parameter(names = {"--" + CLINICAL_ANALYISIS_PARAM_NAME}, description = EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, + required = true, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion; + @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index aa25fe630e1..46ef28a52f6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -70,6 +70,8 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSampleQueryCommandOptions.SAMPLE_QUERY_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexCommandOptions.SECONDARY_INDEX_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.VariantSecondaryIndexDeleteCommandOptions.SECONDARY_INDEX_DELETE_COMMAND; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_SAMPLE_DESCRIPTION; +import static org.opencb.opencga.core.api.FieldConstants.EXOMISER_VERSION_DESCRIPTION; import static org.opencb.opencga.core.api.ParamConstants.*; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND; import static org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.AggregateCommandOptions.AGGREGATE_COMMAND_DESCRIPTION; @@ -1842,12 +1844,15 @@ public class ExomiserAnalysisCommandOptions { @Parameter(names = {"--study"}, description = "Study where all the samples belong to.") public String study; - @Parameter(names = {"--sample"}, description = FieldConstants.SAMPLE_ID_DESCRIPTION, required = true) + @Parameter(names = {"--sample"}, description = EXOMISER_SAMPLE_DESCRIPTION, required = true) public String sample; @Parameter(names = {"--clinical-analysis-type"}, description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION) public String clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE.name(); + @Parameter(names = {"--exomiser-version"}, description = EXOMISER_VERSION_DESCRIPTION) + public String exomiserVersion; + @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index 5e3150d36aa..07dbf9230f9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -583,6 +583,7 @@ private RestResponse runInterpreterExomiser() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "clinicalAnalysis", commandOptions.clinicalAnalysis, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion", commandOptions.exomiserVersion, true); exomiserInterpretationAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 316985784d5..57edea737fa 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -433,6 +433,7 @@ private RestResponse runExomiser() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "sample", commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "exomiserVersion", commandOptions.exomiserVersion, true); putNestedIfNotEmpty(beanParams, "clinicalAnalysisType", commandOptions.clinicalAnalysisType, true); putNestedIfNotEmpty(beanParams, "outdir", commandOptions.outdir, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index 68a4eb5dc0d..1294b1397e7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -301,7 +301,6 @@ private RestResponse updateUser() throws Exception { putNestedIfNotNull(beanParams, "quota.cpuUsage", commandOptions.quotaCpuUsage, true); putNestedIfNotNull(beanParams, "quota.maxDisk", commandOptions.quotaMaxDisk, true); putNestedIfNotNull(beanParams, "quota.maxCpu", commandOptions.quotaMaxCpu, true); - putNestedIfNotEmpty(beanParams, "account.expirationDate", commandOptions.accountExpirationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index a59f09c0e42..b9dc5e1956a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -679,9 +679,12 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) public Boolean jobDryRun; - @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) + @Parameter(names = {"--clinical-analysis"}, description = "Clinical analysis ID.", required = false, arity = 1) public String clinicalAnalysis; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + public String exomiserVersion; + } @Parameters(commandNames = {"interpreter-team-run"}, commandDescription ="Run TEAM interpretation analysis") diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 1c83e822d82..44a0ad64dd7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -431,6 +431,9 @@ public class RunExomiserCommandOptions { @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; + @Parameter(names = {"--exomiser-version"}, description = "Exomiser version in the format X.Y where X is the major version and Y the minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration file to view all installed Exomiser versions and identify the default version.", required = false, arity = 1) + public String exomiserVersion; + @Parameter(names = {"--clinical-analysis-type"}, description = "Clinical analysis type: SINGLE or FAMILY.", required = false, arity = 1) public String clinicalAnalysisType = "SINGLE"; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java index 7b60a3059b2..0f9f2a18c49 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -305,9 +305,6 @@ public class UpdateUserCommandOptions { @Parameter(names = {"--quota-max-cpu"}, description = "The body web service maxCpu parameter", required = false, arity = 1) public Integer quotaMaxCpu; - @Parameter(names = {"--account-expiration-date"}, description = "The body web service expirationDate parameter", required = false, arity = 1) - public String accountExpirationDate; - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java similarity index 97% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java index ae73bcdc578..e1953673252 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_4/catalog/FixStatusIndexesMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_4.catalog; +package org.opencb.opencga.app.migrations.v2.v2_12_4.catalog; import org.bson.Document; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java similarity index 95% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java index 4efa260d965..93104000f20 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/AddAllelesColumnToPhoenix.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java similarity index 99% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java index 132e1357f1b..6d30f1a4de7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v2_12_5.storage; +package org.opencb.opencga.app.migrations.v2.v2_12_5.storage; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java new file mode 100644 index 00000000000..f016e564c77 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java @@ -0,0 +1,45 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; + +import static org.opencb.opencga.catalog.db.mongodb.UserMongoDBAdaptor.*; + +@Migration(id = "add_archivePasswords_array", + description = "Add password history #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class AddPasswordHistoryMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists(PRIVATE_PASSWORD_ARCHIVE, false); + Bson projection = Projections.include(PRIVATE_PASSWORD); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + String currentPassword = document.getString("_password"); + + Document passwordDoc = new Document() + .append(HASH, currentPassword) + .append(SALT, ""); + Document privatePassword = new Document(); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + updateDocument.getSet().put(PRIVATE_PASSWORD, privatePassword); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java new file mode 100644 index 00000000000..68f2cc00e7c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java @@ -0,0 +1,50 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_1; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "move_user_account_to_internal", + description = "Move account to internal.account #6494", version = "3.2.1", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) +public class MoveUserAccountToInternalMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("account", true); + Bson projection = Projections.include("internal", "account"); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), + query, projection, (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + Document account = document.get("account", Document.class); + Document internal = document.get("internal", Document.class); + internal.put("account", account); + + updateDocument.getSet().put("modificationDate", internal.get("lastModified")); + updateDocument.getSet().put("creationDate", account.get("creationDate")); + account.remove("creationDate"); + + Document password = new Document() + .append("expirationDate", null) + .append("lastModified", internal.get("lastModified")); + account.put("password", password); + account.put("failedAttempts", internal.get("failedAttempts")); + internal.remove("failedAttempts"); + + updateDocument.getSet().put("internal", internal); + updateDocument.getUnset().add("account"); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } +} diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 7457317446b..c55f55fadd5 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -61,6 +61,11 @@ + + org.mockito + mockito-core + test + org.apache.commons commons-collections4 diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java index f104f8fdf93..6a3a0480c28 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationProvider; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.user.*; @@ -382,8 +383,9 @@ private List extractUserInformation(List(), attributes); userList.add(user); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java index 65280a1be61..61f9fef8079 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/CatalogAuthenticationManager.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.auth.authentication; import io.jsonwebtoken.SignatureAlgorithm; -import org.apache.commons.lang3.RandomStringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; @@ -26,6 +25,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.MailUtils; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Email; import org.opencb.opencga.core.models.user.AuthenticationResponse; @@ -125,7 +125,7 @@ public String createNonExpiringToken(String organizationId, String userId, Map user = dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java index 35fe615d161..f79eb85b19d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java @@ -198,9 +198,11 @@ public List getRemoteUserInformation(List userStringList) throws C Map attributes = new HashMap<>(); attributes.put("LDAP_RDN", rdn); - User user = new User(uid, displayName, mail, usersSearch, new Account() - .setAuthentication(new Account.AuthenticationOrigin(originId, false)), new UserInternal(new UserStatus()), - new UserQuota(-1, -1, -1, -1), new ArrayList<>(), new HashMap<>(), new LinkedList<>(), attributes); + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(originId, false)); + User user = new User(uid, displayName, mail, usersSearch, TimeUtils.getTime(), TimeUtils.getTime(), + new UserInternal(new UserStatus(), account), + new UserQuota(-1, -1, -1, -1), new HashMap<>(), new LinkedList<>(), attributes); userList.add(user); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java index c86787e84df..09c142e68e9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationDBAdaptor.java @@ -16,10 +16,8 @@ package org.opencb.opencga.catalog.auth.authorization; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.common.Enums; @@ -80,27 +78,21 @@ > OpenCGAResult> get(List resourceIds, L */ OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resource entry) throws CatalogException; - OpenCGAResult setToMembers(long studyId, List members, - List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(long studyId, List members, List aclParams) + throws CatalogException; // Special method only to set acls in study - OpenCGAResult setToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult addToMembers(long studyId, List members, - List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(long studyId, List members, List aclParams) + throws CatalogException; // Special method only to add acls in study - OpenCGAResult addToMembers(List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException; - OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeFromMembers(List members, List aclParams) throws CatalogException; - OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException; OpenCGAResult setAcls(List resourceIds, AclEntryList aclEntryList, Enums.Resource resource) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java index 5f3367d4d1c..883943f1a4b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java @@ -19,10 +19,8 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.organizations.Organization; @@ -83,8 +81,7 @@ default String getCatalogDatabase(String prefix, String organization) { MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException; - OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) throws CatalogException; void deleteOrganization(Organization organization) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java index a06f3762b4d..ccca18225c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/AnnotationSetDBAdaptor.java @@ -21,6 +21,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.study.Variable; @@ -62,12 +63,10 @@ OpenCGAResult update(Query query, ObjectMap parameters, List variab * @param variableSetId variable set id to identify the annotations that will add a new annotation. * @param variable new variable that will be added. * @return a OpenCGAResult object. - * @throws CatalogDBException if the variable could not be added to an existing annotationSet. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if the variable could not be added to an existing annotationSet, there is any unexpected parameter or + * the operation is not authorized. */ - OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException; // /** // * This method will rename the id of all the annotations corresponding to the variableSetId changing oldName per newName. @@ -88,12 +87,10 @@ OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variab * @param variableSetId variable set id for which the annotationSets have to delete the annotation. * @param annotationName Annotation name. * @return a OpenCGAResult object. - * @throws CatalogDBException when there is an error in the database. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException when there is an error in the database, there is any unexpected parameter or the operation is not + * authorized. */ - OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String annotationName) throws CatalogException; /** * Makes a groupBy to obtain the different values that every annotation has and the total number of each. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 73fe6ac7e20..3935f24836d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -237,8 +237,7 @@ default void checkId(long clinicalAnalysisId) throws CatalogDBException, Catalog OpenCGAResult nativeInsert(Map clinicalAnalysis, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList, QueryOptions options) throws CatalogException; OpenCGAResult update(long id, ObjectMap parameters, List variableSetList, List clinicalAuditList, QueryOptions queryOptions) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java index d8b6f82182e..c95d006161d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/CohortDBAdaptor.java @@ -133,8 +133,7 @@ default void checkId(long cohortId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map cohort, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) throws CatalogException; OpenCGAResult get(long cohortId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java index da321435f29..fe9fd976e64 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java @@ -149,7 +149,7 @@ default void checkId(long familyId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map family, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + QueryOptions options) throws CatalogException; OpenCGAResult get(long familyId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 2ffe6fd1ee9..a341840a6bf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -233,13 +233,10 @@ default void checkId(long fileId) throws CatalogDBException, CatalogParameterExc * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due to any formatting error or the user is not authorised. */ OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List variableSetList, QueryOptions options) throws CatalogException; /*** * Inserts the passed file in the database. @@ -252,13 +249,11 @@ OpenCGAResult insert(long studyId, File file, List existingSamples, List * @param variableSetList Variable set list. * @param options Options to filter the output that will be returned after the insertion of the file. * @return A OpenCGAResult object containing the time spent. - * @throws CatalogDBException when the file could not be inserted due to different reasons. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. + * @throws CatalogException when the file could not be inserted due any formatting error or the user is not authorised. */ OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; /*** * Retrieves the file from the database containing the fileId given. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java index d55e1019e0e..b6dbc94b861 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java @@ -163,7 +163,7 @@ default void checkId(long individualId) throws CatalogDBException, CatalogParame OpenCGAResult nativeInsert(Map individual, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long individualId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java index 13e2cb64dfa..a49e7fd6262 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/InterpretationDBAdaptor.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -136,8 +137,7 @@ default void checkId(long interpretationId) throws CatalogDBException, CatalogPa OpenCGAResult nativeInsert(Map interpretation, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + List clinicalAuditList) throws CatalogException; OpenCGAResult update(long uid, ObjectMap parameters, List clinicalAuditList, ParamUtils.SaveInterpretationAs action, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index a6016da7441..ebbc7b62143 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -54,8 +54,7 @@ default void checkId(long jobId) throws CatalogDBException, CatalogParameterExce OpenCGAResult nativeInsert(Map job, String userId) throws CatalogDBException; - OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException; default OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { //return updateStatus(query, new Job.JobStatus(Job.JobStatus.PREPARED)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java index 34d3495407d..eeecd6d55df 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/NoteDBAdaptor.java @@ -6,6 +6,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.response.OpenCGAResult; @@ -16,7 +17,7 @@ public interface NoteDBAdaptor extends DBAdaptor { - OpenCGAResult insert(Note note) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Note note) throws CatalogException; default OpenCGAResult get(long noteUid, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java index d8d8e8a0a51..93e65c63121 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java @@ -5,6 +5,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.response.OpenCGAResult; @@ -98,8 +99,7 @@ public static QueryParams getParam(String key) { // // OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; // - OpenCGAResult insert(Organization organization, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java index a6ce18d5a2f..63bdd0d5e18 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/PanelDBAdaptor.java @@ -153,11 +153,9 @@ default void checkUid(long panelUid) throws CatalogDBException, CatalogParameter } } - OpenCGAResult insert(long studyUid, List panelList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyUid, List panelList) throws CatalogException; - OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(long studyId, Panel panel, QueryOptions options) throws CatalogException; OpenCGAResult get(long panelId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java index fcdfba9034d..1ff667bdbac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ProjectDBAdaptor.java @@ -23,6 +23,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.StudyPermissions; @@ -133,8 +134,7 @@ default void checkId(long projectId) throws CatalogDBException { OpenCGAResult nativeInsert(Map project, String userId) throws CatalogDBException; - OpenCGAResult insert(Project project, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException; OpenCGAResult get(long project, QueryOptions options) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index b9f0b5b1b36..485ab7b10dd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -63,7 +63,7 @@ default void checkId(long sampleId) throws CatalogDBException, CatalogParameterE OpenCGAResult nativeInsert(Map sample, String userId) throws CatalogDBException; OpenCGAResult insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(long sampleId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -86,13 +86,11 @@ OpenCGAResult updateProjectRelease(long studyId, int release) */ OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) throws CatalogException; - default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + default OpenCGAResult setRgaIndexes(long studyUid, RgaIndex rgaIndex) throws CatalogException { return setRgaIndexes(studyUid, Collections.emptyList(), rgaIndex); } - OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException; enum QueryParams implements QueryParam { ID("id", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index b4e0aea1d2e..0afd3f255d4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; @@ -249,8 +250,7 @@ OpenCGAResult setUsersToGroup(long studyId, String groupId, List */ OpenCGAResult removeUsersFromGroup(long studyId, String groupId, List members) throws CatalogDBException; - OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException; /** * Delete a group. @@ -273,12 +273,9 @@ OpenCGAResult removeUsersFromAllGroups(long studyId, List users) * @param groupList List containing possible groups that are synced and where the user should be added to. * @param authOrigin Authentication origin of the synced groups. * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException; /** * ADD or REMOVE user to list of provided groups. @@ -288,13 +285,10 @@ OpenCGAResult resyncUserWithSyncedGroups(String user, List groupL * @param groupList List of group ids. * @param action Update action [ADD, REMOVE] * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException - * @throws CatalogParameterException CatalogParameterException - * @throws CatalogAuthorizationException CatalogAuthorizationException + * @throws CatalogException CatalogException */ - OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, ParamUtils.AddRemoveAction action) + throws CatalogException; /** * Create the permission rule to the list of permission rules defined for the entry in the studyId. @@ -381,13 +375,13 @@ default void checkVariableSetExists(String variableSetId, long studyId) throws C OpenCGAResult createVariableSet(long studyId, VariableSet variableSet) throws CatalogDBException; OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult renameFieldVariableSet(long variableSetId, String oldName, String newName, String user) throws CatalogDBException, CatalogAuthorizationException; OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + throws CatalogException; OpenCGAResult getVariableSet(long variableSetUid, QueryOptions options) throws CatalogDBException; @@ -411,8 +405,7 @@ OpenCGAResult getVariableSet(long variableSetId, QueryOptions optio OpenCGAResult getVariableSets(Query query, QueryOptions queryOptions, String user) throws CatalogDBException, CatalogAuthorizationException; - OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException; void updateDiskUsage(ClientSession clientSession, long studyId, long size) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index 3099262e917..84efa64aea2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -22,10 +22,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.response.OpenCGAResult; @@ -75,7 +72,7 @@ default void checkIds(List userIds) throws CatalogDBException, CatalogPa void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException; OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + throws CatalogException; OpenCGAResult get(String userId, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; @@ -97,9 +94,10 @@ OpenCGAResult delete(String userId, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException; + throws CatalogException; - OpenCGAResult resetPassword(String userId, String email, String newCryptPass) throws CatalogDBException; + OpenCGAResult resetPassword(String userId, String email, String newCryptPass) + throws CatalogException; // Config operations OpenCGAResult setConfig(String userId, String name, Map config) throws CatalogDBException; @@ -118,14 +116,20 @@ enum QueryParams implements QueryParam { NAME("name", TEXT_ARRAY, ""), EMAIL("email", TEXT_ARRAY, ""), ORGANIZATION("organization", TEXT_ARRAY, ""), + CREATION_DATE("creationDate", TEXT_ARRAY, ""), + MODIFICATION_DATE("modificationDate", TEXT_ARRAY, ""), + DEPRECATED_ACCOUNT("account", OBJECT, ""), // Deprecated since 3.2.1 #TASK-6494 TODO: Remove in future releases + DEPRECATED_ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), // Deprecated since 3.2.1 #TASK-6494 INTERNAL("internal", OBJECT, ""), - INTERNAL_FAILED_ATTEMPTS("internal.failedAttempts", INTEGER, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), - ACCOUNT("account", TEXT_ARRAY, ""), - ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), - ACCOUNT_CREATION_DATE("account.creationDate", TEXT, ""), - ACCOUNT_EXPIRATION_DATE("account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT("internal.account", TEXT_ARRAY, ""), + INTERNAL_ACCOUNT_AUTHENTICATION_ID("internal.account.authentication.id", TEXT, ""), + INTERNAL_ACCOUNT_FAILED_ATTEMPTS("internal.account.failedAttempts", INTEGER, ""), + INTERNAL_ACCOUNT_CREATION_DATE("internal.account.creationDate", TEXT, ""), + INTERNAL_ACCOUNT_EXPIRATION_DATE("internal.account.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE("internal.account.password.expirationDate", TEXT, ""), + INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED("internal.account.password.lastModified", TEXT, ""), QUOTA("quota", OBJECT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java index d1e48be9331..02ce5649729 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java @@ -31,6 +31,7 @@ import org.opencb.opencga.catalog.db.mongodb.converters.AnnotationConverter; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.AnnotationUtils; import org.opencb.opencga.catalog.utils.Constants; @@ -687,8 +688,7 @@ private List getNewAnnotationList(List annotationSetLis return annotationList; } - public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, Variable variable) throws CatalogException { long startTime = startQuery(); // We generate the generic document that should be inserted @@ -764,12 +764,9 @@ public OpenCGAResult addVariableToAnnotations(long studyUid, long variableSetId, * @param variableSetId Variable set id. * @param fieldId Field id corresponds with the variable name whose annotations have to be removed. * @return A OpenCGAResult object. - * @throws CatalogDBException if there is any unexpected error. - * @throws CatalogParameterException if there is any unexpected parameter. - * @throws CatalogAuthorizationException if the operation is not authorized. + * @throws CatalogException if there is any unexpected error or parameter, or if the operation is not authorized. */ - public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeAnnotationField(long studyUid, long variableSetId, String fieldId) throws CatalogException { long startTime = startQuery(); UpdateDocument updateDocument = new UpdateDocument(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index 445c1c6047a..ac6dd7af9fe 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -34,10 +34,8 @@ import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; @@ -578,7 +576,7 @@ public OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resou @Override public OpenCGAResult setToMembers(long studyId, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -598,8 +596,7 @@ public OpenCGAResult setToMembers(long studyId, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -652,7 +649,7 @@ private void setToMembers(List resourceIds, List members, List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); addToMembersGroupInStudy(studyId, members, clientSession); @@ -696,8 +693,7 @@ private void addToMembers(List resourceIds, List members, List studyIds, List members, List permissions) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult addToMembers(List studyIds, List members, List permissions) throws CatalogException { return runTransaction((clientSession) -> { long startTime = startQuery(); for (Long studyId : studyIds) { @@ -724,7 +720,7 @@ private void addToMembersGroupInStudy(long studyId, List members, Client @Override public OpenCGAResult removeFromMembers(List members, List aclParams) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -766,8 +762,7 @@ private void removeFromMembers(ClientSession clientSession, List resourceI } @Override - public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resetMembersFromAllEntries(long studyId, List members) throws CatalogException { if (members == null || members.isEmpty()) { throw new CatalogDBException("Missing 'members' array."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index e4fb4672afb..7a359e5a7eb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -263,7 +263,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va try { return runTransaction(clientSession -> transactionalUpdate(clientSession, result.first(), parameters, variableSetList, clinicalAuditList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update clinical analysis {}: {}", clinicalAnalysisId, e.getMessage(), e); throw new CatalogDBException("Could not update clinical analysis " + clinicalAnalysisId + ": " + e.getMessage(), e.getCause()); } @@ -695,7 +695,7 @@ public OpenCGAResult delete(ClinicalAnalysis clinicalAnalysis, List privateDelete(clientSession, clinicalAnalysis, clinicalAuditList)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete Clinical Analysis " + clinicalAnalysis.getId() + ": " + e.getMessage(), e.getCause()); @@ -713,7 +713,7 @@ public OpenCGAResult delete(Query query, List c try { result.append(runTransaction(clientSession -> privateDelete(clientSession, clinicalAnalysis, clinicalAuditList))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete Clinical Analysis {}: {}", clinicalAnalysis.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, clinicalAnalysis.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1021,8 +1021,7 @@ public OpenCGAResult nativeInsert(Map clinicalAnalysis, String u @Override public OpenCGAResult insert(long studyId, ClinicalAnalysis clinicalAnalysis, List variableSetList, - List clinicalAuditList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index 7f25b1ee7b9..6b948a722ba 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -96,7 +96,7 @@ public OpenCGAResult nativeInsert(Map cohort, String userId) thr @Override public OpenCGAResult insert(long studyId, Cohort cohort, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long startTime = startQuery(); @@ -275,7 +275,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, cohort, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update cohort {}: {}", cohort.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohort.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -545,7 +545,7 @@ public OpenCGAResult delete(Cohort cohort) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find cohort " + cohort.getId() + " with uid " + cohort.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohort.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete cohort '" + cohort.getId() + "': " + e.getMessage(), e.getCause()); } @@ -561,7 +561,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String cohortId = cohort.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, cohort))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete cohort {}: {}", cohortId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, cohortId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 9e8c35d00ab..97bfd92a671 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -115,8 +115,7 @@ public OpenCGAResult nativeInsert(Map family, String userId) thr @Override public OpenCGAResult insert(long studyId, Family family, List members, List variableSetList, - QueryOptions options) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + QueryOptions options) throws CatalogException { try { AtomicReference familyCopy = new AtomicReference<>(); OpenCGAResult result = runTransaction(clientSession -> { @@ -333,7 +332,7 @@ public OpenCGAResult update(long familyUid, ObjectMap parameters, List transactionalUpdate(clientSession, familyDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", familyDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update family " + familyDataResult.first().getId() + ": " + e.getMessage(), e.getCause()); @@ -368,7 +367,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, family, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update family {}: {}", family.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, family.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -784,7 +783,7 @@ public OpenCGAResult delete(Family family) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find family " + family.getId() + " with uid " + family.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", family.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete family " + family.getId() + ": " + e.getMessage(), e.getCause()); } @@ -801,7 +800,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String familyId = family.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, family))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete family {}: {}", familyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, familyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index f214620bf39..0f8383a9905 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -127,8 +127,7 @@ public OpenCGAResult nativeInsert(Map file, String userId) throw @Override public OpenCGAResult insert(long studyId, File file, List existingSamples, List nonExistingSamples, - List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List variableSetList, QueryOptions options) throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -144,7 +143,7 @@ public OpenCGAResult insert(long studyId, File file, List existingSample @Override public OpenCGAResult insertWithVirtualFile(long studyId, File file, File virtualFile, List existingSamples, List nonExistingSamples, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction( (clientSession) -> { long tmpStartTime = startQuery(); @@ -344,7 +343,7 @@ public OpenCGAResult update(long fileUid, ObjectMap parameters, List transactionalUpdate(clientSession, fileDataResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", fileDataResult.first().getPath(), e.getMessage(), e); throw new CatalogDBException("Could not update file " + fileDataResult.first().getPath() + ": " + e.getMessage(), e.getCause()); } @@ -370,7 +369,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, file, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update file {}: {}", file.getPath(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, file.getPath(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -931,7 +930,7 @@ public OpenCGAResult delete(File file, String status) try { return runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", file.getPath(), e.getMessage(), e); throw new CatalogDBException("Could not delete file " + file.getPath() + ": " + e.getMessage(), e.getCause()); } @@ -962,7 +961,7 @@ public OpenCGAResult delete(Query query, String status) Document fileDocument = iterator.next(); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, fileDocument, status))); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete file {}: {}", fileDocument.getString(QueryParams.PATH.key()), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, fileDocument.getString(QueryParams.ID.key()), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 13dbb40741c..03a6704af17 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -119,7 +119,7 @@ public OpenCGAResult nativeInsert(Map individual, String userId) @Override public OpenCGAResult insert(long studyId, Individual individual, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -329,7 +329,7 @@ public OpenCGAResult update(long individualUid, ObjectMap parameters, List transactionalUpdate(clientSession, individualUid, parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not update individual: " + e.getMessage(), e.getCause()); } } @@ -363,7 +363,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> transactionalUpdate(clientSession, individual, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update individual {}: {}", individual.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individual.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1051,7 +1051,7 @@ public OpenCGAResult delete(Individual individual) throws CatalogDBException, Ca throw new CatalogDBException("Could not find individual " + individual.getId() + " with uid " + individual.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individual.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e); } @@ -1068,7 +1068,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String individualId = individual.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, individual))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete individual {}: {}", individualId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, individualId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java index 2181e1e50bf..0c833de52ed 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.InterpretationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; import org.opencb.opencga.catalog.utils.Constants; @@ -120,8 +121,7 @@ public OpenCGAResult nativeInsert(Map interpretation, String use @Override public OpenCGAResult insert(long studyId, Interpretation interpretation, ParamUtils.SaveInterpretationAs action, - List clinicalAuditList) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + List clinicalAuditList) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting interpretation insert transaction for interpretation id '{}'", interpretation.getId()); @@ -635,7 +635,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List try { return runTransaction(clientSession -> update(clientSession, interpretation.first(), parameters, clinicalAuditList, action, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not update interpretation " + interpretationId + ": " + e.getMessage(), e.getCause()); } @@ -654,7 +654,7 @@ public OpenCGAResult revert(long id, int previousVersion, List c return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); }); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); throw new CatalogDBException("Could not delete interpretation " + interpretation.getId() + ": " + e.getMessage(), e.getCause()); } @@ -852,7 +852,7 @@ public OpenCGAResult delete(Query query, List cli return delete(clientSession, interpretation, clinicalAuditList, clinicalResult.first()); })); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete interpretation {}: {}", interpretationId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, interpretationId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 53c9a5f157e..d89e0020f93 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -100,8 +100,7 @@ public OpenCGAResult nativeInsert(Map job, String userId) throws } @Override - public OpenCGAResult insert(long studyId, Job job, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyId, Job job, QueryOptions options) throws CatalogException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -228,7 +227,7 @@ public OpenCGAResult update(long jobUid, ObjectMap parameters, QueryOptions quer try { return runTransaction(session -> privateUpdate(session, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update job " + dataResult.first().getId() + ": " + e.getMessage(), e.getCause()); } @@ -254,7 +253,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Job job = iterator.next(); try { result.append(runTransaction(session -> privateUpdate(session, job, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update job {}: {}", job.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, job.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -306,7 +305,7 @@ public OpenCGAResult delete(Job job) throws CatalogDBException, CatalogParameter throw new CatalogDBException("Could not find job " + job.getId() + " with uid " + job.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", job.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete job " + job.getId() + ": " + e.getMessage(), e.getCause()); } @@ -322,7 +321,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String jobId = job.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, job))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete job {}: {}", jobId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, jobId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index b663ffdf755..32c10c362f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -94,16 +94,15 @@ public MongoDBAdaptor(Configuration configuration, Logger logger) { } public interface TransactionBodyWithException { - T execute(ClientSession session) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException; + T execute(ClientSession session) throws CatalogException; } - protected T runTransaction(TransactionBodyWithException body) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + protected T runTransaction(TransactionBodyWithException body) throws CatalogException { return runTransaction(body, null); } protected T runTransaction(TransactionBodyWithException inputBody, Consumer onException) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { ClientSession session = dbAdaptorFactory.getMongoDataStore().startSession(); try { TransactionBodyWithException body; @@ -125,7 +124,7 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum return session.withTransaction(() -> { try { return body.execute(session); - } catch (CatalogDBException | CatalogAuthorizationException | CatalogParameterException e) { + } catch (CatalogException e) { throw new CatalogDBRuntimeException(e); } }); @@ -148,6 +147,12 @@ protected T runTransaction(TransactionBodyWithException inputBody, Consum onException.accept(cause); } throw cause; + } else if (e.getCause() instanceof CatalogAuthenticationException) { + CatalogAuthenticationException cause = (CatalogAuthenticationException) e.getCause(); + if (onException != null) { + onException.accept(cause); + } + throw cause; } else { throw e; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 621c537cddd..e262c3d3595 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -27,10 +27,8 @@ import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.managers.NoteManager; import org.opencb.opencga.core.api.ParamConstants; @@ -261,7 +259,7 @@ public MetaDBAdaptor getCatalogMetaDBAdaptor(String organizationId) throws Catal @Override public OpenCGAResult createOrganization(Organization organization, QueryOptions options, String userId) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { OrganizationMongoDBAdaptorFactory orgFactory = getOrganizationMongoDBAdaptorFactory(organization.getId(), false); if (orgFactory != null && orgFactory.isCatalogDBReady()) { throw new CatalogDBException("Organization '" + organization.getId() + "' already exists."); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java index 820f15ed2f7..0b953919713 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/NoteMongoDBAdaptor.java @@ -14,6 +14,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -51,8 +52,7 @@ public NoteMongoDBAdaptor(MongoDBCollection noteCollection, MongoDBCollection ar } @Override - public OpenCGAResult insert(Note note) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Note note) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting note insert transaction for note id '{}'", note.getId()); @@ -142,7 +142,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, QueryOptions queryOp throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateUpdate(clientSession, uid, parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update note: {}", e.getMessage(), e); throw new CatalogDBException("Could not update note: " + e.getMessage(), e.getCause()); } @@ -262,7 +262,7 @@ public OpenCGAResult delete(Note note) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, note)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { throw new CatalogDBException("Could not delete note " + note.getId() + ": " + e.getMessage(), e); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java index aa090f498d1..b971f5a5740 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java @@ -17,6 +17,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.OrganizationCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.Constants; @@ -52,8 +53,7 @@ public OrganizationMongoDBAdaptor(MongoDBCollection organizationCollection, Conf } @Override - public OpenCGAResult insert(Organization organization, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Organization organization, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting organization insert transaction for organization id '{}'", organization.getId()); @@ -124,7 +124,7 @@ public OpenCGAResult update(String organizationId, ObjectMap param try { QueryOptions options = queryOptions != null ? new QueryOptions(queryOptions) : QueryOptions.empty(); return runTransaction(clientSession -> privateUpdate(clientSession, organizationId, parameters, options)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update organization {}: {}", organizationId, e.getMessage(), e); throw new CatalogDBException("Could not update organization " + organizationId + ": " + e.getMessage(), e.getCause()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index be76b219705..9884bbccc6c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -121,7 +121,7 @@ public class OrganizationMongoDBAdaptorFactory { private final NoteMongoDBAdaptor notesDBAdaptor; private final OrganizationMongoDBAdaptor organizationDBAdaptor; - private final UserMongoDBAdaptor userDBAdaptor; + private UserMongoDBAdaptor userDBAdaptor; private final ProjectMongoDBAdaptor projectDBAdaptor; private final StudyMongoDBAdaptor studyDBAdaptor; private final IndividualMongoDBAdaptor individualDBAdaptor; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index d82535b9351..fcc8f32fd6f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -33,6 +33,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -87,8 +88,7 @@ public MongoDBCollection getPanelArchiveCollection() { } @Override - public OpenCGAResult insert(long studyUid, List panelList) throws CatalogDBException, CatalogParameterException, - CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, List panelList) throws CatalogException { if (panelList == null || panelList.isEmpty()) { throw new CatalogDBException("Missing panel list"); } @@ -108,8 +108,7 @@ public OpenCGAResult insert(long studyUid, List panelList) throws Catalog } @Override - public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(long studyUid, Panel panel, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting insert transaction of panel id '{}'", panel.getId()); @@ -291,7 +290,7 @@ public OpenCGAResult update(long panelUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, dataResult.first(), parameters, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", dataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update panel '" + dataResult.first().getId() + "': " + e.getMessage(), e.getCause()); } @@ -317,7 +316,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Panel panel = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, panel, parameters, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update panel {}: {}", panel.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panel.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -464,7 +463,7 @@ public OpenCGAResult delete(Panel panel) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find panel " + panel.getId() + " with uid " + panel.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panel.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete panel '" + panel.getId() + "': " + e.getMessage(), e.getCause()); } @@ -480,7 +479,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParam String panelId = panel.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, panel))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete panel {}: {}", panelId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, panelId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index 89d90aab932..3a91bc26284 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -32,6 +32,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.ProjectCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.UuidUtils; @@ -82,8 +83,7 @@ public OpenCGAResult nativeInsert(Map project, String userId) th } @Override - public OpenCGAResult insert(Project project, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(Project project, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting project insert transaction for project id '{}'", project.getId()); @@ -201,7 +201,7 @@ public OpenCGAResult update(long projectUid, ObjectMap parameters, QueryOptions try { return runTransaction(clientSession -> privateUpdate(clientSession, projectDataResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", projectDataResult.first().getId(), e.getMessage(), e); throw new CatalogDBException("Could not update project '" + projectDataResult.first().getId() + "': " + e.getMessage(), e.getCause()); @@ -220,7 +220,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Project project = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, project, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -383,7 +383,7 @@ public OpenCGAResult delete(long id, QueryOptions queryOptions) throws CatalogDB public OpenCGAResult delete(Project project) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateDelete(clientSession, project)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete project '" + project.getId() + "': " + e.getMessage(), e.getCause()); } @@ -412,7 +412,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, project))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete project {}: {}", project.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, project.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index 198c9fce456..d9058193e7d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -39,6 +39,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.SampleCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.IndividualManager; import org.opencb.opencga.catalog.managers.SampleManager; @@ -199,7 +200,7 @@ Sample insert(ClientSession clientSession, long studyUid, Sample sample, List insert(long studyId, Sample sample, List variableSetList, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting sample insert transaction for sample id '{}'", sample.getId()); @@ -262,7 +263,7 @@ public OpenCGAResult update(long uid, ObjectMap parameters, List va try { return runTransaction(clientSession -> privateUpdate(clientSession, documentResult.first(), parameters, variableSetList, queryOptions)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); throw new CatalogDBException("Could not update sample " + sampleId + ": " + e.getMessage(), e.getCause()); } @@ -297,7 +298,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, sampleDocument, parameters, variableSetList, queryOptions))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -857,8 +858,7 @@ public OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) } @Override - public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult setRgaIndexes(long studyUid, List sampleUids, RgaIndex rgaIndex) throws CatalogException { ObjectMap params; try { params = new ObjectMap(getDefaultObjectMapper().writeValueAsString(rgaIndex)); @@ -943,7 +943,7 @@ public OpenCGAResult delete(Sample sample) throws CatalogDBException, CatalogPar throw new CatalogDBException("Could not find sample " + sample.getId() + " with uid " + sample.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sample.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e); } @@ -960,7 +960,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { try { result.append(runTransaction(clientSession -> privateDelete(clientSession, sample))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete sample {}: {}", sampleId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, sampleId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index 5c6e0ff2035..c91fd4c3665 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -38,6 +38,7 @@ import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.FqnUtils; @@ -490,8 +491,7 @@ OpenCGAResult removeUsersFromAdminsGroup(ClientSession clientSession, Lis } @Override - public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult removeUsersFromAllGroups(long studyId, List users) throws CatalogException { if (users == null || users.size() == 0) { throw new CatalogDBException("Unable to remove users from groups. List of users is empty"); } @@ -546,8 +546,7 @@ public OpenCGAResult syncGroup(long studyId, String groupId, Group.Sync s } @Override - public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogDBException("Missing user field"); } @@ -600,8 +599,7 @@ public OpenCGAResult resyncUserWithSyncedGroups(String user, List @Override public OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, - ParamUtils.AddRemoveAction action) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + ParamUtils.AddRemoveAction action) throws CatalogException { if (StringUtils.isEmpty(user)) { throw new CatalogParameterException("Missing user parameter"); @@ -820,7 +818,7 @@ public OpenCGAResult createVariableSet(long studyId, VariableSet va @Override public OpenCGAResult addFieldToVariableSet(long studyUid, long variableSetId, Variable variable, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableNotInVariableSet(variableSet.first(), variable.getId()); @@ -897,9 +895,7 @@ public OpenCGAResult renameFieldVariableSet(long variableSetId, Str @Override public OpenCGAResult removeFieldFromVariableSet(long studyUid, long variableSetId, String name, String user) - throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { - long startTime = startQuery(); - + throws CatalogException { OpenCGAResult variableSet = getVariableSet(variableSetId, new QueryOptions(), user); checkVariableInVariableSet(variableSet.first(), name); @@ -1167,8 +1163,7 @@ public OpenCGAResult getVariableSets(Query query, QueryOptions quer } @Override - public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult deleteVariableSet(long studyUid, VariableSet variableSet, boolean force) throws CatalogException { try { return runTransaction(clientSession -> { if (force) { @@ -1341,7 +1336,7 @@ public OpenCGAResult update(long studyUid, ObjectMap parameters, QueryOptions qu try { return runTransaction(clientSession -> privateUpdate(clientSession, studyResult.first(), parameters)); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", studyId, e.getMessage(), e); throw new CatalogDBException("Could not update study '" + studyId + "': " + e.getMessage(), e.getCause()); } @@ -1364,7 +1359,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer Study study = iterator.next(); try { result.append(runTransaction(clientSession -> privateUpdate(clientSession, study, parameters))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not update study {}: {}", study.getId(), e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, study.getId(), e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); @@ -1478,7 +1473,7 @@ public OpenCGAResult delete(Study study) throws CatalogDBException, CatalogParam throw new CatalogDBException("Could not find study " + study.getId() + " with uid " + study.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); - } catch (CatalogDBException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", study.getId(), e.getMessage(), e); throw new CatalogDBException("Could not delete study " + study.getId() + ": " + e.getMessage(), e.getCause()); } @@ -1494,7 +1489,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException { String studyId = study.getString(QueryParams.ID.key()); try { result.append(runTransaction(clientSession -> privateDelete(clientSession, study))); - } catch (CatalogDBException | CatalogParameterException | CatalogAuthorizationException e) { + } catch (CatalogException e) { logger.error("Could not delete study {}: {}", studyId, e.getMessage(), e); result.getEvents().add(new Event(Event.Type.ERROR, studyId, e.getMessage())); result.setNumMatches(result.getNumMatches() + 1); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 19a9a6cd1f3..24c71463e32 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -18,7 +18,9 @@ import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.bson.Document; @@ -36,13 +38,13 @@ import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.UserConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.CatalogMongoDBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.managers.StudyManager; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.project.Project; @@ -54,6 +56,7 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.function.Consumer; @@ -71,7 +74,22 @@ public class UserMongoDBAdaptor extends CatalogMongoDBAdaptor implements UserDBA private final MongoDBCollection deletedUserCollection; private UserConverter userConverter; - private static final String PRIVATE_PASSWORD = "_password"; + // --- Password constants --- + public static final String HASH = "hash"; + public static final String SALT = "salt"; + + public static final String PRIVATE_PASSWORD = "_password"; + + public static final String CURRENT = "current"; + private static final String PRIVATE_PASSWORD_CURRENT = "_password." + CURRENT; + private static final String PRIVATE_PASSWORD_CURRENT_HASH = PRIVATE_PASSWORD_CURRENT + "." + HASH; + private static final String PRIVATE_PASSWORD_CURRENT_SALT = PRIVATE_PASSWORD_CURRENT + "." + SALT; + + public static final String ARCHIVE = "archive"; + public static final String PRIVATE_PASSWORD_ARCHIVE = "_password." + ARCHIVE; + private static final String PRIVATE_PASSWORD_ARCHIVE_HASH = PRIVATE_PASSWORD_ARCHIVE + "." + HASH; + private static final String PRIVATE_PASSWORD_ARCHIVE_SALT = PRIVATE_PASSWORD_ARCHIVE + "." + SALT; + // -------------------------- public UserMongoDBAdaptor(MongoDBCollection userCollection, MongoDBCollection deletedUserCollection, Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { @@ -93,8 +111,7 @@ boolean exists(ClientSession clientSession, String userId) throws CatalogDBExcep } @Override - public OpenCGAResult insert(User user, String password, QueryOptions options) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult insert(User user, String password, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); @@ -117,7 +134,18 @@ private void insert(ClientSession clientSession, User user, String password) thr Document userDocument = userConverter.convertToStorageType(user); userDocument.append(ID, user.getId()); - userDocument.append(PRIVATE_PASSWORD, encryptPassword(password)); + + Document privatePassword = new Document(); + if (StringUtils.isNotEmpty(password)) { + String salt = PasswordUtils.getStrongRandomSalt(); + String hash = encryptPassword(password, salt); + Document passwordDoc = new Document() + .append(HASH, hash) + .append(SALT, salt); + privatePassword.put(CURRENT, passwordDoc); + privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); + } + userDocument.put(PRIVATE_PASSWORD, privatePassword); userCollection.insert(clientSession, userDocument, null); } @@ -130,47 +158,133 @@ public OpenCGAResult get(String userId, QueryOptions options) } @Override - public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) - throws CatalogDBException, CatalogAuthenticationException { - Document bson = new Document(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(oldPassword)); - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); - - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); - } - return new OpenCGAResult(result); + public OpenCGAResult changePassword(String userId, String oldPassword, String newPassword) throws CatalogException { + return setPassword(userId, oldPassword, newPassword); } @Override - public void authenticate(String userId, String password) throws CatalogAuthenticationException { - Document bson; - try { - bson = new Document() - .append(ID, userId) - .append(PRIVATE_PASSWORD, encryptPassword(password)); - } catch (CatalogDBException e) { - throw new CatalogAuthenticationException("Could not encrypt password: " + e.getMessage(), e); + public void authenticate(String userId, String password) throws CatalogDBException, CatalogAuthenticationException { + Bson query = Filters.and( + Filters.eq(QueryParams.ID.key(), userId), + // TODO: Deprecated. Remove Filters.or using the deprecated account authentication id + Filters.or( + Filters.eq(DEPRECATED_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA), + Filters.eq(INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), AuthenticationOrigin.AuthenticationType.OPENCGA) + ) + ); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult dataResult = userCollection.find(query, projection, QueryOptions.empty()); + if (dataResult.getNumResults() == 0) { + throw new CatalogDBException("User " + userId + " not found"); + } + Document userDocument = dataResult.first(); + Object rootPasswordObject = userDocument.get(PRIVATE_PASSWORD); + Document rootPasswordDoc; + // TODO: Remove this block of code in the future when all users have been migrated + if (rootPasswordObject instanceof String) { + if (ParamConstants.OPENCGA_USER_ID.equals(userId)) { + logger.warn("User {} is using the deprecated password format. Please, migrate your code as soon as possible.", userId); + if (!encryptPassword(password, "").equals(rootPasswordObject)) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); + } + return; + } else { + throw new CatalogDBException("User '" + userId + "' is using the deprecated password format. Please, ask your" + + " administrator to run the pending migrations to fix this issue."); + } + } else { + rootPasswordDoc = (Document) rootPasswordObject; + } + // TODO: End of block of code to remove (and replace using commented code below) +// Document rootPasswordDoc = userDocument.get(PRIVATE_PASSWORD, Document.class); + if (rootPasswordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); } - if (userCollection.count(bson).getNumMatches() == 0) { - throw CatalogAuthenticationException.incorrectUserOrPassword("Internal"); + Document passwordDoc = rootPasswordDoc.get(CURRENT, Document.class); + if (passwordDoc == null) { + throw new CatalogDBException("Critical error. User '" + userId + "' does not have any password set. Please, contact" + + " with the developers."); + } + + String salt = passwordDoc.getString(SALT); + String hash = encryptPassword(password, salt); + if (!hash.equals(passwordDoc.getString(HASH))) { + throw CatalogAuthenticationException.incorrectUserOrPassword(AuthenticationOrigin.AuthenticationType.OPENCGA.name()); } } @Override - public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogDBException { - Query query = new Query(QueryParams.ID.key(), userId); - query.append(QueryParams.EMAIL.key(), email); - Bson bson = parseQuery(query); + public OpenCGAResult resetPassword(String userId, String email, String newPassword) throws CatalogException { + return setPassword(userId, null, newPassword); + } - Bson set = Updates.set(PRIVATE_PASSWORD, encryptPassword(newPassword)); + public OpenCGAResult setPassword(String userId, @Nullable String oldPassword, String newPassword) throws CatalogException { + String prefixErrorMsg = "Could not update the password. "; + return runTransaction(clientSession -> { + // 1. Obtain archived passwords + Bson query = Filters.eq(QueryParams.ID.key(), userId); + Bson projection = Projections.include(PRIVATE_PASSWORD); + DataResult userQueryResult = userCollection.find(clientSession, query, projection, QueryOptions.empty()); + if (userQueryResult.getNumResults() == 0) { + throw new CatalogDBException(prefixErrorMsg + "User " + userId + " not found."); + } + Document userDoc = userQueryResult.first(); + Document passwordDoc = userDoc.get(PRIVATE_PASSWORD, Document.class); + + // 1.1. Check oldPassword + if (StringUtils.isNotEmpty(oldPassword)) { + Document currentPasswordDoc = passwordDoc.get(CURRENT, Document.class); + String currentSalt = currentPasswordDoc.getString(SALT); + String currentHash = encryptPassword(oldPassword, currentSalt); + if (!currentHash.equals(currentPasswordDoc.getString(HASH))) { + throw new CatalogAuthenticationException(prefixErrorMsg + "Please, verify that the current password is correct."); + } + } - DataResult result = userCollection.update(bson, set, null); - if (result.getNumUpdated() == 0) { //0 query matches. - throw new CatalogDBException("Bad user or email"); - } - return new OpenCGAResult(result); + // 2. Check new password has not been used before + for (Document document : passwordDoc.getList(ARCHIVE, Document.class)) { + String hashValue = document.getString(HASH); + String saltValue = document.getString(SALT); + String encryptedPassword = encryptPassword(newPassword, saltValue); + if (encryptedPassword.equals(hashValue)) { + throw new CatalogAuthenticationException(prefixErrorMsg + "The new password has already been used." + + " Please, use a different one."); + } + } + + // 3. Generate new salt for current password + String newSalt = PasswordUtils.getStrongRandomSalt(); + String newHash = encryptPassword(newPassword, newSalt); + + // 4. Generate update document + UpdateDocument updateDocument = new UpdateDocument(); + // add to current + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_HASH, newHash); + updateDocument.getSet().put(PRIVATE_PASSWORD_CURRENT_SALT, newSalt); + + // add to archive + Document document = new Document() + .append(HASH, newHash) + .append(SALT, newSalt); + updateDocument.getPush().put(PRIVATE_PASSWORD_ARCHIVE, document); + + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED.key(), TimeUtils.getTime()); + if (configuration.getAccount().getPasswordExpirationDays() > 0) { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + String stringDate = TimeUtils.getTime(date); + updateDocument.getSet().put(INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE.key(), stringDate); + } + Document update = updateDocument.toFinalUpdateDocument(); + + logger.debug("Change password: query '{}'; update: '{}'", query.toBsonDocument(), update); + DataResult result = userCollection.update(clientSession, query, update, null); + if (result.getNumUpdated() == 0) { + throw new CatalogAuthenticationException("Could not update the password. Please, verify that the current password is" + + " correct."); + } + return new OpenCGAResult(result); + }, e -> logger.error("User {}: {}", userId, e.getMessage())); } @Override @@ -409,7 +523,7 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog for (Document user : queryResult.getResults()) { ArrayList projects = (ArrayList) user.get("projects"); - if (projects.size() > 0) { + if (CollectionUtils.isNotEmpty(projects)) { List projectsTmp = new ArrayList<>(projects.size()); for (Document project : projects) { Query query1 = new Query(ProjectDBAdaptor.QueryParams.UID.key(), project.get(ProjectDBAdaptor @@ -429,7 +543,7 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException { UpdateDocument document = new UpdateDocument(); - final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), ACCOUNT_EXPIRATION_DATE.key()}; + final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), INTERNAL_ACCOUNT_EXPIRATION_DATE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); if (parameters.containsKey(QueryParams.INTERNAL_STATUS_ID.key())) { @@ -437,7 +551,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions quer document.getSet().put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); } - final String[] acceptedIntParams = {INTERNAL_FAILED_ATTEMPTS.key()}; + final String[] acceptedIntParams = {INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key()}; filterIntParams(parameters, document.getSet(), acceptedIntParams); final String[] acceptedObjectParams = {QueryParams.QUOTA.key()}; @@ -618,18 +732,10 @@ public void forEach(Query query, Consumer action, QueryOptions o } } - public static void main(String[] args) throws CatalogDBException { - System.out.println(encryptPassword("admin")); - } - - private static String encryptPassword(String password) throws CatalogDBException { + private static String encryptPassword(String password, String salt) throws CatalogDBException { if (StringUtils.isNotEmpty(password)) { - if (password.matches("^[a-fA-F0-9]{40}$")) { - // Password already cyphered - return password; - } try { - return CryptoUtils.sha1(password); + return CryptoUtils.sha1(password + salt); } catch (NoSuchAlgorithmException e) { throw new CatalogDBException("Could not encrypt password", e); } @@ -670,8 +776,8 @@ private Bson parseQuery(Query query) throws CatalogDBException { case EMAIL: case ORGANIZATION: case INTERNAL_STATUS_DATE: - case ACCOUNT_AUTHENTICATION_ID: - case ACCOUNT_CREATION_DATE: + case INTERNAL_ACCOUNT_AUTHENTICATION_ID: + case INTERNAL_ACCOUNT_CREATION_DATE: case TOOL_ID: case TOOL_NAME: case TOOL_ALIAS: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java index 604a6073d0f..837349b6b8a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/UserConverter.java @@ -18,32 +18,77 @@ import org.apache.avro.generic.GenericRecord; import org.bson.Document; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserInternal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Created by pfurio on 19/01/16. */ public class UserConverter extends OpenCgaMongoConverter { + protected static Logger logger = LoggerFactory.getLogger(UserConverter.class); + public UserConverter() { super(User.class); getObjectMapper().addMixIn(GenericRecord.class, GenericRecordAvroJsonMixin.class); } + @Override + public Document convertToStorageType(User object) { + Document userDocument = super.convertToStorageType(object); + removeDeprecatedAccountObject(userDocument); + return userDocument; + } + @Override public User convertToDataModelType(Document document) { - // TODO: Remove this piece of code once we are sure User contains the migrated new account type from 1.4.2 - Document account = (Document) document.get("account"); - if (account != null && account.get("authentication") == null) { - String authOrigin = account.getString("authOrigin"); - Document authentication = new Document() - .append("id", authOrigin) - .append("application", false); - account.put("authentication", authentication); + User user = super.convertToDataModelType(document); + + restoreFromDeprecatedAccountObject(user); + addToDeprecatedAccountObject(user); + + return user; + } + + /** + * Remove 'account' object from the User document so it is no longer stored in the database. + * Remove after a few releases. + * + * @param userDocument User document. + */ + @Deprecated + private void removeDeprecatedAccountObject(Document userDocument) { + userDocument.remove(UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key()); + } + + /** + * Restores information from the account object to the corresponding internal.account object. + * Added to maintain backwards compatibility with the deprecated account object in TASK-6494 (v3.2.1) + * Remove after a few releases. + * + * @param user User object. + */ + @Deprecated + private void restoreFromDeprecatedAccountObject(User user) { + if (user.getAccount() != null) { + if (user.getInternal() == null) { + user.setInternal(new UserInternal()); + } + user.getInternal().setAccount(user.getAccount()); + logger.warn("Restoring user account information from deprecated account object to internal.account object. " + + "Please, run 'opencga-admin.sh migration run'."); } + } - return super.convertToDataModelType(document); + private void addToDeprecatedAccountObject(User user) { + // Add account to deprecated place + if (user.getInternal() != null && user.getInternal().getAccount() != null && user.getAccount() == null) { + user.setAccount(user.getInternal().getAccount()); + } } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java index 3853e91607f..8ac1b5c385f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java @@ -76,6 +76,11 @@ public static CatalogAuthenticationException accountIsExpired(String userId, Str + " talk to your organization owner/administrator."); } + public static CatalogAuthenticationException passwordExpired(String userId, String expirationDate) { + return new CatalogAuthenticationException("The password for the user account '" + userId + "' expired on " + expirationDate + + ". Please, reset your password or talk to your organization owner/administrator."); + } + public static CatalogAuthenticationException userNotAllowed(String domain) { return new CatalogAuthenticationException(domain + ": User not allowed to access the system."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java index 3d3611c5414..da7566340ac 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java @@ -73,68 +73,72 @@ public abstract class AbstractManager { logger = LoggerFactory.getLogger(this.getClass()); } + protected DBAdaptorFactory getCatalogDBAdaptorFactory() { + return catalogDBAdaptorFactory; + } + protected MigrationDBAdaptor getMigrationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getMigrationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getMigrationDBAdaptor(organization); } protected MetaDBAdaptor getCatalogMetaDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogMetaDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogMetaDBAdaptor(organization); } protected OrganizationDBAdaptor getOrganizationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogOrganizationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogOrganizationDBAdaptor(organization); } protected UserDBAdaptor getUserDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogUserDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogUserDBAdaptor(organization); } protected ProjectDBAdaptor getProjectDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogProjectDbAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogProjectDbAdaptor(organization); } protected StudyDBAdaptor getStudyDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogStudyDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogStudyDBAdaptor(organization); } protected FileDBAdaptor getFileDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogFileDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogFileDBAdaptor(organization); } protected SampleDBAdaptor getSampleDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogSampleDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogSampleDBAdaptor(organization); } protected IndividualDBAdaptor getIndividualDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogIndividualDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogIndividualDBAdaptor(organization); } protected JobDBAdaptor getJobDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organization); } protected AuditDBAdaptor getAuditDbAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogAuditDbAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogAuditDbAdaptor(organization); } protected CohortDBAdaptor getCohortDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogCohortDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogCohortDBAdaptor(organization); } protected PanelDBAdaptor getPanelDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogPanelDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogPanelDBAdaptor(organization); } protected FamilyDBAdaptor getFamilyDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getCatalogFamilyDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getCatalogFamilyDBAdaptor(organization); } protected ClinicalAnalysisDBAdaptor getClinicalAnalysisDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getClinicalAnalysisDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getClinicalAnalysisDBAdaptor(organization); } protected InterpretationDBAdaptor getInterpretationDBAdaptor(String organization) throws CatalogDBException { - return catalogDBAdaptorFactory.getInterpretationDBAdaptor(organization); + return getCatalogDBAdaptorFactory().getInterpretationDBAdaptor(organization); } protected void fixQueryObject(Query query) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java index 09bc57aabf1..7edf1030496 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java @@ -67,7 +67,7 @@ public OpenCGAResult userSearch(String organizationId, Query query, QueryO query.remove(ParamConstants.USER); } if (query.containsKey(ParamConstants.USER_CREATION_DATE)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_CREATION_DATE.key(), query.get(ParamConstants.USER_CREATION_DATE)); query.remove(ParamConstants.USER_CREATION_DATE); } @@ -134,7 +134,7 @@ public List getOrganizationIds(String token) throws CatalogException { JwtPayload payload = catalogManager.getUserManager().validateToken(token); try { authorizationManager.checkIsOpencgaAdministrator(payload); - List organizationIds = catalogDBAdaptorFactory.getOrganizationIds(); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); auditManager.audit(ParamConstants.ADMIN_ORGANIZATION, payload.getUserId(), Enums.Action.FETCH_ORGANIZATION_IDS, Enums.Resource.STUDY, "", "", "", "", new ObjectMap(), new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 1409b1e9d4c..030505395d0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -48,7 +48,6 @@ import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -278,7 +277,7 @@ private void privateInstall(String algorithm, String secretKey, String password, organizationConfiguration, null), QueryOptions.empty(), null); - User user = new User(OPENCGA, new Account().setExpirationDate("")) + User user = new User(OPENCGA) .setEmail(StringUtils.isEmpty(email) ? "opencga@admin.com" : email) .setOrganization(ADMIN_ORGANIZATION); userManager.create(user, password, null); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index b8b9ba55bae..fe7ffc18e44 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -421,7 +421,7 @@ public OpenCGAResult kill(String studyStr, String jobId, String token) thro } ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_KILL_JOB_REQUESTED.key(), true); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, QueryOptions.empty()); auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), @@ -816,7 +816,7 @@ public DBIterator iterator(String studyId, Query query, QueryOptions option public OpenCGAResult countInOrganization(String organizationId, Query query, String token) throws CatalogException { JwtPayload jwtPayload = userManager.validateToken(token); authorizationManager.checkIsOpencgaAdministrator(jwtPayload); - return catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).count(query); + return getCatalogDBAdaptorFactory().getCatalogJobDBAdaptor(organizationId).count(query); } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java index e81112cd04f..f32c8422dec 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/NoteManager.java @@ -54,7 +54,7 @@ private OpenCGAResult internalGet(String organizationId, long studyUid, St } else { query.put(NoteDBAdaptor.QueryParams.ID.key(), noteId); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, QueryOptions.empty()); if (result.getNumResults() == 0) { throw CatalogException.notFound("note", Collections.singletonList(noteId)); } @@ -88,7 +88,7 @@ public OpenCGAResult searchOrganizationNote(Query query, QueryOptions opti queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; @@ -131,7 +131,7 @@ public OpenCGAResult searchStudyNote(String studyStr, Query query, QueryOp queryCopy.put(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PUBLIC); } - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(queryCopy, optionsCopy); auditManager.auditSearch(organizationId, tokenPayload.getUserId(), Enums.Resource.NOTE, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; @@ -209,11 +209,11 @@ public OpenCGAResult createStudyNote(String studyStr, NoteCreateParams not private OpenCGAResult create(Note note, QueryOptions options, JwtPayload tokenPayload) throws CatalogException { String organizationId = tokenPayload.getOrganization(); validateNewNote(note, tokenPayload.getUserId()); - OpenCGAResult insert = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).insert(note); + OpenCGAResult insert = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).insert(note); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created note Query query = new Query(NoteDBAdaptor.QueryParams.UID.key(), note.getUid()); - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(query, options); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } return insert; @@ -302,10 +302,11 @@ private OpenCGAResult update(long noteUid, NoteUpdateParams noteUpdatePara // Write who's performing the update updateMap.put(NoteDBAdaptor.QueryParams.USER_ID.key(), tokenPayload.getUserId()); - OpenCGAResult update = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, options); + OpenCGAResult update = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).update(noteUid, updateMap, + options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated note - OpenCGAResult result = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); + OpenCGAResult result = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(organizationId).get(noteUid, options); update.setResults(result.getResults()); } return update; @@ -378,7 +379,7 @@ public OpenCGAResult deleteStudyNote(String studyStr, String noteId, Query } private OpenCGAResult delete(Note note, QueryOptions options, JwtPayload jwtPayload) throws CatalogException { - OpenCGAResult delete = catalogDBAdaptorFactory.getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); + OpenCGAResult delete = getCatalogDBAdaptorFactory().getCatalogNoteDBAdaptor(jwtPayload.getOrganization()).delete(note); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { delete.setResults(Collections.singletonList(note)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java index 90b83a92728..20ec12a7f2b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java @@ -185,7 +185,7 @@ public OpenCGAResult create(OrganizationCreateParams organizationC organization = organizationCreateParams.toOrganization(); validateOrganizationForCreation(organization, userId); - queryResult = catalogDBAdaptorFactory.createOrganization(organization, options, userId); + queryResult = getCatalogDBAdaptorFactory().createOrganization(organization, options, userId); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { OpenCGAResult result = getOrganizationDBAdaptor(organization.getId()).get(options); organization = result.first(); @@ -211,7 +211,7 @@ public OpenCGAResult create(OrganizationCreateParams organizationC auditManager.auditCreate(ParamConstants.ADMIN_ORGANIZATION, userId, Enums.Resource.ORGANIZATION, organization.getId(), "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); try { - catalogDBAdaptorFactory.deleteOrganization(organization); + getCatalogDBAdaptorFactory().deleteOrganization(organization); } catch (Exception e1) { logger.error("Error deleting organization from catalog after failing creating the folder in the filesystem", e1); throw e; @@ -327,9 +327,6 @@ public OpenCGAResult updateUser(@Nullable String organizationId, String us throw new CatalogException("Max CPU cannot be negative"); } } - if (updateParams.getAccount() != null && StringUtils.isNotEmpty(updateParams.getAccount().getExpirationDate())) { - ParamUtils.checkDateIsNotExpired(updateParams.getAccount().getExpirationDate(), "expirationDate"); - } ObjectMap updateMap; try { @@ -337,6 +334,12 @@ public OpenCGAResult updateUser(@Nullable String organizationId, String us } catch (JsonProcessingException e) { throw new CatalogException("Could not parse OrganizationUserUpdateParams object: " + e.getMessage(), e); } + + if (updateParams.getInternal() != null && updateParams.getInternal().getAccount() != null + && StringUtils.isNotEmpty(updateParams.getInternal().getAccount().getExpirationDate())) { + ParamUtils.checkDateIsNotExpired(updateParams.getInternal().getAccount().getExpirationDate(), "expirationDate"); + } + OpenCGAResult updateResult = getUserDBAdaptor(myOrganizationId).update(userId, updateMap); auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); @@ -592,7 +595,7 @@ Set getOrganizationOwnerAndAdmins(String organizationId) throws CatalogE public List getOrganizationIds(String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); authorizationManager.checkIsOpencgaAdministrator(tokenPayload, "get all organization ids"); - return catalogDBAdaptorFactory.getOrganizationIds(); + return getCatalogDBAdaptorFactory().getOrganizationIds(); } private void privatizeResults(OpenCGAResult result) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index beda16bfa2e..dcfecf341f6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -562,7 +562,7 @@ public void importReleases(String organizationId, String owner, String inputDirS } OpenCGAResult userDataResult = getUserDBAdaptor(organizationId).get(owner, new QueryOptions(QueryOptions.INCLUDE, - Collections.singletonList(UserDBAdaptor.QueryParams.ACCOUNT.key()))); + Collections.singletonList(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT.key()))); if (userDataResult.getNumResults() == 0) { throw new CatalogException("User " + owner + " not found"); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index 4c9f03d12e1..8070d177233 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -36,10 +36,12 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupUpdateParams; @@ -60,8 +62,8 @@ */ public class UserManager extends AbstractManager { - static final QueryOptions INCLUDE_ACCOUNT_AND_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.ACCOUNT.key(), UserDBAdaptor.QueryParams.INTERNAL.key())); + static final QueryOptions INCLUDE_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.ID.key(), + UserDBAdaptor.QueryParams.INTERNAL.key(), UserDBAdaptor.QueryParams.DEPRECATED_ACCOUNT.key())); protected static Logger logger = LoggerFactory.getLogger(UserManager.class); private final CatalogIOManager catalogIOManager; private final AuthenticationFactory authenticationFactory; @@ -84,6 +86,9 @@ public void changePassword(String organizationId, String userId, String oldPassw if (oldPassword.equals(newPassword)) { throw new CatalogException("New password is the same as the old password."); } + if (!PasswordUtils.isStrongPassword(newPassword)) { + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); + } getUserDBAdaptor(organizationId).checkId(userId); String authOrigin = getAuthenticationOriginId(organizationId, userId); @@ -127,36 +132,54 @@ public OpenCGAResult create(User user, String password, String token) thro // Initialise fields ParamUtils.checkObj(user, "User"); ParamUtils.checkValidUserId(user.getId()); - ParamUtils.checkParameter(user.getName(), "name"); + user.setName(ParamUtils.defaultString(user.getName(), user.getId())); user.setEmail(ParamUtils.defaultString(user.getEmail(), "")); if (StringUtils.isNotEmpty(user.getEmail())) { checkEmail(user.getEmail()); } - user.setAccount(ParamUtils.defaultObject(user.getAccount(), Account::new)); - user.getAccount().setCreationDate(TimeUtils.getTime()); - if (StringUtils.isEmpty(user.getAccount().getExpirationDate())) { - // By default, user accounts will be valid for 1 year when they are created. - user.getAccount().setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); - Date date = TimeUtils.add1YeartoDate(new Date()); - user.getAccount().setExpirationDate(TimeUtils.getTime(date)); - } else { - // Validate expiration date is not over - ParamUtils.checkDateIsNotExpired(user.getAccount().getExpirationDate(), "account.expirationDate"); - } - user.setInternal(new UserInternal(new UserStatus(UserStatus.READY))); + user.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(user.getCreationDate(), + UserDBAdaptor.QueryParams.CREATION_DATE.key())); + user.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(user.getModificationDate(), + UserDBAdaptor.QueryParams.MODIFICATION_DATE.key())); + + user.setInternal(ParamUtils.defaultObject(user.getInternal(), UserInternal::new)); + user.getInternal().setStatus(new UserStatus(InternalStatus.READY)); user.setQuota(ParamUtils.defaultObject(user.getQuota(), UserQuota::new)); user.setProjects(ParamUtils.defaultObject(user.getProjects(), Collections::emptyList)); user.setConfigs(ParamUtils.defaultObject(user.getConfigs(), HashMap::new)); user.setFilters(ParamUtils.defaultObject(user.getFilters(), LinkedList::new)); user.setAttributes(ParamUtils.defaultObject(user.getAttributes(), Collections::emptyMap)); + // Init account + user.getInternal().setAccount(ParamUtils.defaultObject(user.getInternal().getAccount(), Account::new)); + Account account = user.getInternal().getAccount(); + account.setPassword(ParamUtils.defaultObject(account.getPassword(), Password::new)); + if (StringUtils.isEmpty(account.getExpirationDate())) { + account.setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); + } else { + // Validate expiration date is not over + ParamUtils.checkDateIsNotExpired(account.getExpirationDate(), UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key()); + } if (StringUtils.isEmpty(password)) { Map authOriginMap = authenticationFactory.getOrganizationAuthenticationManagers(organizationId); - if (!authOriginMap.containsKey(user.getAccount().getAuthentication().getId())) { - throw new CatalogException("Unknown authentication origin id '" + user.getAccount().getAuthentication() + "'"); + if (!authOriginMap.containsKey(account.getAuthentication().getId())) { + throw new CatalogException("Unknown authentication origin id '" + account.getAuthentication() + "'"); } } else { - user.getAccount().setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); + account.setAuthentication(new Account.AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, false)); + } + + // Set password expiration + if (AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId())) { + account.getPassword().setLastModified(TimeUtils.getTime()); + } + if (!AuthenticationOrigin.AuthenticationType.OPENCGA.name().equals(account.getAuthentication().getId()) + || configuration.getAccount().getPasswordExpirationDays() <= 0) { + // User password doesn't expire or it's not managed by OpenCGA + account.getPassword().setExpirationDate(null); + } else { + Date date = TimeUtils.addDaysToCurrentDate(configuration.getAccount().getPasswordExpirationDays()); + account.getPassword().setExpirationDate(TimeUtils.getTime(date)); } if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) || !OPENCGA.equals(user.getId())) { @@ -174,7 +197,7 @@ public OpenCGAResult create(User user, String password, String token) thro try { if (StringUtils.isNotEmpty(password) && !PasswordUtils.isStrongPassword(password)) { - throw new CatalogException("Invalid password. Check password strength for user " + user.getId()); + throw new CatalogException("Invalid password. " + PasswordUtils.PASSWORD_REQUIREMENT); } if (user.getProjects() != null && !user.getProjects().isEmpty()) { throw new CatalogException("Creating user and projects in a single transaction is forbidden"); @@ -210,7 +233,6 @@ public OpenCGAResult create(User user, String password, String token) thro public OpenCGAResult create(String id, String name, String email, String password, String organization, Long quota, String token) throws CatalogException { User user = new User(id, name, email, organization, new UserInternal(new UserStatus())) - .setAccount(new Account("", "", null)) .setQuota(new UserQuota().setMaxDisk(quota != null ? quota : -1)); return create(user, password, token); } @@ -241,7 +263,8 @@ public OpenCGAResult search(@Nullable String organizationId, Query query, // Fix query params if (query.containsKey(ParamConstants.USER_AUTHENTICATION_ORIGIN)) { - query.put(UserDBAdaptor.QueryParams.ACCOUNT_AUTHENTICATION_ID.key(), query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); + query.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_AUTHENTICATION_ID.key(), + query.get(ParamConstants.USER_AUTHENTICATION_ORIGIN)); query.remove(ParamConstants.USER_AUTHENTICATION_ORIGIN); } @@ -266,11 +289,12 @@ public JwtPayload validateToken(String token) throws CatalogException { authOrigin = CatalogAuthenticationManager.OPENCGA; } else { OpenCGAResult userResult = getUserDBAdaptor(jwtPayload.getOrganization()).get(jwtPayload.getUserId(), - INCLUDE_ACCOUNT_AND_INTERNAL); + INCLUDE_INTERNAL); if (userResult.getNumResults() == 0) { throw new CatalogException("User '" + jwtPayload.getUserId() + "' could not be found."); } - authOrigin = userResult.first().getAccount().getAuthentication().getId(); + User user = userResult.first(); + authOrigin = user.getInternal().getAccount().getAuthentication().getId(); } authenticationFactory.validateToken(jwtPayload.getOrganization(), authOrigin, token); @@ -475,8 +499,10 @@ public void importRemoteEntities(String organizationId, String authOrigin, List< } } else { for (String applicationId : idList) { - User application = new User(applicationId, new Account() - .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true))) + Account account = new Account() + .setAuthentication(new Account.AuthenticationOrigin(authOrigin, true)); + User application = new User(applicationId) + .setInternal(new UserInternal(new UserStatus(UserStatus.READY), account)) .setEmail("mail@mail.co.uk"); application.setOrganization(organizationId); create(application, null, token); @@ -782,7 +808,7 @@ public AuthenticationResponse login(String organizationId, String username, Stri if (OPENCGA.equals(username)) { organizationId = ParamConstants.ADMIN_ORGANIZATION; } else { - List organizationIds = catalogDBAdaptorFactory.getOrganizationIds(); + List organizationIds = getCatalogDBAdaptorFactory().getOrganizationIds(); if (organizationIds.size() == 2) { organizationId = organizationIds.stream().filter(s -> !ParamConstants.ADMIN_ORGANIZATION.equals(s)).findFirst().get(); } else { @@ -791,38 +817,56 @@ public AuthenticationResponse login(String organizationId, String username, Stri } } - OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_ACCOUNT_AND_INTERNAL); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { User user = userOpenCGAResult.first(); // Only local OPENCGA users that are not superadmins can be automatically banned or their accounts be expired boolean userCanBeBanned = !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) - && CatalogAuthenticationManager.OPENCGA.equals(user.getAccount().getAuthentication().getId()); + && CatalogAuthenticationManager.OPENCGA.equals(user.getInternal().getAccount().getAuthentication().getId()); // We check if (userCanBeBanned) { // Check user is not banned, suspended or has an expired account if (UserStatus.BANNED.equals(user.getInternal().getStatus().getId())) { throw CatalogAuthenticationException.userIsBanned(username); } - Date date = TimeUtils.toDate(user.getAccount().getExpirationDate()); - if (date == null) { - throw new CatalogException("Unexpected null expiration date for user '" + username + "'."); + if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsSuspended(username); } - if (date.before(new Date())) { - throw CatalogAuthenticationException.accountIsExpired(username, user.getAccount().getExpirationDate()); + Account account2 = user.getInternal().getAccount(); + if (account2.getPassword().getExpirationDate() != null) { + Account account1 = user.getInternal().getAccount(); + Date passwordExpirationDate = TimeUtils.toDate(account1.getPassword().getExpirationDate()); + if (passwordExpirationDate == null) { + throw new CatalogException("Unexpected null 'passwordExpirationDate' for user '" + username + "'."); + } + if (passwordExpirationDate.before(new Date())) { + Account account = user.getInternal().getAccount(); + throw CatalogAuthenticationException.passwordExpired(username, account.getPassword().getExpirationDate()); + } + } + if (user.getInternal().getAccount().getExpirationDate() != null) { + Date date = TimeUtils.toDate(user.getInternal().getAccount().getExpirationDate()); + if (date == null) { + throw new CatalogException("Unexpected null 'expirationDate' for user '" + username + "'."); + } + if (date.before(new Date())) { + throw CatalogAuthenticationException.accountIsExpired(username, + user.getInternal().getAccount().getExpirationDate()); + } } } - if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { - throw CatalogAuthenticationException.userIsSuspended(username); - } - authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); + User user1 = userOpenCGAResult.first(); + authId = user1.getInternal().getAccount().getAuthentication().getId(); try { response = authenticationFactory.authenticate(organizationId, authId, username, password); } catch (CatalogAuthenticationException e) { if (userCanBeBanned) { // We can only lock the account if it is not the root user - int failedAttempts = userOpenCGAResult.first().getInternal().getFailedAttempts(); - ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), failedAttempts + 1); - if (failedAttempts >= (configuration.getMaxLoginAttempts() - 1)) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + int failedAttempts = userInternal.getAccount().getFailedAttempts(); + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), + failedAttempts + 1); + if (failedAttempts >= (configuration.getAccount().getMaxLoginAttempts() - 1)) { // Ban the account updateParams.append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.BANNED); } @@ -835,10 +879,13 @@ public AuthenticationResponse login(String organizationId, String username, Stri } // If it was a local user and the counter of failed attempts was greater than 0, we reset it - if (userCanBeBanned && userOpenCGAResult.first().getInternal().getFailedAttempts() > 0) { - // Reset login failed attempts counter - ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); - getUserDBAdaptor(organizationId).update(username, updateParams); + if (userCanBeBanned) { + UserInternal userInternal = userOpenCGAResult.first().getInternal(); + if (userInternal.getAccount().getFailedAttempts() > 0) { + // Reset login failed attempts counter + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); + getUserDBAdaptor(organizationId).update(username, updateParams); + } } } else { // We attempt to login the user with the different authentication managers @@ -989,7 +1036,7 @@ public OpenCGAResult changeStatus(String organizationId, String userId, St // Update user status and reset failed attempts to 0 ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), status); if (UserStatus.READY.equals(status)) { - updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); + updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_FAILED_ATTEMPTS.key(), 0); } OpenCGAResult result = getUserDBAdaptor(userIdOrganization).update(userId, updateParams); @@ -1056,9 +1103,10 @@ public String getNonExpiringToken(String organizationId, String userId, Map userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_ACCOUNT_AND_INTERNAL); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { - String authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); + User user1 = userOpenCGAResult.first(); + String authId = user1.getInternal().getAccount().getAuthentication().getId(); return authenticationFactory.getOrganizationAuthenticationManager(organizationId, authId); } else { throw new CatalogException("User '" + user + "' not found."); @@ -1489,7 +1537,8 @@ private String getAuthenticationOriginId(String organizationId, String userId) t if (user == null || user.getNumResults() == 0) { throw new CatalogException(userId + " user not found"); } - return user.first().getAccount().getAuthentication().getId(); + User user1 = user.first(); + return user1.getInternal().getAccount().getAuthentication().getId(); } /** diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java index 4cae5cd565b..8882260fabc 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptorTest.java @@ -26,9 +26,8 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.IndividualPopulation; @@ -187,7 +186,7 @@ public void testModifyIndividualNegativeFatherId() throws Exception { } @Test - public void testAvoidDuplicatedSamples() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void testAvoidDuplicatedSamples() throws CatalogException { dbAdaptorFactory.getCatalogSampleDBAdaptor(organizationId).insert(studyUid, new Sample().setId("sample1").setInternal(SampleInternal.init()), Collections.emptyList(), QueryOptions.empty()); Sample sample1 = getSample(studyUid, "sample1"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java index 9c6d82a4c79..a382282764d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptorTest.java @@ -60,7 +60,7 @@ private Job getNewJob(String id) { } @Test - public void createJobTest() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void createJobTest() throws CatalogException { Job job = getNewJob("jobName1"); System.out.println(catalogJobDBAdaptor.insert(studyUid, job, null)); @@ -120,7 +120,7 @@ public void getJobTest() throws CatalogException { } @Test - public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void testSortResultsPriorityAndCreationDate() throws CatalogException { Date startDate = TimeUtils.getDate(); // Create 100 jobs @@ -176,7 +176,7 @@ public void testSortResultsPriorityAndCreationDate() throws CatalogDBException, // } @Test - public void getJobsOrderedByDate() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void getJobsOrderedByDate() throws CatalogException { // Job with current date Job job1 = getNewJob("job1"); @@ -250,7 +250,7 @@ public void updateInputAndOutputFiles() throws Exception { } @Test - public void groupByStatus() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void groupByStatus() throws CatalogException { for (int i = 0; i < 10; i++) { Enums.ExecutionStatus status = new Enums.ExecutionStatus(); if (i < 5) { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java index 5488bc6a545..fa908630690 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptorTest.java @@ -27,10 +27,8 @@ import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.managers.SampleManager; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.cohort.Cohort; @@ -175,7 +173,7 @@ public class SampleMongoDBAdaptorTest extends AbstractMongoDBAdaptorTest { // } @Test - public void searchByOntology() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void searchByOntology() throws CatalogException { List ontologyList = Arrays.asList( new Phenotype("hpo:123", "One hpo term", "hpo", Phenotype.Status.UNKNOWN), new Phenotype("hpo:456", "Another hpo term", "hpo", Phenotype.Status.UNKNOWN), @@ -387,7 +385,7 @@ public void caseInsensitiveSearchTest() throws Exception { // Test if we can search for samples of an individual @Test - public void getSampleWithIndividual() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void getSampleWithIndividual() throws CatalogException { QueryOptions queryOptions = new QueryOptions(); // We create a new sample with the individual diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index 697a77d894f..f81f28b37a6 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -20,10 +20,8 @@ import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -112,7 +110,7 @@ public void createVariableSetTest() throws CatalogDBException { } @Test - public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void testRemoveFieldFromVariableSet() throws CatalogException { DataResult variableSetDataResult = createExampleVariableSet("VARSET_1", false); DataResult result = catalogStudyDBAdaptor.removeFieldFromVariableSet(5L, variableSetDataResult.first().getUid(), "NAME", orgAdminUserId1); @@ -165,7 +163,7 @@ public void testRemoveFieldFromVariableSet() throws CatalogDBException, CatalogA * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void addFieldToVariableSetTest1() throws CatalogException { DataResult varset1 = createExampleVariableSet("VARSET_1", false); createExampleVariableSet("VARSET_2", true); Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, @@ -191,7 +189,7 @@ public void addFieldToVariableSetTest1() throws CatalogDBException, CatalogAutho * @throws CatalogDBException */ @Test - public void addFieldToVariableSetTest2() throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { + public void addFieldToVariableSetTest2() throws CatalogException { Variable variable = new Variable("NAM", "", Variable.VariableType.STRING, "", true, false, Collections.emptyList(), null, 0, "", "", null, Collections.emptyMap()); thrown.expect(CatalogDBException.class); @@ -208,7 +206,7 @@ public void createGroup() throws CatalogDBException { } @Test - public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void removeUsersFromAllGroups() throws CatalogException { catalogStudyDBAdaptor.createGroup(studyUid, new Group("name1", Arrays.asList(normalUserId1, normalUserId2))); catalogStudyDBAdaptor.createGroup(studyUid, new Group("name2", Arrays.asList(normalUserId1, normalUserId2, normalUserId3))); catalogStudyDBAdaptor.createGroup(studyUid, new Group("name3", Arrays.asList(normalUserId1, normalUserId3))); @@ -221,7 +219,7 @@ public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParamet } @Test - public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void resyncUserWithSyncedGroups() throws CatalogException { // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Arrays.asList(normalUserId1, normalUserId2, normalUserId3)); catalogStudyDBAdaptor.createGroup(studyUid, group); @@ -293,7 +291,7 @@ public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParam } @Test - public void updateUserToGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public void updateUserToGroups() throws CatalogException { // We create synced groups and not synced groups in study studyUid Group group = new Group("@notSyncedGroup", Collections.emptyList()); catalogStudyDBAdaptor.createGroup(studyUid, group); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java index a007c100b0f..8e4571d86ea 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptorTest.java @@ -100,7 +100,7 @@ public void getUserTest() throws CatalogDBException, CatalogParameterException, } @Test - public void changePasswordTest() throws CatalogDBException, CatalogAuthenticationException { + public void changePasswordTest() throws CatalogException { DataResult result = catalogUserDBAdaptor.changePassword(normalUserId1, TestParamConstants.PASSWORD, "1234"); assertEquals(1, result.getNumUpdated()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index 9044dd2f7c0..4bafb0cc4e0 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -23,13 +23,17 @@ import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.junit.rules.TestName; +import org.mockito.Mockito; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.test.GenericTest; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -474,4 +478,23 @@ private void createDummyData(CatalogManager catalogManager) throws CatalogExcept } + protected CatalogManager mockCatalogManager() throws CatalogDBException { + CatalogManager spy = Mockito.spy(catalogManager); + UserManager userManager = spy.getUserManager(); + UserManager userManagerSpy = Mockito.spy(userManager); + Mockito.doReturn(userManagerSpy).when(spy).getUserManager(); + MongoDBAdaptorFactory mongoDBAdaptorFactory = mockMongoDBAdaptorFactory(); + Mockito.doReturn(mongoDBAdaptorFactory).when(userManagerSpy).getCatalogDBAdaptorFactory(); + return spy; + } + + protected MongoDBAdaptorFactory mockMongoDBAdaptorFactory() throws CatalogDBException { + MongoDBAdaptorFactory catalogDBAdaptorFactory = (MongoDBAdaptorFactory) catalogManager.getUserManager().getCatalogDBAdaptorFactory(); + MongoDBAdaptorFactory dbAdaptorFactorySpy = Mockito.spy(catalogDBAdaptorFactory); + UserDBAdaptor userDBAdaptor = dbAdaptorFactorySpy.getCatalogUserDBAdaptor(organizationId); + UserDBAdaptor userDBAdaptorSpy = Mockito.spy(userDBAdaptor); + Mockito.doReturn(userDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogUserDBAdaptor(organizationId); + return dbAdaptorFactorySpy; + } + } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 73cab79045b..8f0bce1401e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -16,13 +16,10 @@ package org.opencb.opencga.catalog.managers; -import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.common.Status; @@ -34,7 +31,10 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.api.*; -import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -43,7 +43,6 @@ import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; -import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; @@ -61,11 +60,10 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import javax.naming.NamingException; import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutorService; @@ -73,22 +71,12 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; -import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; @Category(MediumTests.class) public class CatalogManagerTest extends AbstractManagerTest { - @Test - public void createOpencgaUserTest() throws CatalogException { - thrown.expect(CatalogException.class); - thrown.expectMessage("forbidden"); - catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) - .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - } - @Test public void createStudyFailMoreThanOneProject() throws CatalogException { catalogManager.getProjectManager().incrementRelease(project1, ownerToken); @@ -102,443 +90,6 @@ public void createStudyFailMoreThanOneProject() throws CatalogException { null, null, null, null, ownerToken); } - @Test - public void testAdminUserExists() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - JwtPayload payload = catalogManager.getUserManager().validateToken(token); - assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); - } - - @Test - public void searchUsersTest() throws CatalogException { - OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); - assertEquals(8, search.getNumResults()); - for (User user : search.getResults()) { - if (noAccessUserId1.equals(user.getId())) { - assertEquals(0, user.getProjects().size()); - } else if (user.getId().startsWith("normalUser")) { - assertEquals(1, user.getProjects().size()); - } else { - assertEquals(2, user.getProjects().size()); - } - } - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); - assertEquals(8, search.getNumResults()); - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); - assertEquals(8, search.getNumResults()); - - search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); - assertEquals(8, search.getNumResults()); - - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), - QueryOptions.empty(), studyAdminToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), - QueryOptions.empty(), normalToken1)); - } - - @Test - public void testGetToken() throws Exception { - String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); - Map claims = new HashMap<>(); - claims.put("a", "hola"); - claims.put("ab", "byw"); - // Create a token valid for 1 second - String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); - assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); - - String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); - assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); - - Thread.sleep(1000); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("expired"); - assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); - } - - @Test - public void loginWithoutOrganizationId() throws CatalogException { - String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - JwtPayload jwtPayload = new JwtPayload(token); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); - - token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - jwtPayload = new JwtPayload(token); - assertEquals(organizationId, jwtPayload.getOrganization()); - - // Create a third organization - catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); - token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertTrue(StringUtils.isNotEmpty(token)); - jwtPayload = new JwtPayload(token); - assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); - - thrown.expect(CatalogParameterException.class); - thrown.expectMessage("organization"); - catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); - } - - @Test - public void testCreateExistingUser() throws Exception { - thrown.expect(CatalogException.class); - thrown.expectMessage(containsString("already exists")); - catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, - null, opencgaToken); - } - - @Test - public void testCreateAnonymousUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, - organizationId, null, opencgaToken); - } - - @Test - public void testCreateRegisteredUser() throws Exception { - thrown.expect(CatalogParameterException.class); - thrown.expectMessage(containsString("reserved")); - catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, - opencgaToken); - } - - @Test - public void testLogin() throws Exception { - catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); - - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); - catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); - } - - @Test - public void refreshTokenTest() throws Exception { - String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); - AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); - assertNotNull(authenticationResponse); - assertNotNull(authenticationResponse.getToken()); - } - - @Test - public void anonymousUserLoginTest() throws CatalogException { - AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); - assertNotNull(authResponse.getToken()); - - String org2 = "otherOrg"; - catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); - catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); - String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); - Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() - .setId("project") - .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), - INCLUDE_RESULT, owner2Token).first(); - Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); - - try { - catalogManager.getUserManager().loginAnonymous(org2); - fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); - } catch (Exception e) { - assertEquals(CatalogAuthenticationException.class, e.getClass()); - assertTrue(e.getMessage().contains("not found")); - } - - catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("*")), owner2Token); - authResponse = catalogManager.getUserManager().loginAnonymous(org2); - assertNotNull(authResponse.getToken()); - - - catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, - new GroupUpdateParams(Collections.singletonList("*")), owner2Token); - thrown.expect(CatalogAuthenticationException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().loginAnonymous(org2); - } - - @Test - public void incrementLoginAttemptsTest() throws CatalogException { - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(1, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - - for (int i = 2; i < 5; i++) { - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(i, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - } - - assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(5, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); - - CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); - assertTrue(incorrect.getMessage().contains("banned")); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(5, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); - - CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); - assertTrue(authException.getMessage().contains("banned")); - - // Remove ban from user - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); - user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); - assertEquals(0, user.getInternal().getFailedAttempts()); - assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); - - String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); - assertNotNull(token); - } - - @Test - public void changeUserStatusTest() throws CatalogException { - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); - assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); - catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); - - catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); - CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); - assertTrue(authException.getMessage().contains("own account")); - - authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); - assertTrue(authException.getMessage().contains("suspend administrators")); - - CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); - assertTrue(incorrect.getMessage().contains("suspended")); - - catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); - String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); - assertNotNull(token); - - CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); - assertTrue(paramException.getMessage().contains("Invalid status")); - - CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); - assertTrue(dbException.getMessage().contains("not exist")); - } - - @Test - public void loginExpiredAccountTest() throws CatalogException { - // Expire account of normalUserId1 - ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); - catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); - - CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); - assertTrue(authException.getMessage().contains("expired")); - - // Ensure it doesn't matter whether opencga account is expired or not - catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); - String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); - assertNotNull(token); - } - - @Test - public void updateUserTest() throws JsonProcessingException, CatalogException { - UserUpdateParams userUpdateParams = new UserUpdateParams() - .setName("newName") - .setEmail("mail@mail.com"); - ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); - User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); - assertEquals(userUpdateParams.getName(), user.getName()); - assertEquals(userUpdateParams.getEmail(), user.getEmail()); - - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); - assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); - - userUpdateParams = new UserUpdateParams() - .setEmail("notAnEmail"); - ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); - assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); - } - - @Test - public void testGetUserInfo() throws CatalogException { - // OpenCGA administrator - DataResult user = catalogManager.getUserManager().get(organizationId, - Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - // Organization owner - user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), - new QueryOptions(), ownerToken); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - // Organization administrator - user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), - new QueryOptions(), orgAdminToken1); - assertEquals(3, user.getNumResults()); - assertEquals(normalUserId1, user.getResults().get(0).getId()); - assertEquals(normalUserId2, user.getResults().get(1).getId()); - assertEquals(normalUserId3, user.getResults().get(2).getId()); - - thrown.expect(CatalogAuthorizationException.class); - thrown.expectMessage("organization"); - catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), - studyAdminToken1); - } - - @Test - public void testGetProjectsFromUserInfo() throws CatalogException { - String userId = organizationId; - catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, - opencgaToken); - catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, - new GroupUpdateParams(Collections.singletonList("test")), ownerToken); - String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); - - DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - - user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - - user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); - assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); - System.out.println(user.first().getProjects().size()); - } - - @Test - public void testModifyUser() throws CatalogException, InterruptedException, IOException { - ObjectMap params = new ObjectMap(); - String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); - String newPassword = RandomStringUtils.randomAlphanumeric(10); - String newEmail = "new@email.ac.uk"; - - params.put("name", newName); - - Thread.sleep(10); - - catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); - catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); - - List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions - .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), - UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); - User userPost = userList.get(0); - System.out.println("userPost = " + userPost); - assertEquals(userPost.getName(), newName); - assertEquals(userPost.getEmail(), newEmail); - - catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD); - catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD); - - try { - params = new ObjectMap(); - params.put("password", "1234321"); - catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); - fail("Expected exception"); - } catch (CatalogDBException e) { - System.out.println(e); - } - - try { - catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); - fail("Expected exception"); - } catch (CatalogException e) { - System.out.println(e); - } - } - - @Test - public void testUpdateUserConfig() throws CatalogException { - Map map = new HashMap<>(); - map.put("key1", "value1"); - map.put("key2", "value2"); - catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); - - Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); - assertEquals(2, config.size()); - assertEquals("value1", config.get("key1")); - assertEquals("value2", config.get("key2")); - - map = new HashMap<>(); - map.put("key2", "value3"); - catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); - config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); - assertEquals(1, config.size()); - assertEquals("value3", config.get("key2")); - - catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); - - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); - } - - private String getAdminToken() throws CatalogException, IOException { - return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - } - - @Test - public void createUserUsingMailAsId() throws CatalogException { - catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); - AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); - assertNotNull(login); - User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); - assertEquals("hello.mail@mymail.org", user.getId()); - } - @Test public void getGroupsTest() throws CatalogException { Group group = new Group("groupId", Arrays.asList(normalUserId2, normalUserId3)).setSyncedFrom(new Group.Sync("ldap", "bio")); @@ -564,82 +115,6 @@ public void getGroupsTest() throws CatalogException { catalogManager.getStudyManager().getCustomGroups(studyFqn, group.getId(), normalToken2); } - @Ignore - @Test - public void importLdapUsers() throws CatalogException, NamingException, IOException { - // Action only for admins - catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, - getAdminToken()); - // TODO: Validate the users have been imported - } - - // To make this test work we will need to add a correct user and password to be able to login - @Ignore - @Test - public void loginNotRegisteredUsers() throws CatalogException { - // Action only for admins - Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); - catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); - catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), - ParamUtils.AclAction.SET, ownerToken); - String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); - - assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - - // We remove the permissions for group ldap - catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), - ParamUtils.AclAction.RESET, this.ownerToken); - - assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); - } - - @Ignore - @Test - public void syncUsers() throws CatalogException { - // Action only for admins - String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); - - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); - DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); - - assertEquals(1, bio.getNumResults()); - assertEquals(0, bio.first().getUserIds().size()); - - catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); - bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); - - assertEquals(1, bio.getNumResults()); - assertTrue(!bio.first().getUserIds().isEmpty()); - } - - @Ignore - @Test - public void importLdapGroups() throws CatalogException, IOException { - // Action only for admins - String remoteGroup = "bio"; - String internalGroup = "test"; - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); - - DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); - assertEquals(1, test.getNumResults()); - assertEquals("@test", test.first().getId()); - assertTrue(test.first().getUserIds().size() > 0); - -// internalGroup = "test1"; -// try { -// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); -// fail("Should not be possible creating another group containing the same users that belong to a different group"); -// } catch (CatalogException e) { -// System.out.println(e.getMessage()); -// } - - remoteGroup = "bioo"; - internalGroup = "test2"; - thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); - catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); - } - @Test public void createEmptyGroup() throws CatalogException { catalogManager.getUserManager().create("test", "test", "test@mail.com", TestParamConstants.PASSWORD, organizationId, 100L, opencgaToken); @@ -681,29 +156,6 @@ public void testAssignPermissions() throws CatalogException { assertTrue(acls.stream().map(x -> String.valueOf(x.get("member"))).collect(Collectors.toSet()).contains("@group_cancer_some_thing_else")); } - @Test - public void getUserInfoTest() throws CatalogException { - OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(2, result.first().getProjects().size()); - - result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); - assertEquals(1, result.getNumResults()); - assertNotNull(result.first().getProjects()); - assertEquals(1, result.first().getProjects().size()); - } - /** * Project methods *************************** */ diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index 540602ede36..6d42dbc6fbd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -253,8 +253,8 @@ public void createMultipleCasesSameFamily() throws CatalogException { public void updateClinicalAnalystsTest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("u2").setName("u2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); // Add analysts OpenCGAResult result = catalogManager.getClinicalAnalysisManager().update(studyFqn, case1.getId(), @@ -306,7 +306,7 @@ public void updateClinicalAnalysisRequest() throws CatalogException { ClinicalAnalysis case1 = createDummyEnvironment(true, true).first(); assertTrue(StringUtils.isEmpty(case1.getRequest().getId())); - catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com").setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setOrganization(organizationId).setEmail("mail@mail.com"), TestParamConstants.PASSWORD, opencgaToken); ClinicalRequest request = new ClinicalRequest("requestId", "bla", null, new ClinicalResponsible().setId("u1"), new HashMap<>()); @@ -344,8 +344,8 @@ public void updateClinicalAnalysisResponsible() throws CatalogException { assertEquals(orgOwnerUserId, case1.getResponsible().getId()); catalogManager.getUserManager().create(new User().setId("u1").setName("u1").setEmail("mail@mail.com") - .setOrganization(organizationId) - .setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); + .setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); ClinicalResponsible responsible = new ClinicalResponsible().setId("u1"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java index ca9323b422d..a72d52eace3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java @@ -287,7 +287,7 @@ private void updateAndAssertChanges(String orgId, OrganizationUserUpdateParams u User user = catalogManager.getOrganizationManager().updateUser(orgId, normalUserId1, userUpdateParams, INCLUDE_RESULT, token).first(); assertEquals(userUpdateParams.getName(), user.getName()); assertEquals(userUpdateParams.getEmail(), user.getEmail()); - assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getAccount().getExpirationDate()); + assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getInternal().getAccount().getExpirationDate()); assertEquals(userUpdateParams.getQuota().getCpuUsage(), user.getQuota().getCpuUsage()); assertEquals(userUpdateParams.getQuota().getDiskUsage(), user.getQuota().getDiskUsage()); assertEquals(userUpdateParams.getQuota().getMaxCpu(), user.getQuota().getMaxCpu()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java index 3a0a4e4a808..85e13914da5 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java @@ -38,7 +38,6 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.study.GroupUpdateParams; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -89,7 +88,7 @@ public void searchSampleNoPermissions() throws CatalogException { public void searchProjects() throws CatalogException { String org2 = "otherOrg"; catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); - catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setAccount(new Account()).setOrganization(org2), + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java new file mode 100644 index 00000000000..f23295fbd60 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java @@ -0,0 +1,664 @@ +package org.opencb.opencga.catalog.managers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; +import org.opencb.commons.datastore.core.DataResult; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.PasswordUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.JwtPayload; +import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; +import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.GroupUpdateParams; +import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.study.StudyAclParams; +import org.opencb.opencga.core.models.user.*; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.MediumTests; + +import javax.naming.NamingException; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +@Category(MediumTests.class) +public class UserManagerTest extends AbstractManagerTest { + + @Test + public void createOpencgaUserTest() throws CatalogException { + thrown.expect(CatalogException.class); + thrown.expectMessage("forbidden"); + catalogManager.getUserManager().create(new User().setId(ParamConstants.OPENCGA_USER_ID).setName(orgOwnerUserId) + .setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); + } + + + @Test + public void testAdminUserExists() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + JwtPayload payload = catalogManager.getUserManager().validateToken(token); + assertEquals(ParamConstants.OPENCGA_USER_ID, payload.getUserId()); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, payload.getOrganization()); + } + + @Test + public void searchUsersTest() throws CatalogException { + OpenCGAResult search = catalogManager.getUserManager().search(organizationId, new Query(), QueryOptions.empty(), opencgaToken); + assertEquals(8, search.getNumResults()); + for (User user : search.getResults()) { + if (noAccessUserId1.equals(user.getId())) { + assertEquals(0, user.getProjects().size()); + } else if (user.getId().startsWith("normalUser")) { + assertEquals(1, user.getProjects().size()); + } else { + assertEquals(2, user.getProjects().size()); + } + } + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), ownerToken); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken2); + assertEquals(8, search.getNumResults()); + + search = catalogManager.getUserManager().search(null, new Query(), QueryOptions.empty(), orgAdminToken1); + assertEquals(8, search.getNumResults()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().search(null, new Query(), + QueryOptions.empty(), normalToken1)); + } + + @Test + public void testGetToken() throws Exception { + String token = catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken(); + Map claims = new HashMap<>(); + claims.put("a", "hola"); + claims.put("ab", "byw"); + // Create a token valid for 1 second + String expiringToken = catalogManager.getUserManager().getToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, 1L, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + + String nonExpiringToken = catalogManager.getUserManager().getNonExpiringToken(ParamConstants.ADMIN_ORGANIZATION, "opencga", claims, token); + assertEquals("opencga", catalogManager.getUserManager().validateToken(nonExpiringToken).getUserId()); + + Thread.sleep(1000); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("expired"); + assertEquals("opencga", catalogManager.getUserManager().validateToken(expiringToken).getUserId()); + } + + @Test + public void loginWithoutOrganizationId() throws CatalogException { + String token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + JwtPayload jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + token = catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(organizationId, jwtPayload.getOrganization()); + + // Create a third organization + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId("other").setName("Test"), QueryOptions.empty(), opencgaToken); + token = catalogManager.getUserManager().login(null, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertTrue(StringUtils.isNotEmpty(token)); + jwtPayload = new JwtPayload(token); + assertEquals(ParamConstants.ADMIN_ORGANIZATION, jwtPayload.getOrganization()); + + thrown.expect(CatalogParameterException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().login(null, orgOwnerUserId, TestParamConstants.PASSWORD); + } + + @Test + public void testCreateExistingUser() throws Exception { + thrown.expect(CatalogException.class); + thrown.expectMessage(containsString("already exists")); + catalogManager.getUserManager().create(orgOwnerUserId, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, + null, opencgaToken); + } + + @Test + public void testCreateAnonymousUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.ANONYMOUS_USER_ID, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, + organizationId, null, opencgaToken); + } + + @Test + public void testCreateRegisteredUser() throws Exception { + thrown.expect(CatalogParameterException.class); + thrown.expectMessage(containsString("reserved")); + catalogManager.getUserManager().create(ParamConstants.REGISTERED_USERS, "User Name", "mail@ebi.ac.uk", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + } + + @Test + public void testLogin() throws Exception { + catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD); + + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage(allOf(containsString("Incorrect"), containsString("password"))); + catalogManager.getUserManager().login(organizationId, normalUserId1, "fakePassword"); + } + + @Test + public void refreshTokenTest() throws Exception { + String refreshToken = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getRefreshToken(); + AuthenticationResponse authenticationResponse = catalogManager.getUserManager().refreshToken(refreshToken); + assertNotNull(authenticationResponse); + assertNotNull(authenticationResponse.getToken()); + } + + @Test + public void anonymousUserLoginTest() throws CatalogException { + AuthenticationResponse authResponse = catalogManager.getUserManager().loginAnonymous(organizationId); + assertNotNull(authResponse.getToken()); + + String org2 = "otherOrg"; + catalogManager.getOrganizationManager().create(new OrganizationCreateParams().setId(org2), QueryOptions.empty(), opencgaToken); + catalogManager.getUserManager().create(new User().setId("userFromOrg2").setName("name").setOrganization(org2), TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getOrganizationManager().update(org2, new OrganizationUpdateParams().setOwner("userFromOrg2"), null, opencgaToken); + String owner2Token = catalogManager.getUserManager().login(org2, "userFromOrg2", TestParamConstants.PASSWORD).getToken(); + Project p = catalogManager.getProjectManager().create(new ProjectCreateParams() + .setId("project") + .setOrganism(new ProjectOrganism("Homo sapiens", "GRCh38")), + INCLUDE_RESULT, owner2Token).first(); + Study study = catalogManager.getStudyManager().create(p.getFqn(), new Study().setId("study"), INCLUDE_RESULT, owner2Token).first(); + + try { + catalogManager.getUserManager().loginAnonymous(org2); + fail("Anonymous user should not get a token for that organization as it has not been granted any kind of access"); + } catch (Exception e) { + assertEquals(CatalogAuthenticationException.class, e.getClass()); + assertTrue(e.getMessage().contains("not found")); + } + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + authResponse = catalogManager.getUserManager().loginAnonymous(org2); + assertNotNull(authResponse.getToken()); + + + catalogManager.getStudyManager().updateGroup(study.getFqn(), ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList("*")), owner2Token); + thrown.expect(CatalogAuthenticationException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().loginAnonymous(org2); + } + + @Test + public void incrementLoginAttemptsTest() throws CatalogException { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal3 = user.getInternal(); + assertEquals(1, userInternal3.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + for (int i = 2; i < 5; i++) { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(i, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + } + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal2 = user.getInternal(); + assertEquals(5, userInternal2.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + assertTrue(incorrect.getMessage().contains("banned")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal1 = user.getInternal(); + assertEquals(5, userInternal1.getAccount().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("banned")); + + // Remove ban from user + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + UserInternal userInternal = user.getInternal(); + assertEquals(0, userInternal.getAccount().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void changeUserStatusTest() throws CatalogException { + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); + assertTrue(authException.getMessage().contains("own account")); + + authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(authException.getMessage().contains("suspend administrators")); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); + assertTrue(incorrect.getMessage().contains("suspended")); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); + String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + + CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); + assertTrue(paramException.getMessage().contains("Invalid status")); + + CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(dbException.getMessage().contains("not exist")); + } + + @Test + public void loginExpiredAccountTest() throws CatalogException { + // Expire account of normalUserId1 + ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); + catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("expired")); + + // Ensure it doesn't matter whether opencga account is expired or not + catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); + String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void updateUserTest() throws JsonProcessingException, CatalogException { + UserUpdateParams userUpdateParams = new UserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com"); + ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); + + userUpdateParams = new UserUpdateParams() + .setEmail("notAnEmail"); + ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); + } + + @Test + public void testGetUserInfo() throws CatalogException { + // OpenCGA administrator + DataResult user = catalogManager.getUserManager().get(organizationId, + Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), opencgaToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization owner + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), ownerToken); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + // Organization administrator + user = catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), + new QueryOptions(), orgAdminToken1); + assertEquals(3, user.getNumResults()); + assertEquals(normalUserId1, user.getResults().get(0).getId()); + assertEquals(normalUserId2, user.getResults().get(1).getId()); + assertEquals(normalUserId3, user.getResults().get(2).getId()); + + thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("organization"); + catalogManager.getUserManager().get(organizationId, Arrays.asList(normalUserId1, normalUserId2, normalUserId3), new QueryOptions(), + studyAdminToken1); + } + + @Test + public void testGetProjectsFromUserInfo() throws CatalogException { + String userId = organizationId; + catalogManager.getUserManager().create(userId, "test", "mail@mail.com", TestParamConstants.PASSWORD, organizationId, null, + opencgaToken); + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.MEMBERS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList("test")), ownerToken); + String token = catalogManager.getUserManager().login(organizationId, userId, TestParamConstants.PASSWORD).getToken(); + + DataResult user = catalogManager.getUserManager().get(organizationId, userId, new QueryOptions(), token); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), normalToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId3, new QueryOptions(), normalToken3); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgOwnerUserId, new QueryOptions(), ownerToken); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, orgAdminUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, studyAdminUserId1, new QueryOptions(), studyAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + + user = catalogManager.getUserManager().get(null, normalUserId1, new QueryOptions(), orgAdminToken1); + assertTrue(CollectionUtils.isNotEmpty(user.first().getProjects())); + System.out.println(user.first().getProjects().size()); + } + + @Test + public void testModifyUser() throws CatalogException, InterruptedException, IOException { + ObjectMap params = new ObjectMap(); + String newName = "Changed Name " + RandomStringUtils.randomAlphanumeric(10); + String newPassword = PasswordUtils.getStrongRandomPassword(); + String newEmail = "new@email.ac.uk"; + + params.put("name", newName); + + Thread.sleep(10); + + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + catalogManager.getUserManager().update(orgOwnerUserId, new ObjectMap("email", newEmail), null, ownerToken); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD, newPassword); + + List userList = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, new QueryOptions(QueryOptions + .INCLUDE, Arrays.asList(UserDBAdaptor.QueryParams.NAME.key(), UserDBAdaptor.QueryParams.EMAIL.key(), + UserDBAdaptor.QueryParams.ATTRIBUTES.key())), ownerToken).getResults(); + User userPost = userList.get(0); + System.out.println("userPost = " + userPost); + assertEquals(userPost.getName(), newName); + assertEquals(userPost.getEmail(), newEmail); + + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("The new password has already been used")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, orgOwnerUserId, anotherPassword); + + try { + params = new ObjectMap(); + params.put("password", "1234321"); + catalogManager.getUserManager().update(orgOwnerUserId, params, null, ownerToken); + fail("Expected exception"); + } catch (CatalogDBException e) { + System.out.println(e); + } + + try { + catalogManager.getUserManager().update(orgOwnerUserId, params, null, orgAdminToken1); + fail("Expected exception"); + } catch (CatalogException e) { + System.out.println(e); + } + } + + @Test + public void automaticPasswordExpirationTest() throws CatalogException { + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(1); + + String oneDay = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(1)); + String twoDays = TimeUtils.getTime(TimeUtils.addDaysToCurrentDate(2)); + + User user = new User().setId("tempUser"); + String password = PasswordUtils.getStrongRandomPassword(); + User storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account2 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(oneDay) <= Long.parseLong(account2.getPassword().getExpirationDate())); + Account account1 = storedUser.getInternal().getAccount(); + assertTrue(Long.parseLong(twoDays) > Long.parseLong(account1.getPassword().getExpirationDate())); + + // Set 1 day of password expiration + catalogManager.getConfiguration().getAccount().setPasswordExpirationDays(-5); + user = new User().setId("tempUser2"); + storedUser = catalogManager.getUserManager().create(user, password, ownerToken).first(); + Account account = storedUser.getInternal().getAccount(); + assertNull(account.getPassword().getExpirationDate()); + } + + @Test + public void loginUserPasswordExpiredTest() throws CatalogException { + try (CatalogManager mockCatalogManager = mockCatalogManager()) { + UserDBAdaptor userDBAdaptor = mockCatalogManager.getUserManager().getUserDBAdaptor(organizationId); + + OpenCGAResult result = mockCatalogManager.getUserManager().get(organizationId, normalUserId1, new QueryOptions(), normalToken1); + + // Set password expired 2 days ago + Date date = TimeUtils.addDaysToCurrentDate(-2); + String beforeYesterday = TimeUtils.getTime(date); + User user = result.first(); + user.getInternal().getAccount().getPassword().setExpirationDate(beforeYesterday); + + Mockito.doReturn(result).when(userDBAdaptor).get(normalUserId1, UserManager.INCLUDE_INTERNAL); + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> mockCatalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(exception.getMessage().contains("expired on " + beforeYesterday)); + } + } + + @Test + public void changePasswordTest() throws CatalogException { + String newPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, newPassword); + + CatalogAuthenticationException exception = assertThrows(CatalogAuthenticationException.class, + () -> catalogManager.getUserManager().changePassword(organizationId, normalUserId1, TestParamConstants.PASSWORD, newPassword)); + assertTrue(exception.getMessage().contains("verify that the current password is correct")); + + String anotherPassword = PasswordUtils.getStrongRandomPassword(); + catalogManager.getUserManager().changePassword(organizationId, normalUserId1, newPassword, anotherPassword); + catalogManager.getUserManager().login(organizationId, normalUserId1, anotherPassword); + } + + @Test + public void testUpdateUserConfig() throws CatalogException { + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", "value2"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + + Map config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(2, config.size()); + assertEquals("value1", config.get("key1")); + assertEquals("value2", config.get("key2")); + + map = new HashMap<>(); + map.put("key2", "value3"); + catalogManager.getUserManager().setConfig(normalUserId1, "a", map, normalToken1); + config = (Map) catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1).first(); + assertEquals(1, config.size()); + assertEquals("value3", config.get("key2")); + + catalogManager.getUserManager().deleteConfig(normalUserId1, "a", normalToken1); + + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().getConfig(normalUserId1, "a", normalToken1); + } + + private String getAdminToken() throws CatalogException, IOException { + return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + } + + @Test + public void createUserUsingMailAsId() throws CatalogException { + catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); + AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); + assertNotNull(login); + User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); + assertEquals("hello.mail@mymail.org", user.getId()); + } + + @Test + public void getUserInfoTest() throws CatalogException { + OpenCGAResult result = catalogManager.getUserManager().get(organizationId, orgOwnerUserId, QueryOptions.empty(), ownerToken); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, orgAdminUserId1, QueryOptions.empty(), orgAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, studyAdminUserId1, QueryOptions.empty(), studyAdminToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(2, result.first().getProjects().size()); + + result = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), normalToken1); + assertEquals(1, result.getNumResults()); + assertNotNull(result.first().getProjects()); + assertEquals(1, result.first().getProjects().size()); + } + + @Ignore + @Test + public void importLdapUsers() throws CatalogException, NamingException, IOException { + // Action only for admins + catalogManager.getUserManager().importRemoteEntities(organizationId, "ldap", Arrays.asList("pfurio", "imedina"), false, null, null, + getAdminToken()); + // TODO: Validate the users have been imported + } + + // To make this test work we will need to add a correct user and password to be able to login + @Ignore + @Test + public void loginNotRegisteredUsers() throws CatalogException { + // Action only for admins + Group group = new Group("ldap", Collections.emptyList()).setSyncedFrom(new Group.Sync("ldap", "bio")); + catalogManager.getStudyManager().createGroup(studyFqn, group, ownerToken); + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", "view_only"), + ParamUtils.AclAction.SET, ownerToken); + String token = catalogManager.getUserManager().login(organizationId, orgOwnerUserId, "password").getToken(); + + assertEquals(9, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + + // We remove the permissions for group ldap + catalogManager.getStudyManager().updateAcl(studyFqn, "@ldap", new StudyAclParams("", ""), + ParamUtils.AclAction.RESET, this.ownerToken); + + assertEquals(0, catalogManager.getSampleManager().count(studyFqn, new Query(), token).getNumTotalResults()); + } + + @Ignore + @Test + public void syncUsers() throws CatalogException { + // Action only for admins + String token = catalogManager.getUserManager().loginAsAdmin("admin").getToken(); + + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", "bio", "bio", studyFqn, true, token); + DataResult bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertEquals(0, bio.first().getUserIds().size()); + + catalogManager.getUserManager().syncAllUsersOfExternalGroup(organizationId, studyFqn, "ldap", token); + bio = catalogManager.getStudyManager().getGroup(studyFqn, "bio", this.ownerToken); + + assertEquals(1, bio.getNumResults()); + assertTrue(!bio.first().getUserIds().isEmpty()); + } + + @Ignore + @Test + public void importLdapGroups() throws CatalogException, IOException { + // Action only for admins + String remoteGroup = "bio"; + String internalGroup = "test"; + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + + DataResult test = catalogManager.getStudyManager().getGroup(studyFqn, "test", ownerToken); + assertEquals(1, test.getNumResults()); + assertEquals("@test", test.first().getId()); + assertTrue(test.first().getUserIds().size() > 0); + +// internalGroup = "test1"; +// try { +// catalogManager.getUserManager().importRemoteGroupOfUsers("ldap", remoteGroup, internalGroup, study, getAdminToken()); +// fail("Should not be possible creating another group containing the same users that belong to a different group"); +// } catch (CatalogException e) { +// System.out.println(e.getMessage()); +// } + + remoteGroup = "bioo"; + internalGroup = "test2"; + thrown.expect(CatalogException.class); + thrown.expectMessage("not found"); + catalogManager.getUserManager().importRemoteGroupOfUsers(organizationId, "ldap", remoteGroup, internalGroup, studyFqn, true, getAdminToken()); + } + + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java index f7b28721987..ca96f875326 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/templates/TemplateManagerTest.java @@ -11,7 +11,6 @@ import org.opencb.opencga.catalog.templates.TemplateManager; import org.opencb.opencga.catalog.templates.config.TemplateManifest; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -25,11 +24,14 @@ public class TemplateManagerTest extends AbstractManagerTest { public void test() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), + TestParamConstants.PASSWORD, opencgaToken); + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId).setAccount(new Account()), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); catalogManager.getStudyManager().create("project", new Study().setId("study"), QueryOptions.empty(), ownerToken); @@ -54,13 +56,13 @@ public void test() throws Exception { public void test_yaml() throws Exception { CatalogManager catalogManager = catalogManagerResource.getCatalogManager(); - catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user1").setName("User 1").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user2").setName("User 2").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user3").setName("User 3").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); - catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId).setAccount(new Account()), + catalogManager.getUserManager().create(new User().setId("user4").setName("User 4").setOrganization(organizationId), TestParamConstants.PASSWORD, opencgaToken); catalogManager.getProjectManager().create("project", "Project", "", "hsapiens", "common", "GRCh38", QueryOptions.empty(), ownerToken); diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 2d7cbe0861a..d0ebcf32ba6 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -17,6 +17,25 @@ analysis: packages: # List of packages where to find analysis tools - "org.opencb.opencga" scratchDir: "/tmp/" # Scratch folder for the analysis. + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + - id: "exomiser" + version: "13.1" + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + - id: "exomiser" + version: "14.0" + defaultVersion: true + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 337020eef82..a45c200d12e 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -225,7 +225,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param jobPriority Priority of the job. #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. - #' @param data Exomizer interpretation analysis params. + #' @param data Exomiser interpretation analysis params. runInterpreterExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/exomiser", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index 3eff20fea86..7e94ff82723 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -296,7 +296,7 @@ public RestResponse runInterpreterCancerTiering(CancerTieringInterpretation /** * Run exomiser interpretation analysis. - * @param data Exomizer interpretation analysis params. + * @param data Exomiser interpretation analysis params. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index d7ad3ed9f8f..931d5fe01ce 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -223,7 +223,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { } /** Run exomiser interpretation analysis - * @param {Object} data - Exomizer interpretation analysis params. + * @param {Object} data - Exomiser interpretation analysis params. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index 1ba7704138c..0a531515217 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -286,7 +286,7 @@ def run_interpreter_exomiser(self, data=None, **options): Run exomiser interpretation analysis. PATH: /{apiVersion}/analysis/clinical/interpreter/exomiser/run - :param dict data: Exomizer interpretation analysis params. (REQUIRED) + :param dict data: Exomiser interpretation analysis params. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 835f89a61a6..dbde894f78f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -45,7 +45,7 @@ public class FieldConstants { public static final String ORGANIZATION_ADMINS_DESCRIPTION = "Administrative users of the organization."; public static final String ORGANIZATION_PROJECTS_DESCRIPTION = "Projects the organization holds."; public static final String ORGANIZATION_NOTES_DESCRIPTION = "Notes of organization scope."; -// public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " + // public static final String ORGANIZATION_AUTHENTICATION_ORIGINS_DESCRIPTION = "Authentication origins used by the organization. This " // + "contains all the configuration necessary to be able to communicate with the external authentication origins."; public static final String ORGANIZATION_CONFIGURATION_DESCRIPTION = "Organization configuration information."; public static final String ORGANIZATION_INTERNAL_DESCRIPTION = "Organization internal information."; @@ -441,9 +441,16 @@ public class FieldConstants { public static final String USER_ACCOUNT = "User account."; //Account - public static final String ACCOUNT_TYPE = "User account type can have the values GUEST, FULL and ADMINISTRATOR."; - public static final String ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; - public static final String ACCOUNT_AUTHENTICATION = "How the account is authenticated"; + public static final String INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION = "Date the account expires."; + public static final String INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION = "Number of consecutive failed attempts. When the user logs" + + " in successfully, this field is automatically reset to 0."; + public static final String INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION = "Object containing the last time the password was changed and the" + + " expiration date."; + public static final String INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION = "Date the user's password was last changed. This " + + "field will be null if the user account origin is different from OpenCGA."; + public static final String INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION = "Date the user's password expires. This field will" + + " be null if the user account origin is different from OpenCGA and the passwordExpiration feature is disabled."; + public static final String INTERNAL_ACCOUNT_AUTHENTICATION = "How the account is authenticated"; public static final String USER_QUOTA = "User quota"; public static final String USER_PROJECTS = "A List with related projects."; public static final String USER_SHARED_PROJECTS = "A List with shared projects."; @@ -504,7 +511,7 @@ public class FieldConstants { public static final String HRDETECT_CNV_QUERY_DESCRIPTION = "CNV query"; public static final String HRDETECT_INDEL_QUERY_DESCRIPTION = "INDEL query"; public static final String HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV3 input for" - + " HRDetect."; + + " HRDetect."; public static final String HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV8 input for" + " HRDetect."; public static final String HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SV3 input for" @@ -541,5 +548,10 @@ public class FieldConstants { public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; // Exomiser + public static final String EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION = "Clinical analysis ID."; + public static final String EXOMISER_SAMPLE_DESCRIPTION = "Sample ID."; public static final String EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION = "Clinical analysis type: SINGLE or FAMILY."; + public static final String EXOMISER_VERSION_DESCRIPTION = "Exomiser version in the format X.Y where X is the major version and Y the" + + " minor version, e.g.: 14.0. If the version is not specified, the default version will be used. Refer to the configuration" + + " file to view all installed Exomiser versions and identify the default version."; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java index 17255ae3255..91fb20b8455 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/PasswordUtils.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.common; +import org.passay.CharacterData; import org.passay.*; import java.util.ArrayList; @@ -7,6 +8,12 @@ public class PasswordUtils { + public static final int MIN_STRONG_PASSWORD_LENGTH = 8; + public static final int DEFAULT_PASSWORD_LENGTH = 10; + public static final int DEFAULT_SALT_LENGTH = 32; + public static final String PASSWORD_REQUIREMENT = "Password must contain at least " + MIN_STRONG_PASSWORD_LENGTH + + " characters, including at least one uppercase letter, one lowercase letter, one digit and one special character."; + private final static CharacterRule SPECIAL_CHARACTER_RULE = new CharacterRule(new CharacterData() { @Override public String getErrorCode() { @@ -20,7 +27,11 @@ public String getCharacters() { }); public static String getStrongRandomPassword() { - return getStrongRandomPassword(10); + return getStrongRandomPassword(DEFAULT_PASSWORD_LENGTH); + } + + public static String getStrongRandomSalt() { + return getStrongRandomPassword(DEFAULT_SALT_LENGTH); } public static String getStrongRandomPassword(int length) { @@ -33,14 +44,10 @@ public static String getStrongRandomPassword(int length) { } public static boolean isStrongPassword(String password) { - return isStrongPassword(password, 8); - } - - public static boolean isStrongPassword(String password, int minLength) { List rules = new ArrayList<>(); //Rule 1: Password length should be in between - //minLength and 100 characters - rules.add(new LengthRule(minLength, 100)); + //MIN_STRONG_PASSWORD_LENGTH and 100 characters + rules.add(new LengthRule(MIN_STRONG_PASSWORD_LENGTH, 100)); //Rule 2: No whitespace allowed rules.add(new WhitespaceRule()); //Rule 3.a: At least one Upper-case character diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java index da6aa39f1f6..c0574f0afdf 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java @@ -135,6 +135,13 @@ public static Date add1YeartoDate(Date date) { return new Date(cal.getTimeInMillis()); } + public static Date addDaysToCurrentDate(int days) { + Calendar cal = new GregorianCalendar(); + cal.setTime(new Date()); + cal.add(Calendar.DAY_OF_YEAR, days); + return new Date(cal.getTimeInMillis()); + } + public static Date toDate(String dateStr) { Date date = null; try { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java new file mode 100644 index 00000000000..f467a93d37d --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AccountConfiguration.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.core.config; + +public class AccountConfiguration { + + private int maxLoginAttempts; + private int passwordExpirationDays; + + public AccountConfiguration() { + } + + public AccountConfiguration(int maxLoginAttempts, int passwordExpirationDays) { + this.maxLoginAttempts = maxLoginAttempts; + this.passwordExpirationDays = passwordExpirationDays; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AccountConfiguration{"); + sb.append("maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", passwordExpirationDays=").append(passwordExpirationDays); + sb.append('}'); + return sb.toString(); + } + + public int getMaxLoginAttempts() { + return maxLoginAttempts; + } + + public AccountConfiguration setMaxLoginAttempts(int maxLoginAttempts) { + this.maxLoginAttempts = maxLoginAttempts; + return this; + } + + public int getPasswordExpirationDays() { + return passwordExpirationDays; + } + + public AccountConfiguration setPasswordExpirationDays(int passwordExpirationDays) { + this.passwordExpirationDays = passwordExpirationDays; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java index 6eb2beecb49..9f1c7b32683 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Analysis.java @@ -17,13 +17,19 @@ package org.opencb.opencga.core.config; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class Analysis { private List packages; private String scratchDir; + private String resourceUrl; + + private String opencgaExtTools; + private List tools; private Execution execution; @@ -31,6 +37,7 @@ public class Analysis { public Analysis() { packages = new ArrayList<>(); + tools = new ArrayList<>(); execution = new Execution(); frameworks = new ArrayList<>(); } @@ -53,6 +60,33 @@ public Analysis setScratchDir(String scratchDir) { return this; } + public String getResourceUrl() { + return resourceUrl; + } + + public Analysis setResourceUrl(String resourceUrl) { + this.resourceUrl = resourceUrl; + return this; + } + + public String getOpencgaExtTools() { + return opencgaExtTools; + } + + public Analysis setOpencgaExtTools(String opencgaExtTools) { + this.opencgaExtTools = opencgaExtTools; + return this; + } + + public List getTools() { + return tools; + } + + public Analysis setTools(List tools) { + this.tools = tools; + return this; + } + public Execution getExecution() { return execution; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java new file mode 100644 index 00000000000..3744915b6b9 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AnalysisTool.java @@ -0,0 +1,110 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.config; + +import java.util.HashMap; +import java.util.Map; + +public class AnalysisTool { + + private String id; + private String version; + private boolean defaultVersion; + private String dockerId; + private String params; + private Map resources; + + public AnalysisTool() { + resources = new HashMap<>(); + } + + public AnalysisTool(String id, String version, boolean defaultVersion, String dockerId, String params, Map resources) { + this.id = id; + this.version = version; + this.defaultVersion = defaultVersion; + this.dockerId = dockerId; + this.params = params; + this.resources = resources; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("AnalysisTool{"); + sb.append("id='").append(id).append('\''); + sb.append(", version='").append(version).append('\''); + sb.append(", defaultVersion=").append(defaultVersion); + sb.append(", dockerId='").append(dockerId).append('\''); + sb.append(", params='").append(params).append('\''); + sb.append(", resources=").append(resources); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public AnalysisTool setId(String id) { + this.id = id; + return this; + } + + public String getVersion() { + return version; + } + + public AnalysisTool setVersion(String version) { + this.version = version; + return this; + } + + public boolean isDefaultVersion() { + return defaultVersion; + } + + public AnalysisTool setDefaultVersion(boolean defaultVersion) { + this.defaultVersion = defaultVersion; + return this; + } + + public String getDockerId() { + return dockerId; + } + + public AnalysisTool setDockerId(String dockerId) { + this.dockerId = dockerId; + return this; + } + + public String getParams() { + return params; + } + + public AnalysisTool setParams(String params) { + this.params = params; + return this; + } + + public Map getResources() { + return resources; + } + + public AnalysisTool setResources(Map resources) { + this.resources = resources; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index 600def109ff..b7b869c728c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -44,7 +44,7 @@ public class Configuration { private String workspace; private String jobDir; - private int maxLoginAttempts; + private AccountConfiguration account; private Monitor monitor; private HealthCheck healthCheck; @@ -79,6 +79,7 @@ public Configuration() { analysis = new Analysis(); panel = new Panel(); server = new ServerConfiguration(); + account = new AccountConfiguration(); } public void serialize(OutputStream configurationOututStream) throws IOException { @@ -113,7 +114,6 @@ public static Configuration load(InputStream configurationInputStream, String fo } catch (IOException e) { throw new IOException("Configuration file could not be parsed: " + e.getMessage(), e); } - addDefaultValueIfMissing(configuration); // We must always overwrite configuration with environment parameters @@ -122,8 +122,15 @@ public static Configuration load(InputStream configurationInputStream, String fo } private static void addDefaultValueIfMissing(Configuration configuration) { - if (configuration.getMaxLoginAttempts() <= 0) { - configuration.setMaxLoginAttempts(5); + if (configuration.getAccount() == null) { + configuration.setAccount(new AccountConfiguration()); + } + if (configuration.getAccount().getMaxLoginAttempts() <= 0) { + configuration.getAccount().setMaxLoginAttempts(5); + } + if (configuration.getAccount().getPasswordExpirationDays() < 0) { + // Disable password expiration by default + configuration.getAccount().setPasswordExpirationDays(0); } } @@ -147,7 +154,8 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio configuration.getMonitor().setPort(Integer.parseInt(value)); break; case "OPENCGA.MAX_LOGIN_ATTEMPTS": - configuration.setMaxLoginAttempts(Integer.parseInt(value)); + case "OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS": + configuration.getAccount().setMaxLoginAttempts(Integer.parseInt(value)); break; case "OPENCGA_EXECUTION_MODE": case "OPENCGA_EXECUTION_ID": @@ -203,6 +211,16 @@ public static void reportUnusedField(String field, Object value) { } } + public static void reportMovedField(String previousField, String newField, Object value) { + // Report only if the value is not null and not an empty string + if (value != null && !(value instanceof String && ((String) value).isEmpty())) { + if (reportedFields.add(previousField)) { + // Only log the first time a field is found + logger.warn("Option '{}' with value '{}' was moved to '{}'.", previousField, value, newField); + } + } + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("Configuration{"); @@ -211,7 +229,7 @@ public String toString() { sb.append(", databasePrefix='").append(databasePrefix).append('\''); sb.append(", workspace='").append(workspace).append('\''); sb.append(", jobDir='").append(jobDir).append('\''); - sb.append(", maxLoginAttempts=").append(maxLoginAttempts); + sb.append(", account=").append(account); sb.append(", monitor=").append(monitor); sb.append(", healthCheck=").append(healthCheck); sb.append(", audit=").append(audit); @@ -292,12 +310,24 @@ public Configuration setJobDir(String jobDir) { return this; } + public AccountConfiguration getAccount() { + return account; + } + + public Configuration setAccount(AccountConfiguration account) { + this.account = account; + return this; + } + + @Deprecated public int getMaxLoginAttempts() { - return maxLoginAttempts; + return account.getMaxLoginAttempts(); } + @Deprecated public Configuration setMaxLoginAttempts(int maxLoginAttempts) { - this.maxLoginAttempts = maxLoginAttempts; + reportMovedField("configuration.yml#maxLoginAttempts", "configuration.yml#account.maxLoginAttempts", maxLoginAttempts); + account.setMaxLoginAttempts(maxLoginAttempts); return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java index 0ebc7f92d22..deb5f28ba00 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserInterpretationAnalysisParams.java @@ -1,26 +1,32 @@ package org.opencb.opencga.core.models.clinical; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; - public class ExomiserInterpretationAnalysisParams extends ToolParams { - public static final String DESCRIPTION = "Exomizer interpretation analysis params"; + public static final String DESCRIPTION = "Exomiser interpretation analysis params"; + @DataField(id = "clinicalAnalysis", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_DESCRIPTION, required = true) private String clinicalAnalysis; + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) + private String exomiserVersion; + public ExomiserInterpretationAnalysisParams() { } - public ExomiserInterpretationAnalysisParams(String clinicalAnalysis) { + public ExomiserInterpretationAnalysisParams(String clinicalAnalysis, String exomiserVersion) { this.clinicalAnalysis = clinicalAnalysis; - + this.exomiserVersion = exomiserVersion; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("TieringInterpretationAnalysisParams{"); + final StringBuilder sb = new StringBuilder("ExomiserInterpretationAnalysisParams{"); sb.append("clinicalAnalysis='").append(clinicalAnalysis).append('\''); + sb.append(", exomiserVersion='").append(exomiserVersion).append('\''); sb.append('}'); return sb.toString(); } @@ -34,5 +40,12 @@ public ExomiserInterpretationAnalysisParams setClinicalAnalysis(String clinicalA return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + public ExomiserInterpretationAnalysisParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 3b836063439..33a5415422a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -4,14 +4,15 @@ import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; -import java.util.Map; - public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; - @DataField(id = "sample", description = FieldConstants.SAMPLE_ID_DESCRIPTION) + @DataField(id = "sample", description = FieldConstants.EXOMISER_SAMPLE_DESCRIPTION, required = true) private String sample; + @DataField(id = "exomiserVersion", description = FieldConstants.EXOMISER_VERSION_DESCRIPTION) + private String exomiserVersion; + @DataField(id = "clinicalAnalysisType", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION, defaultValue = "SINGLE") private String clinicalAnalysisType; @@ -22,9 +23,10 @@ public class ExomiserWrapperParams extends ToolParams { public ExomiserWrapperParams() { } - public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String outdir) { + public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String exomiserVersion, String outdir) { this.sample = sample; this.clinicalAnalysisType = clinicalAnalysisType; + this.exomiserVersion = exomiserVersion; this.outdir = outdir; } @@ -33,6 +35,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("ExomiserWrapperParams{"); sb.append("sample='").append(sample).append('\''); sb.append(", clinicalAnalysisType=").append(clinicalAnalysisType); + sb.append("exomiserVersion='").append(exomiserVersion).append('\''); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -56,6 +59,15 @@ public ExomiserWrapperParams setClinicalAnalysisType(String clinicalAnalysisType return this; } + public String getExomiserVersion() { + return exomiserVersion; + } + + public ExomiserWrapperParams setExomiserVersion(String exomiserVersion) { + this.exomiserVersion = exomiserVersion; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java index f31d4d477b4..293ade331be 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Account.java @@ -27,44 +27,49 @@ */ public class Account { - @DataField(id = "creationDate", indexed = true, - description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) - private String creationDate; - @DataField(id = "expirationDate", indexed = true, - description = FieldConstants.ACCOUNT_EXPIRATION_DATE_DESCRIPTION) + description = FieldConstants.INTERNAL_ACCOUNT_EXPIRATION_DATE_DESCRIPTION) private String expirationDate; + @DataField(id = "password", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_DESCRIPTION) + private Password password; + + @DataField(id = "failedAttempts", description = FieldConstants.INTERNAL_ACCOUNT_FAILED_ATTEMPTS_DESCRIPTION) + private int failedAttempts; @DataField(id = "authentication", indexed = true, uncommentedClasses = {"AccountType"}, - description = FieldConstants.ACCOUNT_AUTHENTICATION) + description = FieldConstants.INTERNAL_ACCOUNT_AUTHENTICATION) private AuthenticationOrigin authentication; public Account() { String creationDate = TimeUtils.getTime(); + // Default 1 year Calendar cal = Calendar.getInstance(); cal.setTime(TimeUtils.toDate(creationDate)); cal.add(Calendar.YEAR, +1); String expirationDate = TimeUtils.getTime(cal.getTime()); - this.creationDate = creationDate; this.expirationDate = expirationDate; + this.password = new Password(); + this.failedAttempts = 0; this.authentication = null; } - public Account(String creationDate, String expirationDate, AuthenticationOrigin authentication) { + public Account(String expirationDate, Password password, int failedAttempts, AuthenticationOrigin authentication) { this.expirationDate = expirationDate; - this.creationDate = creationDate; + this.password = password; + this.failedAttempts = failedAttempts; this.authentication = authentication; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Account{"); - sb.append("creationDate='").append(creationDate).append('\''); - sb.append(", expirationDate='").append(expirationDate).append('\''); - sb.append(", authentication='").append(authentication).append('\''); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", password=").append(password); + sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", authentication=").append(authentication); sb.append('}'); return sb.toString(); } @@ -78,12 +83,21 @@ public Account setExpirationDate(String expirationDate) { return this; } - public String getCreationDate() { - return creationDate; + public Password getPassword() { + return password; + } + + public Account setPassword(Password password) { + this.password = password; + return this; + } + + public int getFailedAttempts() { + return failedAttempts; } - public Account setCreationDate(String creationDate) { - this.creationDate = creationDate; + public Account setFailedAttempts(int failedAttempts) { + this.failedAttempts = failedAttempts; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java index ce30fd63c41..938ebd6dc7f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java @@ -11,16 +11,16 @@ public class OrganizationUserUpdateParams extends UserUpdateParams { private UserQuota quota; - private Account account; + private Internal internal; private Map attributes; public OrganizationUserUpdateParams() { } - public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Account account, Map attributes) { + public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Internal internal, Map attributes) { super(name, email); this.quota = quota; - this.account = account; + this.internal = internal; this.attributes = attributes; } @@ -33,7 +33,7 @@ public ObjectMap getUpdateMap() throws JsonProcessingException { public String toString() { final StringBuilder sb = new StringBuilder("OrganizationUserUpdateParams{"); sb.append("quota=").append(quota); - sb.append(", account=").append(account); + sb.append(", internal=").append(internal); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); @@ -48,12 +48,27 @@ public OrganizationUserUpdateParams setQuota(UserQuota quota) { return this; } + @Deprecated + @JsonIgnore public Account getAccount() { - return account; + return getInternal().getAccount(); } + @Deprecated public OrganizationUserUpdateParams setAccount(Account account) { - this.account = account; + if (internal == null) { + internal = new Internal(); + } + internal.setAccount(account); + return this; + } + + public Internal getInternal() { + return internal; + } + + public OrganizationUserUpdateParams setInternal(Internal internal) { + this.internal = internal; return this; } @@ -78,6 +93,30 @@ public OrganizationUserUpdateParams setEmail(String email) { return this; } + public static class Internal { + private Account account; + + public Internal() { + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Internal{"); + sb.append("account=").append(account); + sb.append('}'); + return sb.toString(); + } + + public Account getAccount() { + return account; + } + + public Internal setAccount(Account account) { + this.account = account; + return this; + } + } + public static class Account { private String expirationDate; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java new file mode 100644 index 00000000000..5b375936bd3 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/Password.java @@ -0,0 +1,48 @@ +package org.opencb.opencga.core.models.user; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class Password { + + @DataField(id = "expirationDate", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_EXPIRATION_DATE_DESCRIPTION) + private String expirationDate; + + @DataField(id = "lastModified", since = "3.2.1", description = FieldConstants.INTERNAL_ACCOUNT_PASSWORD_LAST_MODIFIED_DESCRIPTION) + private String lastModified; + + public Password() { + } + + public Password(String expirationDate, String lastModified) { + this.expirationDate = expirationDate; + this.lastModified = lastModified; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Password{"); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getExpirationDate() { + return expirationDate; + } + + public Password setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; + return this; + } + + public String getLastModified() { + return lastModified; + } + + public Password setLastModified(String lastModified) { + this.lastModified = lastModified; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java index b4d3ef26c02..98273a80528 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/User.java @@ -49,14 +49,15 @@ public class User { @DataField(id = "organization", indexed = true, description = FieldConstants.USER_ORGANIZATION) private String organization; - @DataField(id = "account", indexed = true, description = FieldConstants.USER_ACCOUNT) - private Account account; + @DataField(id = "creationDate", since = "3.2.1", description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; - /** - * An object describing the internal information of the User. This is managed by OpenCGA. - * - * @apiNote Internal - */ + @DataField(id = "modificationDate", since = "3.2.1", description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "account", description = FieldConstants.USER_ACCOUNT) + @Deprecated + private Account account; @DataField(id = "internal", indexed = true, description = FieldConstants.GENERIC_INTERNAL) private UserInternal internal; @@ -89,22 +90,21 @@ public class User { public User() { } - public User(String id, Account account) { - this(id, id, null, null, account, new UserInternal(new UserStatus()), null, Collections.emptyList(), Collections.emptyMap(), + public User(String id) { + this(id, id, null, null, null, null, new UserInternal(new UserStatus()), null, Collections.emptyMap(), new LinkedList<>(), Collections.emptyMap()); } public User(String id, String name, String email, String organization, UserInternal internal) { - this(id, name, email, organization, null, internal, null, new ArrayList<>(), new HashMap<>(), new LinkedList<>(), new HashMap<>()); + this(id, name, email, organization, null, null, internal, null, new HashMap<>(), new LinkedList<>(), new HashMap<>()); } - public User(String id, String name, String email, String organization, Account account, UserInternal internal, UserQuota quota, - List projects, Map configs, List filters, Map attributes) { + public User(String id, String name, String email, String organization, UserInternal internal, UserQuota quota, List projects, + Map configs, List filters, Map attributes) { this.id = id; this.name = name; this.email = email; this.organization = organization; - this.account = account != null ? account : new Account(); this.internal = internal; this.quota = quota; this.projects = projects; @@ -113,6 +113,22 @@ public User(String id, String name, String email, String organization, Account a this.attributes = attributes; } + public User(String id, String name, String email, String organization, String creationDate, String modificationDate, + UserInternal internal, UserQuota quota, Map configs, List filters, + Map attributes) { + this.id = id; + this.name = name; + this.email = email; + this.organization = organization; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.internal = internal; + this.quota = quota; + this.configs = configs; + this.filters = filters; + this.attributes = attributes; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("User{"); @@ -120,7 +136,8 @@ public String toString() { sb.append(", name='").append(name).append('\''); sb.append(", email='").append(email).append('\''); sb.append(", organization='").append(organization).append('\''); - sb.append(", account=").append(account); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", internal=").append(internal); sb.append(", quota=").append(quota); sb.append(", projects=").append(projects); @@ -167,10 +184,30 @@ public User setOrganization(String organization) { return this; } + public String getCreationDate() { + return creationDate; + } + + public User setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public User setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + @Deprecated public Account getAccount() { return account; } + @Deprecated public User setAccount(Account account) { this.account = account; return this; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java index 329db613b5b..55a75709bd1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java @@ -16,32 +16,41 @@ package org.opencb.opencga.core.models.user; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; public class UserInternal extends Internal { + @DataField(id = "status", description = FieldConstants.INTERNAL_STATUS_DESCRIPTION) private UserStatus status; - private int failedAttempts; + + @DataField(id = "account", since = "3.2.1", description = FieldConstants.USER_ACCOUNT) + private Account account; public UserInternal() { } public UserInternal(UserStatus status) { - this(TimeUtils.getTime(), TimeUtils.getTime(), status); + this(TimeUtils.getTime(), TimeUtils.getTime(), status, new Account()); + } + + public UserInternal(UserStatus status, Account account) { + this(TimeUtils.getTime(), TimeUtils.getTime(), status, account); } - public UserInternal(String registrationDate, String lastModified, UserStatus status) { + public UserInternal(String registrationDate, String lastModified, UserStatus status1, Account account) { super(null, registrationDate, lastModified); - this.status = status; - this.failedAttempts = 0; + this.status = status1; + this.account = account; } @Override public String toString() { final StringBuilder sb = new StringBuilder("UserInternal{"); sb.append("status=").append(status); - sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", account=").append(account); sb.append(", registrationDate='").append(registrationDate).append('\''); sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); @@ -57,12 +66,13 @@ public UserInternal setStatus(UserStatus status) { return this; } - public int getFailedAttempts() { - return failedAttempts; + public Account getAccount() { + return account; } - public UserInternal setFailedAttempts(int failedAttempts) { - this.failedAttempts = failedAttempts; + public UserInternal setAccount(Account account) { + this.account = account; return this; } + } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java index 15cf7a51215..0b16875bebb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/OpenCgaToolExecutor.java @@ -17,9 +17,10 @@ package org.opencb.opencga.core.tools; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.core.tools.annotations.ToolExecutor; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import java.nio.file.Path; @@ -31,6 +32,7 @@ public abstract class OpenCgaToolExecutor { private ObjectMap executorParams; private Path outDir; private ExecutionResultManager arm; + private Configuration configuration; protected OpenCgaToolExecutor() { } @@ -51,10 +53,11 @@ public final ToolExecutor.Source getSource() { return this.getClass().getAnnotation(ToolExecutor.class).source(); } - public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir) { + public final void setUp(ExecutionResultManager arm, ObjectMap executorParams, Path outDir, Configuration configuration) { this.arm = arm; this.executorParams = executorParams; this.outDir = outDir; + this.configuration = configuration; } public final void execute() throws ToolException { @@ -77,6 +80,10 @@ public final Path getOutDir() { return outDir; } + public final Configuration getConfiguration() { + return configuration; + } + protected final String getToken() { return getExecutorParams().getString("token"); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java index 2af4d769513..980a0a8d494 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/GwasAnalysisExecutor.java @@ -33,7 +33,7 @@ public abstract class GwasAnalysisExecutor extends OpenCgaToolExecutor { private String cohort1; private String cohort2; private Path outputFile; - private GwasConfiguration configuration; + private GwasConfiguration gwasConfiguration; public GwasAnalysisExecutor() { } @@ -49,7 +49,7 @@ protected List getHeaderColumns() { columns.add("gene"); columns.add("biotype"); columns.add("consequence-types"); - if (configuration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { + if (gwasConfiguration.getMethod() == GwasConfiguration.Method.CHI_SQUARE_TEST) { columns.add("chi-square"); } columns.add("p-value"); @@ -70,7 +70,7 @@ public String toString() { sb.append(", phenotype2='").append(phenotype2).append('\''); sb.append(", cohort1='").append(cohort1).append('\''); sb.append(", cohort2='").append(cohort2).append('\''); - sb.append(", configuration=").append(configuration); + sb.append(", configuration=").append(gwasConfiguration); sb.append(", executorParams=").append(getExecutorParams()); sb.append(", outDir=").append(getOutDir()); sb.append('}'); @@ -149,12 +149,12 @@ public GwasAnalysisExecutor setOutputFile(Path outputFile) { return this; } - public GwasConfiguration getConfiguration() { - return configuration; + public GwasConfiguration getGwasConfiguration() { + return gwasConfiguration; } - public GwasAnalysisExecutor setConfiguration(GwasConfiguration configuration) { - this.configuration = configuration; + public GwasAnalysisExecutor setGwasConfiguration(GwasConfiguration gwasConfiguration) { + this.gwasConfiguration = gwasConfiguration; return this; } } diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index e3661d0167a..0b2b007817e 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -7,7 +7,9 @@ workspace: ${OPENCGA.USER.WORKSPACE} jobDir: ${OPENCGA.USER.WORKSPACE}/jobs # Maximum number of login attempts before banning a user account -maxLoginAttempts: ${OPENCGA.MAX_LOGIN_ATTEMPTS} +account: + maxLoginAttempts: ${OPENCGA.ACCOUNT.MAX_LOGIN_ATTEMPTS} + passwordExpirationDays: 0 panel: host: "http://resources.opencb.org/opencb/opencga/disease-panels" @@ -65,9 +67,32 @@ healthCheck: analysis: - packages: # List of packages where to find analysis tools + # List of packages where to find analysis tools + packages: - "org.opencb.opencga" - scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" # Scratch folder for the analysis. + # Scratch folder for the analysis. + scratchDir: "${OPENCGA.ANALYSIS.SCRATCH.DIR}" + # Default URL for downloading analysis resources. + resourceUrl: "http://resources.opencb.org/opencb/opencga/analysis/" + # Docker used by OpenCGA analysis and containing external tools such as samtools, bcftools, tabix, fastqc, plink1.9, bwa and r-base + # You can indicate the version, e.g: opencb/opencga-ext-tools:2.12.0, otherwise the current OpenCGA version will be used + opencgaExtTools: "opencb/opencga-ext-tools" + tools: + - id: "exomiser" + version: "13.1" + dockerId: "exomiser/exomiser-cli:13.1.0" + resources: + HG19: "exomiser/2109_hg19.zip" + HG38: "exomiser/2109_hg38.zip" + PHENOTYPE: "exomiser/2109_phenotype.zip" + - id: "exomiser" + version: "14.0" + defaultVersion: true + dockerId: "exomiser/exomiser-cli:14.0.0" + resources: + HG19: "exomiser/2402_hg19.zip" + HG38: "exomiser/2402_hg38.zip" + PHENOTYPE: "exomiser/2402_phenotype.zip" execution: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index be6ec91c91f..70748fffb0c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -1332,7 +1332,7 @@ public Response circos( ObjectMap executorParams = new ObjectMap(); executorParams.put("opencgaHome", opencgaHome); executorParams.put("token", token); - executor.setUp(null, executorParams, outDir.toPath()); + executor.setUp(null, executorParams, outDir.toPath(), null); // Run Circos executor StopWatch watch = StopWatch.createStarted(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 680d30a20db..1e8dcdc94dc 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -56,7 +56,6 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -191,14 +190,7 @@ public ObjectMap getConfiguration() { public Lock lockGlobal(long lockDuration, long timeout, String lockName) throws StorageEngineException { - try { - return projectDBAdaptor.lockProject(lockDuration, timeout, lockName); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new StorageEngineException("Unable to lock the Project", e); - } catch (TimeoutException e) { - throw new StorageEngineException("Unable to lock the Project", e); - } + return projectDBAdaptor.lockProject(lockDuration, timeout, lockName); } public Lock lockStudy(int studyId) throws StorageEngineException { @@ -282,8 +274,7 @@ public StudyMetadata updateStudyMetadata(Object study, Upd throws StorageEngineException, E { int studyId = getStudyId(study); - Lock lock = lockStudy(studyId); - try { + try (Lock lock = lockStudy(studyId)) { StudyMetadata sm = getStudyMetadata(studyId); sm = updater.update(sm); @@ -291,8 +282,6 @@ public StudyMetadata updateStudyMetadata(Object study, Upd lock.checkLocked(); unsecureUpdateStudyMetadata(sm); return sm; - } finally { - lock.unlock(); } } @@ -557,16 +546,8 @@ public ProjectMetadata updateProjectMetadata(UpdateConsume public ProjectMetadata updateProjectMetadata(UpdateFunction function) throws StorageEngineException, E { Objects.requireNonNull(function); - Lock lock; - try { - lock = projectDBAdaptor.lockProject(lockDuration, lockTimeout); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new StorageEngineException("Unable to lock the Project", e); - } catch (TimeoutException e) { - throw new StorageEngineException("Unable to lock the Project", e); - } - try { + + try (Lock lock = projectDBAdaptor.lockProject(lockDuration, lockTimeout)) { ProjectMetadata projectMetadata = getProjectMetadata(); int countersHash = (projectMetadata == null ? Collections.emptyMap() : projectMetadata.getCounters()).hashCode(); @@ -579,8 +560,6 @@ public ProjectMetadata updateProjectMetadata(UpdateFunctio lock.checkLocked(); projectDBAdaptor.updateProjectMetadata(projectMetadata, updateCounters); return projectMetadata; - } finally { - lock.unlock(); } } @@ -594,11 +573,15 @@ public ProjectMetadata getProjectMetadata() { public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws StorageEngineException { ProjectMetadata projectMetadata = getProjectMetadata(); + + checkSameSpeciesAndAssembly(options, projectMetadata); if (options != null && (projectMetadata == null || StringUtils.isEmpty(projectMetadata.getSpecies()) && options.containsKey(SPECIES.key()) || StringUtils.isEmpty(projectMetadata.getAssembly()) && options.containsKey(ASSEMBLY.key()))) { projectMetadata = updateProjectMetadata(pm -> { + // Check again, in case it was updated by another thread + checkSameSpeciesAndAssembly(options, pm); if (pm == null) { pm = new ProjectMetadata(); } @@ -619,6 +602,25 @@ public ProjectMetadata getAndUpdateProjectMetadata(ObjectMap options) throws Sto return projectMetadata; } + private static void checkSameSpeciesAndAssembly(ObjectMap options, ProjectMetadata projectMetadata) throws StorageEngineException { + if (options != null && projectMetadata != null) { + if (options.containsKey(ASSEMBLY.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getAssembly()) && !projectMetadata.getAssembly() + .equalsIgnoreCase(options.getString(ASSEMBLY.key()))) { + throw new StorageEngineException("Incompatible assembly change from '" + projectMetadata.getAssembly() + "' to '" + + options.getString(ASSEMBLY.key()) + "'"); + } + } + if (options.containsKey(SPECIES.key())) { + if (StringUtils.isNotEmpty(projectMetadata.getSpecies()) && !projectMetadata.getSpecies() + .equalsIgnoreCase(toCellBaseSpeciesName(options.getString(SPECIES.key())))) { + throw new StorageEngineException("Incompatible species change from '" + projectMetadata.getSpecies() + "' to '" + + options.getString(SPECIES.key()) + "'"); + } + } + } + } + public DataResult getVariantFileMetadata(int studyId, int fileId, QueryOptions options) throws StorageEngineException { return fileDBAdaptor.getVariantFileMetadata(studyId, fileId, options); @@ -673,16 +675,14 @@ public void unsecureUpdateFileMetadata(int studyId, FileMetadata file) { public FileMetadata updateFileMetadata(int studyId, int fileId, UpdateConsumer update) throws E, StorageEngineException { getFileName(studyId, fileId); // Check file exists - Lock lock = fileDBAdaptor.lock(studyId, fileId, lockDuration, lockTimeout); - try { + + try (Lock lock = fileDBAdaptor.lock(studyId, fileId, lockDuration, lockTimeout)) { FileMetadata fileMetadata = getFileMetadata(studyId, fileId); update.update(fileMetadata); lock.checkLocked(); unsecureUpdateFileMetadata(studyId, fileMetadata); fileIdIndexedCache.put(studyId, fileId, fileMetadata.isIndexed()); return fileMetadata; - } finally { - lock.unlock(); } } @@ -863,6 +863,19 @@ public Iterator fileMetadataIterator(int studyId) { return fileDBAdaptor.fileIterator(studyId); } + public SampleMetadata getSampleMetadata(Integer studyId, Integer sampleId) { + return getSampleMetadata(studyId.intValue(), sampleId.intValue()); + } + + public SampleMetadata getSampleMetadata(int studyId, Integer sampleId) { + return getSampleMetadata(studyId, sampleId.intValue()); + } + + public SampleMetadata getSampleMetadata(int studyId, Object sample) { + int sampleId = getSampleIdOrFail(studyId, sample); + return getSampleMetadata(studyId, sampleId); + } + public SampleMetadata getSampleMetadata(int studyId, int sampleId) { return sampleDBAdaptor.getSampleMetadata(studyId, sampleId, null); } @@ -875,15 +888,13 @@ public void unsecureUpdateSampleMetadata(int studyId, SampleMetadata sample) { public SampleMetadata updateSampleMetadata(int studyId, int sampleId, UpdateConsumer consumer) throws E, StorageEngineException { getSampleName(studyId, sampleId); // Check sample exists - Lock lock = sampleDBAdaptor.lock(studyId, sampleId, lockDuration, lockTimeout); - try { + + try (Lock lock = sampleDBAdaptor.lock(studyId, sampleId, lockDuration, lockTimeout)) { SampleMetadata sample = getSampleMetadata(studyId, sampleId); sample = consumer.toFunction().update(sample); lock.checkLocked(); unsecureUpdateSampleMetadata(studyId, sample); return sample; - } finally { - lock.unlock(); } } @@ -1054,15 +1065,12 @@ public void unsecureUpdateCohortMetadata(int studyId, CohortMetadata cohort) { public CohortMetadata updateCohortMetadata(int studyId, int cohortId, UpdateConsumer update) throws E, StorageEngineException { getCohortName(studyId, cohortId); // Check cohort exists - Lock lock = cohortDBAdaptor.lock(studyId, cohortId, lockDuration, lockTimeout); - try { + try (Lock lock = cohortDBAdaptor.lock(studyId, cohortId, lockDuration, lockTimeout)) { CohortMetadata cohortMetadata = getCohortMetadata(studyId, cohortId); update.update(cohortMetadata); lock.checkLocked(); unsecureUpdateCohortMetadata(studyId, cohortMetadata); return cohortMetadata; - } finally { - lock.unlock(); } } @@ -1190,13 +1198,19 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle for (Integer sampleId : sampleIds) { Integer finalCohortId = cohortId; if (secondaryIndexCohort) { - updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.addSecondaryIndexCohort(finalCohortId); - }); + if (!getSampleMetadata(studyId, sampleId).getSecondaryIndexCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.addSecondaryIndexCohort(finalCohortId); + }); + } } else { - updateSampleMetadata(studyId, sampleId, sampleMetadata -> { - sampleMetadata.addCohort(finalCohortId); - }); + if (!getSampleMetadata(studyId, sampleId).getCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + sampleMetadata.addCohort(finalCohortId); + }); + } } } @@ -1209,13 +1223,19 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle Integer finalCohortId = cohortId; if (!sampleIds.contains(sampleFromCohort)) { if (secondaryIndexCohort) { - updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { - sampleMetadata.getSecondaryIndexCohorts().remove(finalCohortId); - }); + if (getSampleMetadata(studyId, sampleFromCohort).getSecondaryIndexCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { + sampleMetadata.getSecondaryIndexCohorts().remove(finalCohortId); + }); + } } else { - updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { - sampleMetadata.getCohorts().remove(finalCohortId); - }); + if (getSampleMetadata(studyId, sampleFromCohort).getCohorts().contains(finalCohortId)) { + // Avoid unnecessary updates + updateSampleMetadata(studyId, sampleFromCohort, sampleMetadata -> { + sampleMetadata.getCohorts().remove(finalCohortId); + }); + } } } } @@ -1326,15 +1346,12 @@ public void unsecureUpdateTask(int studyId, TaskMetadata task) throws StorageEng public TaskMetadata updateTask(int studyId, int taskId, UpdateConsumer consumer) throws E, StorageEngineException { getTask(studyId, taskId); // Check task exists - Lock lock = taskDBAdaptor.lock(studyId, taskId, lockDuration, lockTimeout); - try { + try (Lock lock = taskDBAdaptor.lock(studyId, taskId, lockDuration, lockTimeout)) { TaskMetadata task = getTask(studyId, taskId); consumer.update(task); lock.checkLocked(); unsecureUpdateTask(studyId, task); return task; - } finally { - lock.unlock(); } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java index dc88a85d338..3045ee8f3cd 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/ProjectMetadataAdaptor.java @@ -7,7 +7,6 @@ import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import java.io.IOException; -import java.util.concurrent.TimeoutException; /** * Created on 02/05/18. @@ -17,14 +16,12 @@ public interface ProjectMetadataAdaptor extends AutoCloseable { default Lock lockProject(long lockDuration, long timeout) - throws InterruptedException, TimeoutException, StorageEngineException { + throws StorageEngineException { return lockProject(lockDuration, timeout, null); } Lock lockProject(long lockDuration, long timeout, String lockName) - throws InterruptedException, TimeoutException, StorageEngineException; - - void unLockProject(long lockId) throws StorageEngineException; + throws StorageEngineException; DataResult getProjectMetadata(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index dd24ee1334d..b541b2f4ae4 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -1348,7 +1348,7 @@ public VariantQueryExecutor getVariantQueryExecutor(Query query, QueryOptions op public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQuery) { try { for (VariantQueryExecutor executor : getVariantQueryExecutors()) { - if (executor.canUseThisExecutor(variantQuery.getQuery(), variantQuery.getInputOptions())) { + if (executor.canUseThisExecutor(variantQuery, variantQuery.getInputOptions())) { logger.info("Using VariantQueryExecutor : " + executor.getClass().getName()); logger.info(" Query : " + VariantQueryUtils.printQuery(variantQuery.getInputQuery())); logger.info(" Options : " + variantQuery.getInputOptions().toJson()); @@ -1362,6 +1362,19 @@ public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQu throw new VariantQueryException("No VariantQueryExecutor found to run the query!"); } + public final VariantQueryExecutor getVariantQueryExecutor(Class clazz) + throws StorageEngineException { + Optional first = getVariantQueryExecutors() + .stream() + .filter(e -> e instanceof SearchIndexVariantQueryExecutor) + .findFirst(); + if (first.isPresent()) { + return first.get(); + } else { + throw new StorageEngineException("VariantQueryExecutor " + clazz + " not found"); + } + } + public Query preProcessQuery(Query originalQuery, QueryOptions options) { try { return getVariantQueryParser().preProcessQuery(originalQuery, options); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java index caefbb5260e..bc1be055466 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageOptions.java @@ -83,8 +83,8 @@ public enum VariantStorageOptions implements ConfigurationOption { INDEX_SEARCH("indexSearch", false), // Build secondary indexes using search engine. - METADATA_LOCK_DURATION("metadata.lock.duration", 5000), - METADATA_LOCK_TIMEOUT("metadata.lock.timeout", 60000), + METADATA_LOCK_DURATION("metadata.lock.duration", 60000), + METADATA_LOCK_TIMEOUT("metadata.lock.timeout", 600000), METADATA_LOAD_BATCH_SIZE("metadata.load.batchSize", 10), METADATA_LOAD_THREADS("metadata.load.numThreads", 4), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 5b37d2512b1..722d79e59fd 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -60,10 +60,7 @@ import org.opencb.opencga.storage.core.io.plain.StringDataReader; import org.opencb.opencga.storage.core.io.plain.StringDataWriter; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java index 8468ab34317..300f2e6b48a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java @@ -267,14 +267,6 @@ public VariantStudyQuery setStudies(ParsedQuery studies) { return this; } - public String getStudyOrFail() { - if (studies == null || studies.size() != 1) { - throw new VariantQueryException("Require exactly one study"); - } else { - return studies.get(0); - } - } - public ParsedQuery>> getGenotypes() { return genotypes; } @@ -311,6 +303,19 @@ public void setDefaultStudy(StudyMetadata defaultStudy) { public StudyMetadata getDefaultStudy() { return defaultStudy; } + + public StudyMetadata getDefaultStudyOrFail() { + if (defaultStudy == null) { + if (studies.size() != 1) { + throw new VariantQueryException("Only one study is allowed. Found " + studies.size() + " studies"); + } else { + throw new VariantQueryException("One study required. None provided"); + } + } else { + return defaultStudy; + } + } + } public static class VariantQueryXref { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java index 641e365a51d..bd02a6b4303 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java @@ -17,10 +17,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; -import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -170,15 +167,14 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.defaultValue())); - variantQuery.setProjection(projectionParser.parseVariantQueryProjection(inputQuery, options)); - VariantQuery query; if (!skipPreProcess) { - query = new VariantQuery(preProcessQuery(inputQuery, options, variantQuery.getProjection())); + query = new VariantQuery(preProcessQuery(inputQuery, options)); } else { query = new VariantQuery(inputQuery); } variantQuery.setQuery(query); + variantQuery.setProjection(projectionParser.parseVariantQueryProjection(query, options)); List geneRegions = Region.parseRegions(query.getString(ANNOT_GENE_REGIONS.key())); variantQuery.setGeneRegions(geneRegions == null ? Collections.emptyList() : geneRegions); @@ -221,10 +217,7 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo = new ParsedQuery<>(sampleDataQuery.getKey(), sampleDataQuery.getOperation(), new ArrayList<>(sampleDataQuery.size())); for (KeyValues> keyValues : sampleDataQuery) { sampleDataQueryWithMetadata.getValues().add( - keyValues.mapKey(sample -> { - int sampleId = metadataManager.getSampleIdOrFail(defaultStudy.getId(), sample); - return metadataManager.getSampleMetadata(defaultStudy.getId(), sampleId); - })); + keyValues.mapKey(sample -> metadataManager.getSampleMetadata(defaultStudy.getId(), sample))); } studyQuery.setSampleDataQuery(sampleDataQueryWithMetadata); } @@ -232,17 +225,13 @@ public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boo return variantQuery; } - public final Query preProcessQuery(Query originalQuery, QueryOptions options) { - return preProcessQuery(originalQuery, options, null); - } - - protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { + public Query preProcessQuery(Query originalQuery, QueryOptions options) { // Copy input query! Do not modify original query! Query query = VariantQueryUtils.copy(originalQuery); preProcessAnnotationParams(query); - preProcessStudyParams(query, options, projection); + preProcessStudyParams(query, options); if (options != null && options.getLong(QueryOptions.LIMIT) < 0) { throw VariantQueryException.malformedParam(QueryOptions.LIMIT, options.getString(QueryOptions.LIMIT), @@ -388,7 +377,7 @@ private VariantType parseVariantType(String type) { } } - protected void preProcessStudyParams(Query query, QueryOptions options, VariantQueryProjection projection) { + protected void preProcessStudyParams(Query query, QueryOptions options) { StudyMetadata defaultStudy = getDefaultStudy(query); QueryOperation formatOperator = null; if (isValidParam(query, SAMPLE_DATA)) { @@ -613,10 +602,13 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ if (isValidParam(query, SAMPLE_MENDELIAN_ERROR) || isValidParam(query, SAMPLE_DE_NOVO) - || isValidParam(query, SAMPLE_DE_NOVO_STRICT)) { + || isValidParam(query, SAMPLE_DE_NOVO_STRICT) + || isValidParam(query, SAMPLE_COMPOUND_HETEROZYGOUS)) { + boolean requireMendelianReady = false; QueryParam param = null; if (isValidParam(query, SAMPLE_MENDELIAN_ERROR)) { param = SAMPLE_MENDELIAN_ERROR; + requireMendelianReady = true; } if (isValidParam(query, SAMPLE_DE_NOVO)) { if (param != null) { @@ -624,6 +616,7 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ param, query.getString(param.key()), SAMPLE_DE_NOVO, query.getString(SAMPLE_DE_NOVO.key())); } + requireMendelianReady = true; param = SAMPLE_DE_NOVO; } if (isValidParam(query, SAMPLE_DE_NOVO_STRICT)) { @@ -632,8 +625,21 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ param, query.getString(param.key()), SAMPLE_DE_NOVO_STRICT, query.getString(SAMPLE_DE_NOVO_STRICT.key())); } + requireMendelianReady = true; param = SAMPLE_DE_NOVO_STRICT; } + if (isValidParam(query, SAMPLE_COMPOUND_HETEROZYGOUS)) { + if (param != null) { + throw VariantQueryException.unsupportedParamsCombination( + param, query.getString(param.key()), + SAMPLE_COMPOUND_HETEROZYGOUS, query.getString(SAMPLE_COMPOUND_HETEROZYGOUS.key())); + } + requireMendelianReady = false; + param = SAMPLE_COMPOUND_HETEROZYGOUS; + } + if (param == null) { + throw new IllegalStateException("Unknown param"); + } if (defaultStudy == null) { throw VariantQueryException.missingStudyForSamples(query.getAsStringList(param.key()), metadataManager.getStudyNames()); @@ -645,15 +651,18 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ genotypeParam, query.getString(genotypeParam.key()) ); } - List samples = query.getAsStringList(param.key()); + Object value = query.get(param.key()); + List samples; + if (value instanceof Trio) { + samples = Collections.singletonList(((Trio) value).getChild()); + } else { + samples = query.getAsStringList(param.key()); + } Set samplesAndParents = new LinkedHashSet<>(samples); for (String sample : samples) { - Integer sampleId = metadataManager.getSampleId(defaultStudy.getId(), sample); - if (sampleId == null) { - throw VariantQueryException.sampleNotFound(sample, defaultStudy.getName()); - } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(defaultStudy.getId(), sampleId); - if (TaskMetadata.Status.READY != sampleMetadata.getMendelianErrorStatus()) { + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(defaultStudy.getId(), sample); + if (requireMendelianReady + && TaskMetadata.Status.READY != sampleMetadata.getMendelianErrorStatus()) { throw VariantQueryException.malformedParam(param, "Sample \"" + sampleMetadata.getName() + "\" does not have the Mendelian Errors precomputed yet"); } @@ -674,6 +683,21 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ } else { query.put(INCLUDE_SAMPLE.key(), new ArrayList<>(samplesAndParents)); } + if (param == SAMPLE_COMPOUND_HETEROZYGOUS) { + int studyId = defaultStudy.getId(); + if (!(value instanceof Trio)) { + if (samples.size() > 1) { + throw VariantQueryException.malformedParam(SAMPLE, value.toString(), + "More than one sample provided for compound heterozygous filter."); + } + SampleMetadata sm = metadataManager.getSampleMetadata(studyId, samples.get(0)); + Trio trio = new Trio(null, + metadataManager.getSampleName(studyId, sm.getFather()), + metadataManager.getSampleName(studyId, sm.getMother()), + sm.getName()); + query.put(SAMPLE_COMPOUND_HETEROZYGOUS.key(), trio); + } + } } if (isValidParam(query, SCORE)) { @@ -704,9 +728,7 @@ protected void preProcessStudyParams(Query query, QueryOptions options, VariantQ || isValidParam(query, SAMPLE_SKIP) || isValidParam(query, SAMPLE_LIMIT) ) { - if (projection == null) { - projection = projectionParser.parseVariantQueryProjection(query, options); - } + VariantQueryProjection projection = projectionParser.parseVariantQueryProjection(query, options); // Apply the sample pagination. // Remove the sampleLimit and sampleSkip to avoid applying the pagination twice query.remove(SAMPLE_SKIP.key()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java index 6eb237ea4b3..bc40c5b6418 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java @@ -14,6 +14,7 @@ import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; @@ -40,7 +41,8 @@ public BreakendVariantQueryExecutor(String storageEngineId, ObjectMap options, } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); return query.getString(VariantQueryParam.TYPE.key()).equals(VariantType.BREAKEND.name()) && VariantQueryUtils.isValidParam(query, VariantQueryParam.GENOTYPE); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java index c6f4b87a5f0..bcecac57b5a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java @@ -64,14 +64,14 @@ public CompoundHeterozygousQueryExecutor(VariantStorageMetadataManager metadataM } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { - return isValidParam(query, VariantQueryUtils.SAMPLE_COMPOUND_HETEROZYGOUS); + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + return isValidParam(variantQuery.getQuery(), VariantQueryUtils.SAMPLE_COMPOUND_HETEROZYGOUS); } @Override protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { Trio trio = getCompHetTrio(variantQuery.getQuery()); - String study = variantQuery.getStudyQuery().getStudyOrFail(); + String study = variantQuery.getStudyQuery().getDefaultStudyOrFail().getName(); return getOrIterator(study, trio.getChild(), trio.getFather(), trio.getMother(), variantQuery, iterator); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java index 474cbc3fa9f..e667e2a1485 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java @@ -52,9 +52,9 @@ protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { for (QueryParam unsupportedParam : UNSUPPORTED_PARAMS) { - if (VariantQueryUtils.isValidParam(query, unsupportedParam)) { + if (VariantQueryUtils.isValidParam(variantQuery.getQuery(), unsupportedParam)) { logger.warn("Unsupported variant query param {} in {}", unsupportedParam.key(), DBAdaptorVariantQueryExecutor.class.getSimpleName()); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java index e286b4a07ce..5f2da2c2056 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java @@ -8,6 +8,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.query.*; @@ -34,7 +35,8 @@ public NoOpVariantQueryExecutor(VariantStorageMetadataManager metadataManager, S } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); boolean sampleQuery = false; String sample = null; if (VariantQueryUtils.isValidParam(query, VariantQueryParam.GENOTYPE)) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java index 26d53e89e5c..49ddc339658 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java @@ -2,7 +2,6 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -65,12 +64,12 @@ public static void setDefaultTimeout(QueryOptions queryOptions, ObjectMap config /** * Determine if this VariantQueryExecutor can run the given query. - * @param query Query to execute + * @param variantQuery Query to execute * @param options Options for the query * @return True if this variant query executor is valid for the query * @throws StorageEngineException if there is an error */ - public abstract boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException; + public abstract boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException; protected abstract Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java index bdcd501ad88..95add22ec8e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java @@ -71,7 +71,14 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti } Map> sampleIdsMap = getIncludeSampleIds(query, options, includeStudies, metadataManager); - int numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + int numTotalSamples; + if (isValidParam(query, NUM_TOTAL_SAMPLES)) { + // NUM_TOTAL_SAMPLES might have been defined in the PreProcess step. + // This implies that the current query has the samples already paginated. + numTotalSamples = query.getInt(NUM_TOTAL_SAMPLES.key()); + } else { + numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + } skipAndLimitSamples(query, sampleIdsMap); int numSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java index 200b2eb463d..cfa794cbe10 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java @@ -7,6 +7,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; @@ -28,7 +29,8 @@ public SamplesSearchIndexVariantQueryExecutor(VariantDBAdaptor dbAdaptor, Varian } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); String samplesCollection = inferSpecificSearchIndexSamplesCollection(query, options, getMetadataManager(), dbName); return samplesCollection != null && searchActiveAndAlive(samplesCollection); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index 12c86cc4e2b..899f71a9b72 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -11,10 +11,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; -import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; -import org.opencb.opencga.storage.core.variant.adaptors.VariantField; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.adaptors.*; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; @@ -76,7 +73,8 @@ public SearchIndexVariantQueryExecutor setIntersectParamsThreshold(int intersect } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) throws StorageEngineException { + VariantQuery query = variantQuery.getQuery(); return doQuerySearchManager(query, options) || doIntersectWithSearch(query, options); } @@ -185,7 +183,9 @@ public VariantQueryResult approximateCount(ParsedVariantQuery variantQuery DataResult nativeResult = searchManager .nativeQuery(dbName, searchEngineQuery, queryOptions); - List variantIds = nativeResult.getResults().stream().map(VariantSearchModel::getId).collect(Collectors.toList()); + List variantIds = nativeResult.getResults().stream() + .map(VariantSearchModel::toVariantSimple) + .collect(Collectors.toList()); // Adjust numSamples if the results from SearchManager is smaller than numSamples // If this happens, the count is not approximated if (variantIds.size() < sampling) { @@ -285,12 +285,12 @@ public boolean doIntersectWithSearch(Query query, QueryOptions options) { return intersect; } - protected Iterator variantIdIteratorFromSearch(Query query) { + protected Iterator variantIdIteratorFromSearch(Query query) { return variantIdIteratorFromSearch(query, Integer.MAX_VALUE, 0, null); } - protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { - Iterator variantsIterator; + protected Iterator variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) { + Iterator variantsIterator; QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, limit) .append(QueryOptions.SKIP, skip) @@ -304,14 +304,14 @@ protected Iterator variantIdIteratorFromSearch(Query query, int limit, i } variantsIterator = nativeResult.getResults() .stream() - .map(VariantSearchModel::getId) + .map(VariantSearchModel::toVariantSimple) .iterator(); } else { SolrNativeIterator nativeIterator = searchManager.nativeIterator(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeIterator.getNumFound()); } - variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::getId); + variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::toVariantSimple); } } catch (VariantSearchException | IOException e) { throw new VariantQueryException("Error querying " + VariantSearchManager.SEARCH_ENGINE_ID, e); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java index 9b0bb69792c..835af18a0a0 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchModel.java @@ -17,12 +17,15 @@ package org.opencb.opencga.storage.core.variant.search; import org.apache.solr.client.solrj.beans.Field; +import org.opencb.biodata.models.variant.Variant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter.HASH_PREFIX; + /** * Created by wasim on 09/11/16. */ @@ -140,6 +143,9 @@ public class VariantSearchModel { @Field("fileInfo_*") private Map fileInfo; + @Field("attr_*") + private Map attr; + public static final double MISSING_VALUE = -100.0; @@ -171,6 +177,7 @@ public VariantSearchModel() { this.qual = new HashMap<>(); this.filter = new HashMap<>(); this.fileInfo = new HashMap<>(); + this.attr = new HashMap<>(); } public VariantSearchModel(VariantSearchModel init) { @@ -210,6 +217,7 @@ public VariantSearchModel(VariantSearchModel init) { this.qual = init.getQual(); this.filter = init.getFilter(); this.fileInfo = init.getFileInfo(); + this.attr = init.getAttr(); } @Override @@ -251,6 +259,7 @@ public String toString() { sb.append(", qual=").append(qual); sb.append(", filter=").append(filter); sb.append(", fileInfo=").append(fileInfo); + sb.append(", attr=").append(attr); sb.append('}'); return sb.toString(); } @@ -259,6 +268,17 @@ public String getId() { return id; } + public Variant toVariantSimple() { + String variantId = getId(); + if (variantId.startsWith(HASH_PREFIX)) { + Object o = getAttr().get("attr_id"); + variantId = o instanceof String ? (String) o : ((List) o).get(0); + } + Variant variant = new Variant(variantId); + variant.setId(variantId); + return variant; + } + public VariantSearchModel setId(String id) { this.id = id; return this; @@ -579,4 +599,12 @@ public VariantSearchModel setFileInfo(Map fileInfo) { return this; } + public Map getAttr() { + return attr; + } + + public VariantSearchModel setAttr(Map attr) { + this.attr = attr; + return this; + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java index 6f55a892f7c..d9c119c2340 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java @@ -56,6 +56,7 @@ public class VariantSearchToVariantConverter implements ComplexTypeConverter includeFields; @@ -79,10 +80,9 @@ public VariantSearchToVariantConverter(Set includeFields) { @Override public Variant convertToDataModelType(VariantSearchModel variantSearchModel) { // set chromosome, start, end, ref, alt from ID - Variant variant = new Variant(variantSearchModel.getId()); + Variant variant = variantSearchModel.toVariantSimple(); - // set ID, chromosome, start, end, ref, alt, type - variant.setId(variantSearchModel.getVariantId()); + // set chromosome, start, end, ref, alt, type // set variant type if (StringUtils.isNotEmpty(variantSearchModel.getType())) { @@ -662,8 +662,10 @@ public VariantSearchModel convertToStorageType(Variant variant) { List other = new ArrayList<>(); // Set general Variant attributes: id, dbSNP, chromosome, start, end, type - variantSearchModel.setId(variant.toString()); // Internal unique ID e.g. 3:1000:AT:- - variantSearchModel.setVariantId(variant.getId()); + String variantId = getVariantId(variant); + variantSearchModel.setId(variantId); // Internal unique ID e.g. 3:1000:AT:- + variantSearchModel.setVariantId(variantId); + variantSearchModel.getAttr().put("attr_id", variant.toString()); variantSearchModel.setChromosome(variant.getChromosome()); variantSearchModel.setStart(variant.getStart()); variantSearchModel.setEnd(variant.getEnd()); @@ -1024,8 +1026,7 @@ public VariantSearchModel convertToStorageType(Variant variant) { // This field contains all possible IDs: id, dbSNP, names, genes, transcripts, protein, clinvar, hpo, ... // This will help when searching by variant id. This is added at the end of the method after collecting all IDs Set xrefs = variantAnnotationModelUtils.extractXRefs(variant.getAnnotation()); - xrefs.add(variantSearchModel.getId()); - xrefs.add(variantSearchModel.getVariantId()); + xrefs.add(variantId); if (variant.getNames() != null && !variant.getNames().isEmpty()) { variant.getNames().forEach(name -> { if (name != null) { @@ -1037,6 +1038,20 @@ public VariantSearchModel convertToStorageType(Variant variant) { return variantSearchModel; } + public static String getVariantId(Variant variant) { + String variantString = variant.toString(); + if (variantString.length() > 32766) { + // variantString.length() >= Short.MAX_VALUE + return hashVariantId(variant, variantString); + } else { + return variantString; + } + } + + public static String hashVariantId(Variant variant, String variantString) { + return HASH_PREFIX + variant.getChromosome() + ":" + variant.getStart() + ":" + Integer.toString(variantString.hashCode()); + } + private void convertStudies(Variant variant, VariantSearchModel variantSearchModel, List other) { // Sanity check if (CollectionUtils.isEmpty(variant.getStudies())) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java index e46c6379dc6..5365643b1fa 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchUtils.java @@ -280,7 +280,9 @@ public static String inferSpecificSearchIndexSamplesCollection( Set samples = new HashSet<>(); if (isValidParam(query, VariantQueryParam.SAMPLE)) { String value = query.getString(VariantQueryParam.SAMPLE.key()); - samples.addAll(splitValue(value).getValue()); + for (String sample : splitValue(value).getValue()) { + samples.add(sample.split(IS)[0]); + } } if (isValidParam(query, VariantQueryParam.GENOTYPE)) { HashMap> map = new HashMap<>(); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java index 461b6e498ba..f12bb85a8b1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParser.java @@ -25,7 +25,6 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrException; import org.opencb.biodata.models.core.Region; -import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.solr.FacetQueryParser; @@ -35,7 +34,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.*; +import org.opencb.opencga.storage.core.variant.query.KeyOpValue; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.Values; +import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.search.VariantSearchToVariantConverter; import org.slf4j.Logger; @@ -79,7 +81,7 @@ public class SolrQueryParser { static { includeMap = new HashMap<>(); - includeMap.put("id", "id,variantId"); + includeMap.put("id", "id,variantId,attr_id"); includeMap.put("chromosome", "chromosome"); includeMap.put("start", "start"); includeMap.put("end", "end"); @@ -477,7 +479,9 @@ private String parseGenomicFilter(Query query) { genes.addAll(variantQueryXref.getGenes()); xrefs.addAll(variantQueryXref.getIds()); xrefs.addAll(variantQueryXref.getOtherXrefs()); - xrefs.addAll(variantQueryXref.getVariants().stream().map(Variant::toString).collect(Collectors.toList())); + xrefs.addAll(variantQueryXref.getVariants().stream() + .map(VariantSearchToVariantConverter::getVariantId) + .collect(Collectors.toList())); // Regions if (StringUtils.isNotEmpty(query.getString(REGION.key()))) { @@ -1616,15 +1620,12 @@ private String[] includeFieldsWithMandatory(String[] includes) { return new String[0]; } - String[] mandatoryIncludeFields = new String[]{"id", "chromosome", "start", "end", "type"}; - String[] includeWithMandatory = new String[includes.length + mandatoryIncludeFields.length]; - for (int i = 0; i < includes.length; i++) { - includeWithMandatory[i] = includes[i]; - } - for (int i = 0; i < mandatoryIncludeFields.length; i++) { - includeWithMandatory[includes.length + i] = mandatoryIncludeFields[i]; - } - return includeWithMandatory; + Set mandatoryIncludeFields = new HashSet<>(Arrays.asList("id", "attr_id", "chromosome", "start", "end", "type")); + Set includeWithMandatory = new LinkedHashSet<>(includes.length + mandatoryIncludeFields.size()); + + includeWithMandatory.addAll(Arrays.asList(includes)); + includeWithMandatory.addAll(mandatoryIncludeFields); + return includeWithMandatory.toArray(new String[0]); } /** diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java index 2aacde2e3ce..30e4bba2e5f 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverterTest.java @@ -9,6 +9,7 @@ import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.metadata.VariantMetadata; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.io.managers.LocalIOConnector; @@ -45,6 +46,7 @@ public class VariantMetadataConverterTest { @Before public void setUp() throws Exception { metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + projectMetadata = metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); URI uri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); variantReaderUtils = new VariantReaderUtils(new IOConnectorProvider(LocalIOConnector.class)); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java index 1b3311958f4..71ea72de3c0 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManagerTest.java @@ -4,12 +4,14 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageTest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -103,4 +105,36 @@ public List getTasks(StudyMetadata study, List stat .map(TaskMetadata::getName) .collect(Collectors.toList()); } + + @Test + public void testAddSampleToCohort() throws Exception { + StudyMetadata study = metadataManager.createStudy("study"); + + metadataManager.registerCohort(study.getName(), "cohort1", Collections.emptyList()); + + int numSamples = 100; + List sampleIds = new ArrayList<>(numSamples); + for (int i = 0; i < numSamples; i++) { + sampleIds.add(metadataManager.registerSample(study.getId(), null, "sample_" + i)); + } + + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 10)); + VariantStorageMetadataManager metadataManager = Mockito.spy(this.metadataManager); + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.times(1)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + Mockito.reset(metadataManager); + metadataManager.addSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.never()).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 11)); + Mockito.verify(metadataManager, Mockito.never()).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 12)); + Mockito.verify(metadataManager, Mockito.times(1)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + + Mockito.reset(metadataManager); + metadataManager.setSamplesToCohort(study.getId(), "cohort1", sampleIds.subList(0, 6)); + Mockito.verify(metadataManager, Mockito.times(6)).updateSampleMetadata(Mockito.anyInt(), Mockito.anyInt(), Mockito.any()); + } + } \ No newline at end of file diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 32e1657fcb8..a675e5304b0 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -48,6 +48,8 @@ public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + if (!loaded) { clearDB(DB_NAME); loadFiles(); @@ -59,6 +61,7 @@ protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index 64fc14c4c6a..2523fad5ba2 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -1,5 +1,6 @@ package org.opencb.opencga.storage.core.variant; +import org.junit.Assume; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -20,6 +21,10 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; +import org.opencb.opencga.storage.core.variant.search.SearchIndexVariantQueryExecutor; import java.net.URI; import java.nio.file.Paths; @@ -53,18 +58,20 @@ public abstract class VariantStorageEngineSVTest extends VariantStorageBaseTest public void before() throws Exception { if (!loaded) { clearDB(DB_NAME); + } + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); + + if (!loaded) { loadFiles(); loaded = true; } } protected void loadFiles() throws Exception { - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); - input1 = getResourceUri("variant-test-sv.vcf"); studyMetadata = new StudyMetadata(1, "s1"); variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); @@ -106,6 +113,19 @@ public void checkCount() throws Exception { assertEquals(expected, count); } + @Test + public void checkSecondaryAnnotationIndex() throws Exception { + Assume.assumeTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor(SearchIndexVariantQueryExecutor.class); + for (Variant variant : variantStorageEngine) { + ParsedVariantQuery query = variantStorageEngine + .parseQuery(new Query(VariantQueryParam.ID.key(), variant.toString()), new QueryOptions()); + VariantQueryResult result = variantQueryExecutor.get(query); + assertEquals(1, result.getNumResults()); + assertEquals(variant.toString(), result.first().toString()); + } + } + @Test public void checkCorrectnessFile1() throws Exception { checkCorrectness(VariantStorageEngineSVTest.input1); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java index 905f64a5373..8071b86a14e 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java @@ -377,18 +377,22 @@ public void testSampleLimitSkip() throws Exception { VariantQueryResult result = query(new Query(SAMPLE_METADATA.key(), true).append(VariantQueryParam.INCLUDE_SAMPLE.key(), ALL), options); System.out.println("samples(ALL) = " + result.getSamples()); - for (int i : new int[]{1, 3, 6, 8, 10}) { + int numSamples = metadataManager.getStudyIds().stream().mapToInt(id -> metadataManager.getIndexedSamples(id).size()).sum(); + assertEquals(8, numSamples); + for (int i : new int[]{1, 3, 6, numSamples, 10}) { result = query(new VariantQuery().sampleSkip(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(SKIP=" + i + ") = " + result.getSamples()); - assertEquals(Math.max(0, 8 - i), result.getSamples().values().stream().mapToInt(List::size).sum()); - assertEquals(Math.max(0, 8 - i), result.getNumSamples().intValue()); - assertEquals(8, result.getNumTotalSamples().intValue()); + int expected = Math.max(0, numSamples - i); + assertEquals("Skip = " + i + " , expected " + expected + " out of 8 samples", expected, result.getSamples().values().stream().mapToInt(List::size).sum()); + assertEquals("Skip = " + i + " , expected " + expected + " out of 8 samples", expected, result.getNumSamples().intValue()); + assertEquals(numSamples, result.getNumTotalSamples().intValue()); result = query(new VariantQuery().sampleLimit(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(LIMIT=" + i + ") = " + result.getSamples()); - assertEquals(Math.min(8, i), result.getSamples().values().stream().mapToInt(List::size).sum()); - assertEquals(Math.min(8, i), result.getNumSamples().intValue()); - assertEquals(8, result.getNumTotalSamples().intValue()); + expected = Math.min(numSamples, i); + assertEquals("Limit = " + i + " , expected " + expected + " out of 8 samples", expected, result.getSamples().values().stream().mapToInt(List::size).sum()); + assertEquals("Limit = " + i + " , expected " + expected + " out of 8 samples", expected, result.getNumSamples().intValue()); + assertEquals(numSamples, result.getNumTotalSamples().intValue()); } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java index bed8d419666..3ba92ed7f1c 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyProjectMetadataAdaptor.java @@ -16,7 +16,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -29,7 +28,7 @@ public class DummyProjectMetadataAdaptor implements ProjectMetadataAdaptor { private static Map counters = new HashMap<>(); @Override - public Lock lockProject(long lockDuration, long timeout, String lockName) throws InterruptedException, TimeoutException { + public Lock lockProject(long lockDuration, long timeout, String lockName) { return new Lock(0) { @Override public void unlock0() { @@ -43,17 +42,12 @@ public void refresh() { }; } - @Override - public void unLockProject(long lockId) { - } - @Override public synchronized DataResult getProjectMetadata() { final DataResult result = new DataResult<>(); - if (projectMetadata == null) { - projectMetadata = new ProjectMetadata("hsapiens", "grch37", 1); + if (projectMetadata != null) { + result.setResults(Collections.singletonList(projectMetadata.copy())); } - result.setResults(Collections.singletonList(projectMetadata.copy())); return result; } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java index 221a126cd92..38f995c823d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/io/VariantWriterFactoryTest.java @@ -23,11 +23,13 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.metadata.VariantFileHeader; import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.io.DataWriter; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantDBAdaptor; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageMetadataDBAdaptorFactory; @@ -85,8 +87,10 @@ public void testContigLengthNull() throws IOException, StorageEngineException { new VariantFileHeaderComplexLine("contig", "chr3", null, null, null, Collections.singletonMap("length", ".")), new VariantFileHeaderComplexLine("contig", "chr4", null, null, null, Collections.singletonMap("length", "1234")) )); - StudyMetadata study = dbAdaptor.getMetadataManager().createStudy("study"); - dbAdaptor.getMetadataManager().unsecureUpdateStudyMetadata(study.setVariantHeader(header)); + VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); + StudyMetadata study = metadataManager.createStudy("study"); + metadataManager.unsecureUpdateStudyMetadata(study.setVariantHeader(header)); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(10000); DataWriter writer = new VariantWriterFactory(dbAdaptor).newDataWriter( VariantWriterFactory.VariantOutputFormat.VCF, diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java index 1fc7617422c..f5e4c2526b9 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java @@ -12,6 +12,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -88,6 +89,9 @@ public void setUp() throws Exception { fileIndexed = true; Integer indexedFileId = metadataManager.getIndexedFiles(studyMetadata.getId()).iterator().next(); + Trio trio = new Trio("NA19660", "NA19661", "NA19685"); + variantStorageEngine.familyIndex(studyMetadata.getName(), Collections.singletonList(trio), new ObjectMap()); + //Calculate stats QueryOptions options = new QueryOptions(VariantStorageOptions.STUDY.key(), STUDY_NAME) .append(VariantStorageOptions.LOAD_BATCH_SIZE.key(), 100) @@ -206,23 +210,53 @@ public void testXRefRs() throws StorageEngineException { } } + @Test + public void testCompHetQuery() throws StorageEngineException { +// Matcher matcher = allOf( +// anyOf( +// samePosition(new Variant("1:2441358:T:C")), +// samePosition(new Variant("1:2458010:G:C")), +// samePosition(new Variant("19:501725:G:A")), +// samePosition(new Variant("19:501900:C:A"))), +// withStudy(STUDY_NAME, withSampleGt("NA19685"))); + Matcher matcher = null; + testQuery(new VariantQuery().sample("NA19685:compoundheterozygous") + .study(STUDY_NAME) + .biotype("protein_coding"), + new QueryOptions(), + matcher, + false); + } + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher) throws StorageEngineException { + return testQuery(query, options, matcher, true); + } + + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher, boolean expectDBAdaptorExecutor) + throws StorageEngineException { logger.info(""); logger.info(""); logger.info("####################################################"); logger.info("########## TEST QUERY :" + query.toJson()); logger.info("####################################################"); logger.info("## Allowed VariantQueryExecutors:"); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { - if (variantQueryExecutor.canUseThisExecutor(query, options)) { + if (variantQueryExecutor.canUseThisExecutor(variantQuery, options)) { logger.info("## - " + variantQueryExecutor.getClass().getSimpleName()); } } - logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); - Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(query, options)); + VariantQueryResult expected; + if (expectDBAdaptorExecutor) { + logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); + Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(variantQuery, options)); - ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); - VariantQueryResult expected = dbQueryExecutor.get(variantQuery); + expected = dbQueryExecutor.get(variantQuery); + } else { + logger.info("## DBAdaptorVariantQueryExecutor can not be used for expected results"); + Assert.assertFalse(dbQueryExecutor.canUseThisExecutor(variantQuery, options)); + expected = null; + } VariantQueryResult unfilteredResult = null; VariantQueryResult result = null; @@ -261,21 +295,24 @@ public VariantQueryResult testQuery(Query query, QueryOptions options, QueryOptions emptyOptions = new QueryOptions(); emptyOptions.putIfNotEmpty(QueryOptions.INCLUDE, options.getString(QueryOptions.INCLUDE)); emptyOptions.putIfNotEmpty(QueryOptions.EXCLUDE, options.getString(QueryOptions.EXCLUDE)); + logger.info("## unfiltered query " + VariantQueryUtils.printQuery(emptyQuery)); + logger.info("## unfiltered options " + emptyOptions.toJson()); unfilteredResult = dbQueryExecutor.get(variantStorageEngine.parseQuery(emptyQuery, emptyOptions)); } for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { - if (variantQueryExecutor.canUseThisExecutor(query, options)) { + if (variantQueryExecutor.canUseThisExecutor(variantQuery, options)) { logger.info(""); logger.info("###################"); logger.info("### Testing " + variantQueryExecutor.getClass().getSimpleName()); result = variantQueryExecutor.get(variantQuery); logger.info("### Num results : " + result.getNumResults()); logger.info("###################"); - expected.getResults().sort(Comparator.comparing(Variant::toString)); - result.getResults().sort(Comparator.comparing(Variant::toString)); - Assert.assertEquals(expected.getResults(), result.getResults()); - + if (expected != null) { + expected.getResults().sort(Comparator.comparing(Variant::toString)); + result.getResults().sort(Comparator.comparing(Variant::toString)); + Assert.assertEquals(expected.getResults(), result.getResults()); + } assertThat(result, numResults(gt(0))); if (matcher != null) { diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java index 5839db1745b..539ea8c22d6 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverterTest.java @@ -49,7 +49,7 @@ public void test() throws Exception { expectedVariant.addStudyEntry(aux.getStudy("2")); VariantSearchModel variantSearchModel = converter.convertToStorageType(expectedVariant); - assertNull(variantSearchModel.getVariantId()); + assertNotNull(variantSearchModel.getVariantId()); assertEquals(variantId, variantSearchModel.getId()); Variant actualVariant = converter.convertToDataModelType(variantSearchModel); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java index d46cafe43df..2b1c3c3ba08 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/solr/SolrQueryParserTest.java @@ -43,8 +43,8 @@ public class SolrQueryParserTest { private String studyName = "platinum"; private String flBase = "fl=other,geneToSoAcc,traits,type,soAcc,score_*,sift,passStats_*,caddRaw,biotypes,polyphenDesc,studies,end,id,variantId," - + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,siftDesc," - + "phastCons,phylop,altStats_*,id,chromosome,start,end,type"; + + "popFreq_*,caddScaled,genes,chromosome,xrefs,start,gerp,polyphen,attr_id,siftDesc," + + "phastCons,phylop,altStats_*"; private String flDefault1 = flBase + ",fileInfo__*,qual__*,filter__*,sampleFormat__*"; private String flDefaultStudy = flBase + ",fileInfo__" + studyName + "__*,qual__" + studyName + "__*," + "filter__" + studyName + "__*,sampleFormat__" + studyName + "__*"; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java index 6fc7609b998..b1f827b2baf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManager.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.solr.common.StringUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.metadata.models.Lock; import org.slf4j.Logger; @@ -134,18 +133,17 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) // Minimum lock duration of 100ms lockDuration = Math.max(lockDuration, 100); - byte[] lockValue; - String readToken = ""; + LockToken lockToken; StopWatch stopWatch = new StopWatch(); stopWatch.start(); do { - lockValue = readLockValue(row, column); + lockToken = readLockToken(row, column); // If the lock is taken, wait - while (isLockTaken(lockValue)) { + while (lockToken.isTaken()) { Thread.sleep(100); - lockValue = readLockValue(row, column); + lockToken = readLockToken(row, column); //Check if the lock is still valid if (stopWatch.getTime() > timeout) { throw new TimeoutException("Unable to get the lock"); @@ -157,19 +155,19 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) } // Try to lock cell - if (tryToPutToken(token, lockDuration, row, column, lockValue, CURRENT)) { - readToken = parseValidLockToken(readLockValue(row, column)); + if (tryToPutToken(token, lockDuration, row, column, lockToken, CURRENT)) { + lockToken = readLockToken(row, column); } - // You win the lock if the first available lock is yours. - } while (!token.equals(readToken)); + // You win the lock if you manage to write your lock. + } while (!lockToken.equals(token)); - boolean prevTokenExpired = lockValue != null && lockValue.length > 0; + boolean prevTokenExpired = !lockToken.isEmpty() && lockToken.isExpired(); boolean slowQuery = stopWatch.getTime() > 60000; if (prevTokenExpired || slowQuery) { StringBuilder msg = new StringBuilder("Lock column '").append(Bytes.toStringBinary(column)).append("'"); if (prevTokenExpired) { - long expireDate = parseExpireDate(lockValue); + long expireDate = lockToken.getExpireDate(); msg.append(". Previous token expired ") .append(TimeUtils.durationToString(System.currentTimeMillis() - expireDate)) .append(" ago"); @@ -181,105 +179,65 @@ public Lock lock(byte[] row, byte[] column, long lockDuration, long timeout) logger.warn(msg.toString()); } - long tokenHash = token.hashCode(); - logger.debug("Won the lock with token " + token + " (" + tokenHash + ")"); - - long finalLockDuration = lockDuration; - return new Lock(threadPool, (int) (finalLockDuration / 4), tokenHash) { - @Override - public void unlock0() { - try { - HBaseLockManager.this.unlock(row, column, tokenHash); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + logger.debug("Won the lock with token " + token + " (" + token.hashCode() + ")"); - @Override - public synchronized void refresh() throws IOException { - HBaseLockManager.this.refresh(row, column, tokenHash, finalLockDuration); - } - }; + return new HBaseLock(lockDuration, token, row, column); } - /** - * Refreshes the lock. - * - * @param column Column to find the lock cell - * @param lockToken Lock token - * @param lockDuration Duration un milliseconds of the token. After this time the token is expired. - * @throws IOException if there is an error writing or reading from HBase. - */ - public void refresh(byte[] column, long lockToken, int lockDuration) throws IOException { - refresh(defaultRow, column, lockToken, lockDuration); - } - - /** * Refreshes the lock. * * @param row Row to find the lock cell * @param column Column to find the lock cell - * @param lockToken Lock token + * @param lockTokenHash Lock token * @param lockDuration Duration un milliseconds of the token. After this time the token is expired. * @throws IOException if there is an error writing or reading from HBase. */ - public void refresh(byte[] row, byte[] column, long lockToken, long lockDuration) throws IOException { + private void refresh(byte[] row, byte[] column, long lockTokenHash, long lockDuration) throws IOException { // Check token is valid - byte[] lockValue = readLockValue(row, column); - String currentLockToken = parseValidLockToken(lockValue); - if (currentLockToken == null || currentLockToken.hashCode() != lockToken) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentLockToken, lockValue); + LockToken currentLockToken = readLockToken(row, column); + if (currentLockToken.isEmpty() || currentLockToken.isExpired() || !currentLockToken.equals(lockTokenHash)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentLockToken); + } + if (currentLockToken.getRemainingTime() < lockDuration / 2) { + logger.warn("Refreshing lock with less than half of the duration remaining. Expected duration: {} Remaining time: {}ms", + lockDuration, + currentLockToken.getRemainingTime()); } - if (!tryToPutToken(currentLockToken, lockDuration, row, column, lockValue, REFRESH)) { + if (!tryToPutToken(currentLockToken.token, lockDuration, row, column, currentLockToken, REFRESH)) { // Error refreshing! - lockValue = readLockValue(row, column); - String newLockToken = parseValidLockToken(lockValue); + LockToken newLockToken = readLockToken(row, column); - logger.error("Current lock token:" + currentLockToken); - logger.error("New lock token: " + newLockToken); - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentLockToken, lockValue); + logger.error("Current lock token:" + currentLockToken.token); + logger.error("New lock token: " + newLockToken.token); + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentLockToken); } } - /** - * Releases the lock. - * - * @param column Column to find the lock cell - * @param lockToken Lock token - * @throws IOException if there is an error writing or reading from HBase. - * @throws IllegalLockStatusException if the lockToken does not match with the current lockToken - */ - public void unlock(byte[] column, long lockToken) throws IOException, IllegalLockStatusException { - unlock(defaultRow, column, lockToken); - } - /** * Releases the lock. * * @param row Row to find the lock cell * @param column Column to find the lock cell - * @param lockToken Lock token + * @param lockTokenHash Lock token * @throws IOException if there is an error writing or reading from HBase. * @throws IllegalLockStatusException if the lockToken does not match with the current lockToken */ - public void unlock(byte[] row, byte[] column, long lockToken) throws IOException, IllegalLockStatusException { - byte[] lockValue = readLockValue(row, column); + private void unlock(byte[] row, byte[] column, long lockTokenHash) throws IOException, IllegalLockStatusException { + LockToken currentToken = readLockToken(row, column); - String currentToken = parseValidLockToken(lockValue); - - if (currentToken == null || currentToken.hashCode() != lockToken) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentToken, lockValue); + if (currentToken.isEmpty() || currentToken.isExpired() || !currentToken.equals(lockTokenHash)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentToken); } - logger.debug("Unlock lock with token " + lockToken); - if (!clearLock(row, column, lockValue)) { - throw IllegalLockStatusException.inconsistentLock(row, column, lockToken, currentToken, lockValue); + logger.debug("Unlock lock with token " + lockTokenHash); + if (!clearLock(row, column, currentToken)) { + throw IllegalLockStatusException.inconsistentLock(row, column, lockTokenHash, currentToken); } } - private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[] qualifier, byte[] lockValue, String type) + private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[] qualifier, LockToken currentLock, String type) throws IOException { return hbaseManager.act(tableName, table -> { Put put = new Put(row) @@ -288,30 +246,31 @@ private Boolean tryToPutToken(String token, long lockDuration, byte[] row, byte[ + token + LOCK_EXPIRING_DATE_SEPARATOR_STR + (System.currentTimeMillis() + lockDuration))); - return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockValue, put); + return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, currentLock.lockValue, put); }); } - private boolean clearLock(byte[] row, byte[] qualifier, byte[] lockValue) throws IOException { + private boolean clearLock(byte[] row, byte[] qualifier, LockToken lockToken) throws IOException { return hbaseManager.act(tableName, table -> { Put put = new Put(row) .addColumn(columnFamily, qualifier, Bytes.toBytes("")); - return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockValue, put); + return table.checkAndPut(row, columnFamily, qualifier, CompareFilter.CompareOp.EQUAL, lockToken.lockValue, put); }); } /** - * Parse non-expired lock token. + * Parse lock token. * @param lockValue lock values - * @return Current lock token, if any + * @return Current lock token. */ - protected static String parseValidLockToken(byte[] lockValue) { + protected static LockToken parseLockToken(byte[] lockValue) { if (lockValue == null || lockValue.length == 0) { - return null; + return new LockToken(); } int idx1 = Bytes.indexOf(lockValue, LOCK_PREFIX_SEPARATOR_BYTE); int idx2 = Bytes.indexOf(lockValue, LOCK_EXPIRING_DATE_SEPARATOR_BYTE); + String type = Bytes.toString(lockValue, 0, idx1); String token = Bytes.toString(lockValue, idx1 + 1, idx2 - idx1 - 1); long expireDate; try { @@ -319,45 +278,82 @@ protected static String parseValidLockToken(byte[] lockValue) { } catch (NumberFormatException e) { // Deprecated token. Assume expired token if (Bytes.contains(lockValue, DEPRECATED_LOCK_SEPARATOR_BYTE)) { - return null; + return new LockToken(); } throw e; } + return new LockToken(lockValue, type, token, expireDate); + } + + protected static final class LockToken { + protected final byte[] lockValue; + protected final String type; + protected final String token; + protected final Long expireDate; + + private LockToken() { + this.lockValue = new byte[0]; + this.type = null; + this.token = null; + this.expireDate = null; + } + + private LockToken(byte[] lockValue, String type, String token, long expireDate) { + this.lockValue = lockValue; + this.type = type; + this.token = token; + this.expireDate = expireDate; + } + + /** + * A lock is taken if there is any lockValue, and + * the token has not expired. + * + * @return if the lock is taken + */ + public boolean isTaken() { + return token != null && !isExpired(); + } + + public boolean isExpired() { + return expireDate != null && expireDate < System.currentTimeMillis(); + } - if (isExpired(expireDate)) { - return null; - } else { + public boolean isEmpty() { + return token == null; + } + + public boolean equals(String token) { + return !isEmpty() && this.token.equals(token); + } + + public boolean equals(long tokenHash) { + return !isEmpty() && this.token.hashCode() == tokenHash; + } + + public byte[] getLockValue() { + return lockValue; + } + + public String getType() { + return type; + } + + public String getToken() { return token; } - } - protected static long parseExpireDate(byte[] lockValue) { - int idx2 = Bytes.indexOf(lockValue, LOCK_EXPIRING_DATE_SEPARATOR_BYTE); - try { - return Long.parseLong(Bytes.toString(lockValue, idx2 + 1)); - } catch (NumberFormatException e) { - // Deprecated token. Assume expired token - if (Bytes.contains(lockValue, DEPRECATED_LOCK_SEPARATOR_BYTE)) { - return -1; - } - throw e; + public Long getExpireDate() { + return expireDate; } - } - /** - * A lock is taken if there is any lockValue in the array, and - * the token has not expired. - * - * - * @param lockValue lock values - * @return if the lock is taken - */ - protected static boolean isLockTaken(byte[] lockValue) { - return parseValidLockToken(lockValue) != null; + public long getRemainingTime() { + return expireDate == null ? 0 : expireDate - System.currentTimeMillis(); + } } - private static boolean isExpired(long expireDate) { - return expireDate < System.currentTimeMillis(); + private LockToken readLockToken(byte[] row, byte[] qualifier) throws IOException { + return parseLockToken(readLockValue(row, qualifier)); } private byte[] readLockValue(byte[] row, byte[] qualifier) throws IOException { @@ -380,18 +376,22 @@ public IllegalLockStatusException(String s) { super(s); } - public static IllegalLockStatusException inconsistentLock(byte[] row, byte[] column, long lockToken, String currentLock, - byte[] lockValue) { - if (StringUtils.isEmpty(currentLock)) { - return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! " + private static IllegalLockStatusException inconsistentLock(byte[] row, byte[] column, long lockTokenHash, LockToken currentLock) { + if (currentLock.isEmpty()) { + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Empty lock. " + + "Row: '" + Bytes.toStringBinary(row) + "', " + + "column: '" + Bytes.toStringBinary(column) + "'. " + + "Lock: " + Bytes.toString(currentLock.lockValue) + "."); + } else if (currentLock.isExpired()) { + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Expired lock. " + "Row: '" + Bytes.toStringBinary(row) + "', " + "column: '" + Bytes.toStringBinary(column) + "'. " - + "Lock: " + Bytes.toString(lockValue) + "."); + + "Lock: " + Bytes.toString(currentLock.lockValue) + "."); } else { - return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! " + return new IllegalLockStatusException("Inconsistent lock status. You don't have the lock! Lock is taken. " + "Row: '" + Bytes.toStringBinary(row) + "', " + "column: '" + Bytes.toStringBinary(column) + "'. " - + lockToken + " != " + currentLock.hashCode() + " from " + Bytes.toString(lockValue)); + + lockTokenHash + " != " + currentLock.token.hashCode() + " from " + Bytes.toString(currentLock.lockValue)); } } } @@ -403,4 +403,38 @@ protected static ExecutorService buildThreadPool() { .build()); } + private final class HBaseLock extends Lock { + private final long lockDuration; + private final String token; + private final long tokenHash; + private final byte[] row; + private final byte[] column; + + private HBaseLock(long lockDuration, String token, byte[] row, byte[] column) { + super(HBaseLockManager.threadPool, (int) (lockDuration / 4), token.hashCode()); + this.lockDuration = lockDuration; + this.token = token; + this.tokenHash = token.hashCode(); + this.row = row; + this.column = column; + } + + @Override + public void unlock0() { + try { + synchronized (this) { + HBaseLockManager.this.unlock(row, column, tokenHash); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void refresh() throws IOException { + synchronized (this) { + HBaseLockManager.this.refresh(row, column, tokenHash, lockDuration); + } + } + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java index 6aa472ee735..45682df31f4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java @@ -5,7 +5,6 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; -import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; import java.util.List; @@ -18,8 +17,8 @@ public HadoopVariantQueryParser(CellBaseUtils cellBaseUtils, VariantStorageMetad } @Override - protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { - Query query = super.preProcessQuery(originalQuery, options, projection); + public Query preProcessQuery(Query originalQuery, QueryOptions options) { + Query query = super.preProcessQuery(originalQuery, options); List studyNames = metadataManager.getStudyNames(); if (isValidParam(query, STUDY) && studyNames.size() == 1) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java index c672e94fb58..ceb52e5eeaa 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java @@ -7,6 +7,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.ParsedQuery; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; @@ -43,8 +44,8 @@ public HBaseColumnIntersectVariantQueryExecutor(VariantDBAdaptor dbAdaptor, Stri } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { - + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { + VariantQuery query = variantQuery.getQuery(); if (!options.getBoolean(HBASE_COLUMN_INTERSECT, ACTIVE_BY_DEFAULT)) { // HBase column intersect not active return false; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java index ce29e3f18fe..102638aab8c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/GwasHBaseMapReduceAnalysisExecutor.java @@ -25,7 +25,7 @@ public void run() throws ToolException { List samples1 = getSampleList1(); List samples2 = getSampleList2(); - if (getConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { + if (getGwasConfiguration().getMethod().equals(GwasConfiguration.Method.CHI_SQUARE_TEST)) { addWarning("Unable to calculate chi-square test."); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java index 4dd50c9eacf..5063ca1fe4b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java @@ -9,7 +9,6 @@ import org.opencb.biodata.tools.pedigree.MendelianError; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.Trio; @@ -33,10 +32,10 @@ public SampleIndexMendelianErrorQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { - if (VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_MENDELIAN_ERROR) - || VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_DE_NOVO) - || VariantQueryUtils.isValidParam(query, VariantQueryUtils.SAMPLE_DE_NOVO_STRICT)) { + public boolean canUseThisExecutor(ParsedVariantQuery query, QueryOptions options) { + if (VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_MENDELIAN_ERROR) + || VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_DE_NOVO) + || VariantQueryUtils.isValidParam(query.getQuery(), VariantQueryUtils.SAMPLE_DE_NOVO_STRICT)) { return super.canUseThisExecutor(query, options); } else { return false; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 371d57ac4da..7b1cdc5dceb 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -86,7 +86,8 @@ public SampleIndexOnlyVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, Sam } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery variantQuery, QueryOptions options) { + VariantQuery query = variantQuery.getQuery(); if (SampleIndexQueryParser.validSampleIndexQuery(query)) { if (isFullyCoveredQuery(query, options)) { @@ -182,7 +183,8 @@ private boolean isFullyCoveredQuery(Query inputQuery, QueryOptions options) { // ParsedVariantQuery parsedVariantQuery = variantQueryProjectionParser.parseQuery(query, options, true); SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); - return isQueryCovered(query) && isIncludeCovered(sampleIndexQuery, inputQuery, options); + return isQueryCovered(sampleIndexQuery.getUncoveredQuery()) + && isIncludeCovered(sampleIndexQuery, inputQuery, options); } private boolean isQueryCovered(Query query) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java index 6175128018d..03e8a5fe880 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java @@ -58,9 +58,9 @@ public SampleIndexVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, SampleI } @Override - public boolean canUseThisExecutor(Query query, QueryOptions options) { + public boolean canUseThisExecutor(ParsedVariantQuery query, QueryOptions options) { if (options.getBoolean(SAMPLE_INDEX_INTERSECT, true)) { - return SampleIndexQueryParser.validSampleIndexQuery(query); + return SampleIndexQueryParser.validSampleIndexQuery(query.getQuery()); } return false; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java index af632d69436..ad09896e627 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java @@ -112,10 +112,9 @@ protected void parseAndValidateParameters() throws IOException { } else { trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getMother())); } - int childId = metadataManager.getSampleIdOrFail(getStudyId(), trio.getChild()); - trioList.add(childId); - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), childId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), trio.getChild()); + trioList.add(sampleMetadata.getId()); if (!overwrite && sampleMetadata.getFamilyIndexStatus(sampleIndexVersion) == TaskMetadata.Status.READY) { LOGGER.info("Skip sample " + sampleMetadata.getName() + ". Already precomputed!"); } else { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 027d241282c..d4605b918f4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -1286,6 +1286,7 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema intergenic == Boolean.FALSE && (!ctFilterCoveredBySummary || (!ctBtCombinationCoveredBySummary && combination.isBiotype()) || combination.isFlag()); + if (useCtIndexFilter) { ctCovered = completeIndex; consequenceTypeFilter = schema.getCtIndex().getField().buildFilter(new OpValue<>("=", soNames)); @@ -1515,6 +1516,8 @@ protected SampleAnnotationIndexQuery parseAnnotationIndexQuery(SampleIndexSchema if (intergenic == null || intergenic) { // If intergenic is undefined, or true, CT and BT filters can not be used. + biotypeFilter = schema.getBiotypeIndex().getField().noOpFilter(); + consequenceTypeFilter = schema.getCtIndex().getField().noOpFilter(); if (!biotypeFilter.isNoOp()) { throw new IllegalStateException("Unexpected BT filter for intergenic=" + intergenic); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java index ef2aecb803b..a087420f425 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchemaFactory.java @@ -78,8 +78,7 @@ public Collection getSampleIndexConfigurationVersions(int studyId, Coll private Collection getSampleIndexConfigurationVersions(int studyId, Object sample, boolean withAnnotation, boolean withFamilyIndex) { - int sampleId = metadataManager.getSampleIdOrFail(studyId, sample); - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sample); Collection versions = sampleMetadata.getSampleIndexVersions(); if (withAnnotation) { versions = CollectionUtils.intersection( diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java index 3f6a23c5abf..fbdfdf920ec 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/AbstractHBaseDBAdaptor.java @@ -279,14 +279,5 @@ protected Lock lockToken(byte[] rowKey, byte[] lockName, long lockDuration, long } } - protected void unLock(byte[] rowKey, byte[] lockName, long token) { - try { - this.lock.unlock(rowKey, lockName, token); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java index 85a1d0cec0c..6ddae38be65 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseProjectMetadataDBAdaptor.java @@ -49,20 +49,14 @@ public HBaseProjectMetadataDBAdaptor(HBaseManager hBaseManager, String metaTable @Override public Lock lockProject(long lockDuration, long timeout, String lockName) - throws InterruptedException, TimeoutException, StorageEngineException { + throws StorageEngineException { try { ensureTableExists(); return lock.lock(getProjectRowKey(), getLockColumn(lockName), lockDuration, timeout); - } catch (IOException e) { - throw new StorageEngineException("Error locking project in HBase", e); - } - } - - @Override - public void unLockProject(long lockId) throws StorageEngineException { - try { - lock.unlock(getProjectRowKey(), getLockColumn(), lockId); - } catch (IOException e) { + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new StorageEngineException("Unable to lock the Project", e); + } catch (IOException | TimeoutException e) { throw new StorageEngineException("Error locking project in HBase", e); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java index 17c61739496..39c63923c02 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopVariantSearchDataWriter.java @@ -68,7 +68,7 @@ protected void add(List batch) throws Exception { return PhoenixHelper.toBytes(studyIds, PIntegerArray.INSTANCE); }); - byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("id").toString())); + byte[] row = VariantPhoenixKeyFactory.generateVariantRowKey(new Variant(document.getFieldValue("attr_id").toString())); variantRows.add(row); mutations.add(new Put(row) .addColumn(family, VariantPhoenixSchema.VariantColumn.INDEX_STUDIES.bytes(), bytes)); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java index 0f421081ff6..5943a43519f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/utils/HBaseLockManagerTest.java @@ -214,23 +214,47 @@ public void testLockRefreshExpiredRefresh() throws Exception { @Test public void testGetCurrent() { long e = System.currentTimeMillis() + 1000; - String s; + HBaseLockManager.LockToken s; + + // null token + s = HBaseLockManager.parseLockToken(null); + assertTrue(s.isEmpty()); + assertFalse(s.isTaken()); + assertEquals(null, s.getType()); + assertArrayEquals(new byte[0], s.getLockValue()); + + // Empty token + s = HBaseLockManager.parseLockToken(Bytes.toBytes("")); + assertTrue(s.isEmpty()); + assertFalse(s.isTaken()); + assertEquals(null, s.getType()); + assertArrayEquals(new byte[0], s.getLockValue()); // Expired current token - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("CURRENT-abc:123")); - assertNull(s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("CURRENT-abc:123")); + assertFalse(s.isEmpty()); + assertEquals("CURRENT", s.getType()); // Valid current token - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("CURRENT-abc:" + e)); - assertEquals("abc", s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("CURRENT-abc:" + e)); + assertEquals("abc", s.token); + assertEquals("CURRENT", s.getType()); + assertFalse(s.isExpired()); + assertTrue(s.isTaken()); // Current expired, first refresh valid - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("REFRESH-abc:" + e)); - assertEquals("abc", s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("REFRESH-abc:" + e)); + assertEquals("abc", s.token); + assertEquals("REFRESH", s.getType()); + assertFalse(s.isExpired()); + assertTrue(s.isTaken()); // Expired refresh - s = HBaseLockManager.parseValidLockToken(Bytes.toBytes("REFRESH-abc:200")); - assertNull(s); + s = HBaseLockManager.parseLockToken(Bytes.toBytes("REFRESH-abc:200")); + assertEquals("abc", s.token); + assertEquals("REFRESH", s.getType()); + assertTrue(s.isExpired()); + assertFalse(s.isTaken()); } } \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java index aea720d356a..b613df935ba 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java @@ -19,7 +19,7 @@ public class HadoopVariantStorageEngineBNDTest extends VariantStorageEngineBNDTe @Override protected void loadFiles() throws Exception { super.loadFiles(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); + VariantHbaseTestUtils.printVariants(((HadoopVariantStorageEngine) variantStorageEngine).getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java index fbcbd773477..26bcb49bb0f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java @@ -1,8 +1,6 @@ package org.opencb.opencga.storage.hadoop.variant; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; @@ -12,13 +10,15 @@ import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import static org.opencb.opencga.core.api.ParamConstants.OVERWRITE; /** @@ -33,9 +33,46 @@ public class HadoopVariantStorageEngineSVTest extends VariantStorageEngineSVTest public static HadoopExternalResource externalResource = new HadoopExternalResource(); private Logger logger = LoggerFactory.getLogger(getClass()); + public static VariantSolrExternalResource solr = new VariantSolrExternalResource(); + + @BeforeClass + public static void beforeClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.before(); + solr.configure(externalResource.getVariantStorageEngine()); + System.out.println("Start embedded solr"); + } else { + System.out.println("Skip embedded solr tests"); + } + } + + @AfterClass + public static void afterClass() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.after(); + } + } + + @Override + public void before() throws Exception { + super.before(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } + } + @Override protected void loadFiles() throws Exception { + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + solr.configure(variantStorageEngine); + } super.loadFiles(); + if (HBaseCompat.getInstance().isSolrTestingAvailable()) { + variantStorageEngine.secondaryIndex(); + assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } else { + assertFalse(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java index d609e769f0d..8cead56116a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorTest.java @@ -119,6 +119,7 @@ public void before() throws Exception { e.printStackTrace(); } } + variantStorageEngine.getOptions().append(VariantStorageOptions.ASSEMBLY.key(), "GRCH38"); cellBaseUtils = variantStorageEngine.getCellBaseUtils(); expectedConnections = GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS.getMetric().getValue(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java index 3d9db73719e..63f579cb99e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java @@ -13,6 +13,7 @@ import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; import org.opencb.biodata.models.variant.protobuf.VcfSliceProtos; import org.opencb.biodata.tools.variant.converters.proto.VariantToVcfSliceConverter; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.testclassification.duration.ShortTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -44,6 +45,7 @@ public class FillGapsTaskTest { public void setUp() throws Exception { DummyVariantStorageMetadataDBAdaptorFactory.clear(); metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory()); + metadataManager.getAndUpdateProjectMetadata(new ObjectMap()); studyMetadata = metadataManager.createStudy("S"); metadataManager.updateStudyMetadata("S", sm -> { sm.getAttributes().put(VariantStorageOptions.EXTRA_FORMAT_FIELDS.key(), "DP"); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index f67d5734d7c..a6aeba91145 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -63,13 +63,13 @@ public class FamilyIndexTest extends VariantStorageBaseTest implements HadoopVar @Before public void before() throws Exception { + HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); if (!loaded) { - HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); - variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); - variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.2"); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease("3"); - variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); - variantStorageEngine.reloadCellbaseConfiguration(); URI outputUri = newOutputUri(); ObjectMap params = new ObjectMap(VariantStorageOptions.ANNOTATE.key(), false) @@ -91,7 +91,7 @@ public void before() throws Exception { variantStorageEngine.annotate(outputUri, new ObjectMap()); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri(getTestName().getMethodName())); mendelianErrorVariants = new HashSet<>(); deNovoVariants = new HashSet<>(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java index 99cf76e71cc..e9c46d5214f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java @@ -1387,7 +1387,6 @@ public void parseIntergenicTest() { checkIntergenic(null, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT)); checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), VariantAnnotationConstants.REGULATORY_REGION_VARIANT) .append(ANNOT_BIOTYPE.key(), "protein_coding")); - // Nonsense combination checkIntergenic(false, new Query(ANNOT_CONSEQUENCE_TYPE.key(), "intergenic_variant").append(ANNOT_BIOTYPE.key(), "protein_coding")); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 55d9d98a0f8..98062c27b8f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -1067,7 +1067,11 @@ public void testFamilyIndexQueryCount() { .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); - System.out.println(result.getResults().stream().map(Variant::getAnnotation).flatMap(v -> v.getConsequenceTypes().stream()).map(ConsequenceType::getGeneName).collect(Collectors.toSet())); + System.out.println(result.getResults().stream() + .map(Variant::getAnnotation) + .flatMap(v -> v.getConsequenceTypes().stream()) + .map(ConsequenceType::getGeneName) + .collect(Collectors.toSet())); result = variantStorageEngine.get( new Query() @@ -1081,7 +1085,11 @@ public void testFamilyIndexQueryCount() { .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); - System.out.println(result.getResults().stream().map(Variant::getAnnotation).flatMap(v -> v.getConsequenceTypes().stream()).map(ConsequenceType::getGeneName).collect(Collectors.toSet())); + System.out.println(result.getResults().stream() + .map(Variant::getAnnotation) + .flatMap(v -> v.getConsequenceTypes().stream()) + .map(ConsequenceType::getGeneName) + .collect(Collectors.toSet())); } @Test diff --git a/pom.xml b/pom.xml index 9dc0893e133..5076fd8b8ec 100644 --- a/pom.xml +++ b/pom.xml @@ -43,13 +43,13 @@ - 3.2.1_dev 3.2.1_dev 6.1.1-SNAPSHOT 3.2.1-SNAPSHOT 5.2.1-SNAPSHOT 3.2.1-SNAPSHOT + 0.2.0 2.14.3 @@ -1368,7 +1368,7 @@ opencga LOCAL - 5 + 5 https://ws.opencb.org/opencga-prod