diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml index 3f64fd392de..5a10146128a 100644 --- a/.github/workflows/manual-deploy-ext-tools.yml +++ b/.github/workflows/manual-deploy-ext-tools.yml @@ -47,4 +47,5 @@ jobs: needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }} - secrets: inherit \ No newline at end of file + secrets: inherit + diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml index c3ca5077a33..faedd27bebc 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merge.yml @@ -21,3 +21,4 @@ jobs: with: cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }} secrets: inherit + diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index fe6350d59ca..b2b5105877b 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -86,3 +86,4 @@ jobs: commit: '${{ github.sha }}' fail_on_test_failures: true + diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java index 34b337731a5..069d7ae46a7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java @@ -202,12 +202,12 @@ protected void run() throws Exception { if (family == null || StringUtils.isEmpty(family.getId())) { continue; } - List> trios = variantStorageManager.getTriosFromFamily(getStudy(), family, true, getToken()); - for (List trio : trios) { - String child = trio.get(2); + List trios = variantStorageManager.getTriosFromFamily(getStudy(), family, true, getToken()); + for (Trio trio : trios) { + String child = trio.getChild(); if (analysisParams.getSample().contains(child)) { - String father = trio.get(0); - String mother = trio.get(1); + String father = trio.getFather(); + String mother = trio.getMother(); triosMap.put(child, new Trio(family.getId(), "-".equals(father) ? null : father, "-".equals(mother) ? null : mother, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java index 2d08fb77566..7a588c900f5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java @@ -1299,10 +1299,10 @@ public Integer getReleaseFilter(Query query, String sessionId) throws CatalogExc return release; } - public List> getTriosFromFamily( + public List getTriosFromFamily( String studyFqn, Family family, VariantStorageMetadataManager metadataManager, boolean skipIncompleteFamily, String sessionId) throws StorageEngineException, CatalogException { - List> trios = getTrios(studyFqn, metadataManager, family.getMembers(), sessionId); + List trios = getTrios(studyFqn, metadataManager, family.getMembers(), sessionId); if (trios.size() == 0) { if (skipIncompleteFamily) { logger.debug("Skip family '" + family.getId() + "'. "); @@ -1313,7 +1313,7 @@ public List> getTriosFromFamily( return trios; } - public List> getTriosFromSamples( + public List getTriosFromSamples( String studyFqn, VariantStorageMetadataManager metadataManager, Collection sampleIds, String token) throws CatalogException { OpenCGAResult individualResult = catalogManager.getIndividualManager() @@ -1330,12 +1330,12 @@ public List> getTriosFromSamples( return getTrios(studyFqn, metadataManager, individualResult.getResults(), token); } - public List> getTrios( + public List getTrios( String studyFqn, VariantStorageMetadataManager metadataManager, List membersList, String sessionId) throws CatalogException { int studyId = metadataManager.getStudyId(studyFqn); Map membersMap = membersList.stream().collect(Collectors.toMap(Individual::getUid, i -> i)); - List> trios = new LinkedList<>(); + List trios = new LinkedList<>(); for (Individual individual : membersList) { String fatherSample = null; String motherSample = null; @@ -1402,10 +1402,7 @@ public List> getTrios( // Allow one missing parent if (childSample != null && (fatherSample != null || motherSample != null)) { - trios.add(Arrays.asList( - fatherSample == null ? "-" : fatherSample, - motherSample == null ? "-" : motherSample, - childSample)); + trios.add(new Trio(fatherSample, motherSample, childSample)); } } return trios; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 22b337e9621..bf0db6f34eb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -63,6 +63,7 @@ import org.opencb.opencga.core.models.operations.variant.*; import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Study; @@ -76,10 +77,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantMetadataFactory; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.BeaconResponse; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -397,7 +395,7 @@ public void sampleIndexAnnotate(String study, List samples, ObjectMap pa }); } - public DataResult> familyIndexUpdate(String study, + public DataResult familyIndexUpdate(String study, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { @@ -405,11 +403,11 @@ public DataResult> familyIndexUpdate(String study, }); } - public DataResult> familyIndex(String study, List familiesStr, boolean skipIncompleteFamilies, + public DataResult familyIndex(String study, List familiesStr, boolean skipIncompleteFamilies, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { - List> trios = new LinkedList<>(); + List trios = new LinkedList<>(); List events = new LinkedList<>(); VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); VariantCatalogQueryUtils catalogUtils = new VariantCatalogQueryUtils(catalogManager); @@ -425,9 +423,9 @@ public DataResult> familyIndex(String study, List familiesS trios.addAll(catalogUtils.getTriosFromFamily(study, family, metadataManager, skipIncompleteFamilies, token)); } } - DataResult> dataResult = engine.familyIndex(study, trios, params); + DataResult dataResult = engine.familyIndex(study, trios, params); getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream() - .flatMap(Collection::stream) + .flatMap(t->t.toList().stream()) .collect(Collectors.toList()), token); return dataResult; }); @@ -439,7 +437,7 @@ private CatalogStorageMetadataSynchronizer getSynchronizer(VariantStorageEngine return synchronizer; } - public DataResult> familyIndexBySamples(String study, Collection samples, ObjectMap params, String token) + public DataResult familyIndexBySamples(String study, Collection samples, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { Collection thisSamples = samples; @@ -447,17 +445,16 @@ public DataResult> familyIndexBySamples(String study, Collection> trios = catalogUtils.getTriosFromSamples(study, engine.getMetadataManager(), thisSamples, token); - - DataResult> dataResult = engine.familyIndex(study, trios, params); + List trios = catalogUtils.getTriosFromSamples(study, engine.getMetadataManager(), thisSamples, token); + DataResult dataResult = engine.familyIndex(study, trios, params); getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream() - .flatMap(Collection::stream) + .flatMap(t -> t.toList().stream()) .collect(Collectors.toList()), token); return dataResult; }); } - public List> getTriosFromFamily(String study, Family family, boolean skipIncompleteFamilies, String token) + public List getTriosFromFamily(String study, Family family, boolean skipIncompleteFamilies, String token) throws CatalogException, StorageEngineException { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(study, token); return catalogUtils.getTriosFromFamily(study, family, variantStorageEngine.getMetadataManager(), skipIncompleteFamilies, token); @@ -1015,11 +1012,21 @@ protected VariantStorageEngine getVariantStorageEngineByProject(String project, return variantStorageEngine; } - private void setCellbaseConfiguration(VariantStorageEngine engine, String project, String token) + private void setCellbaseConfiguration(VariantStorageEngine engine, String projectId, String token) throws CatalogException { - CellBaseConfiguration cellbase = catalogManager.getProjectManager() - .get(project, new QueryOptions(INCLUDE, ProjectDBAdaptor.QueryParams.CELLBASE.key()), token) - .first().getCellbase(); + Project project = catalogManager.getProjectManager() + .get(projectId, new QueryOptions(INCLUDE, Arrays.asList( + ProjectDBAdaptor.QueryParams.CELLBASE.key(), + ProjectDBAdaptor.QueryParams.ORGANISM.key())), token) + .first(); + CellBaseConfiguration cellbase = project.getCellbase(); + ProjectOrganism organism = project.getOrganism(); + if (organism == null) { + throw new CatalogException("Missing organism in project '" + project.getFqn()+ "'"); + } else { + engine.getOptions().put(VariantStorageOptions.SPECIES.key(), organism.getScientificName()); + engine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), organism.getAssembly()); + } if (cellbase != null) { // To ensure that we use the project API key before using the storage API key if (StringUtils.isEmpty(cellbase.getApiKey()) && storageConfiguration.getCellbase() != null) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java index 48ff94151d7..94bfada60d5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java @@ -100,6 +100,7 @@ public class VariantFileIndexerOperationManager extends OperationManager { private List filesToIndex; private CatalogStorageMetadataSynchronizer synchronizer; private boolean fullSynchronize = false; + private boolean force; public VariantFileIndexerOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) { super(variantStorageManager, engine); @@ -138,6 +139,7 @@ private void check(String study, ObjectMap params, String token) throws Exceptio } resume = params.getBoolean(VariantStorageOptions.RESUME.key()); skipIndexedFiles = params.getBoolean(SKIP_INDEXED_FILES); + force = params.getBoolean(VariantStorageOptions.FORCE.key()); // Obtain the type of analysis (transform, load or index) step = getType(load, transform); @@ -589,6 +591,7 @@ private List filterTransformFiles(List fileList, boolean resume) thr break; case VariantIndexStatus.INDEXING: case VariantIndexStatus.TRANSFORMING: + case VariantIndexStatus.LOADING: if (resume) { filteredFiles.add(file); } else { @@ -603,14 +606,17 @@ private List filterTransformFiles(List fileList, boolean resume) thr } break; case VariantIndexStatus.TRANSFORMED: - case VariantIndexStatus.LOADING: case VariantIndexStatus.READY: default: String msg = "We can only " + step + " VCF files not transformed, the status is " + indexStatus; - if (skipIndexedFiles) { - logger.warn(msg); + if (force) { + filteredFiles.add(file); } else { - throw new StorageEngineException(msg); + if (skipIndexedFiles) { + logger.warn(msg); + } else { + throw new StorageEngineException(msg); + } } break; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java index 4f6f7bb6452..3d97af2c238 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java @@ -508,6 +508,16 @@ protected boolean synchronizeFiles(StudyMetadata study, List files, String } fileSamplesMap.put(fileMetadata.getName(), samples); allSamples.addAll(fileMetadata.getSamples()); + if (samples.size() > 100) { + // Try to reuse value. + // If the file holds more than 100 samples, it's most likely this same set of samples is already present + for (Set value : fileSamplesMap.values()) { + if (value.equals(samples)) { + fileSamplesMap.put(fileMetadata.getName(), value); + break; + } + } + } } if (!indexedFilesFromStorage.isEmpty()) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index c542521329a..042784b844a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -592,7 +592,9 @@ private void computeSignatureFitting() throws IOException, ToolException, Catalo StringBuilder scriptParams = new StringBuilder("R CMD Rscript --vanilla ") .append("/opt/opencga/signature.tools.lib/scripts/signatureFit") .append(" --catalogues=/data/input/").append(cataloguesFile.getName()) - .append(" --outdir=/data/output"); + .append(" --outdir=/data/output") + .append(" --commonsigtier=T2"); + if (StringUtils.isNotEmpty(getFitMethod())) { scriptParams.append(" --fitmethod=").append(getFitMethod()); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java index ed774beca11..c7258c79f74 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java @@ -24,6 +24,7 @@ import org.opencb.opencga.core.models.operations.variant.VariantFamilyIndexParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; +import org.opencb.opencga.storage.core.metadata.models.Trio; import java.util.Collections; import java.util.List; @@ -67,7 +68,7 @@ protected void check() throws Exception { @Override protected void run() throws Exception { step(() -> { - DataResult> trios; + DataResult trios; if (variantFamilyIndexParams.isUpdateIndex()) { trios = variantStorageManager.familyIndexUpdate(study, params, token); } else { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java index c4ba731b647..cce9fc91e8a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java @@ -90,6 +90,7 @@ protected void check() throws Exception { params.put(VariantStorageOptions.ANNOTATE.key(), indexParams.isAnnotate()); params.putIfNotEmpty(VariantStorageOptions.ANNOTATOR.key(), indexParams.getAnnotator()); params.put(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), indexParams.isOverwriteAnnotations()); + params.put(VariantStorageOptions.FORCE.key(), indexParams.isForceReload()); params.put(VariantStorageOptions.RESUME.key(), indexParams.isResume()); params.put(VariantStorageOptions.NORMALIZATION_SKIP.key(), indexParams.getNormalizationSkip()); params.putIfNotEmpty(VariantStorageOptions.NORMALIZATION_REFERENCE_GENOME.key(), indexParams.getReferenceGenome()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java index 1954d7eb790..d4dd7ab8a89 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java @@ -24,6 +24,7 @@ import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; +import org.opencb.opencga.storage.core.metadata.models.Trio; import java.util.ArrayList; import java.util.List; @@ -80,7 +81,7 @@ protected void run() throws Exception { } if (sampleIndexParams.isFamilyIndex()) { step("familyIndex", () -> { - DataResult> result = variantStorageManager.familyIndexBySamples(study, sampleIndexParams.getSample(), params, + DataResult result = variantStorageManager.familyIndexBySamples(study, sampleIndexParams.getSample(), params, getToken()); if (result.getEvents() != null) { for (Event event : result.getEvents()) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java index b855a379b08..71af17f701e 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java @@ -18,6 +18,7 @@ import org.junit.AfterClass; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -95,6 +96,7 @@ public AlignmentAnalysisTest(String storageEngine) { private CatalogManager catalogManager; private VariantStorageManager variantStorageManager; + @ClassRule public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource(); public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java index f45e8e77ec2..e528a8156bc 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java @@ -6,16 +6,20 @@ import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.core.NodeConfig; import org.junit.rules.ExternalResource; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; import org.opencb.commons.datastore.solr.SolrManager; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; public class RgaSolrExtenalResource extends ExternalResource { @@ -23,6 +27,7 @@ public class RgaSolrExtenalResource extends ExternalResource { private SolrClient solrClient; protected boolean embeded = true; + private Class testClass; public RgaSolrExtenalResource() { this(true); @@ -32,16 +37,23 @@ public RgaSolrExtenalResource(boolean embeded) { this.embeded = embeded; } + @Override + public Statement apply(Statement base, Description description) { + testClass = description.getTestClass(); + return super.apply(base, description); + } + @Override protected void before() throws Throwable { super.before(); - Path rootDir = getTmpRootDir(); + Path rootDir = Paths.get("target/test-data", "junit-rga-solr-" + TimeUtils.getTimeMillis()); + Files.createDirectories(rootDir); String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.getInstance().getBuildVersion(); String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.getInstance().getBuildVersion(); - copyConfigSetConfiguration(mainConfigSet, "managed-schema"); - copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema"); + copyConfigSetConfiguration(mainConfigSet, "managed-schema", rootDir); + copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema", rootDir); String solrHome = rootDir.resolve("solr").toString(); @@ -78,15 +90,15 @@ protected void after() { } } - private void copyConfigSetConfiguration(String configSet, String managedSchemaFile) throws IOException { + private void copyConfigSetConfiguration(String configSet, String managedSchemaFile, Path rootDir) throws IOException { // Copy configuration - getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml"); - getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema"); - getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json"); - getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt"); - getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt"); - getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt"); - getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt"); + getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml", rootDir); + getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema", rootDir); + getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json", rootDir); + getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt", rootDir); + getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt", rootDir); } public RgaEngine configure(StorageConfiguration storageConfiguration) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index b089cc79869..621239dcbf1 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -22,6 +22,7 @@ import org.hamcrest.CoreMatchers; import org.junit.*; import org.junit.experimental.categories.Category; +import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.opencb.biodata.models.clinical.Disorder; @@ -59,6 +60,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; @@ -73,6 +75,8 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.Location; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleReferenceParam; @@ -84,6 +88,7 @@ import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.storage.core.StorageEngineFactory; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; @@ -130,6 +135,8 @@ public class VariantAnalysisTest { private static String cancer_sample = "AR2.10039966-01T"; private static String germline_sample = "AR2.10039966-01G"; + @Rule + public ExpectedException thrown = ExpectedException.none(); @Parameterized.Parameters(name = "{0}") public static Object[][] parameters() { @@ -1100,6 +1107,16 @@ public void testClinicalAnalysisLoading() throws IOException, ToolException, Cat Assert.assertEquals(ca2Id, clinicalAnalysis.getId()); } + @Test + public void testCellbaseConfigure() throws Exception { + String project = "Project_test_cellbase_configure"; + catalogManager.getProjectManager().create(new ProjectCreateParams(project, project, "", "", "", new ProjectOrganism("hsapiens", "grch38"), null, null), QueryOptions.empty(), token); + + thrown.expect(StorageEngineException.class); + thrown.expectMessage("The storage engine is in mode=READ_ONLY"); + variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration("https://uk.ws.zettagenomics.com/cellbase/", "v5.2", "1", ""), false, null, token); + } + public void checkExecutionResult(ExecutionResult er) { checkExecutionResult(er, true); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java index 58aa48f7828..b38101ef933 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java @@ -60,6 +60,7 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; @@ -868,9 +869,9 @@ public void getTriosFromFamily() throws Exception { sampleMetadata -> sampleMetadata.setIndexStatus(TaskMetadata.Status.READY)); } - List> trios = queryUtils.getTriosFromFamily("s1", f1, metadataManager, true, sessionId); + List trios = queryUtils.getTriosFromFamily("s1", f1, metadataManager, true, sessionId); // System.out.println("trios = " + trios); - assertEquals(Arrays.asList(Arrays.asList("sample1", "sample2", "sample3"), Arrays.asList("sample1", "sample2", "sample4")), trios); + assertEquals(Arrays.asList(new Trio("sample1", "sample2", "sample3"), new Trio("sample1", "sample2", "sample4")), trios); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 38ef6f98a21..a5ec92aafec 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -36,20 +36,22 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.YesNoAuto; +import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortCreateParams; import org.opencb.opencga.core.models.common.IndexStatus; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualInternal; -import org.opencb.opencga.core.models.individual.Location; +import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.operations.variant.VariantAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondaryAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.variant.VariantIndexParams; @@ -57,7 +59,9 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.core.tools.result.ExecutionResult; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageEngine; @@ -75,6 +79,8 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; @RunWith(Parameterized.class) @Category(LongTests.class) @@ -148,6 +154,16 @@ public void setUp() throws Throwable { @After public void tearDown() { if (hadoopExternalResource != null) { + + try { + VariantStorageEngine engine = opencga.getStorageEngineFactory().getVariantStorageEngine(storageEngine, DB_NAME); + if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { + VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants_AFTER")).toUri()); + } + } catch (Exception ignore) { + ignore.printStackTrace(); + } + hadoopExternalResource.after(); hadoopExternalResource = null; } @@ -246,7 +262,7 @@ private void loadDataset() throws Throwable { Collections.emptyList(), false, 0, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), Collections.emptyMap()).setFather(individuals.get(0)).setMother(individuals.get(1)), Collections.singletonList(daughter), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first()); catalogManager.getFamilyManager().create( STUDY, - new Family("f1", "f1", Collections.singletonList(phenotype), Collections.singletonList(disorder), null, null, 3, null, null), + new Family("f1", "f1", Collections.singletonList(phenotype), Collections.singletonList(disorder), null, null, 4, null, null), individuals.stream().map(Individual::getId).collect(Collectors.toList()), new QueryOptions(), token); @@ -277,6 +293,28 @@ public void setUpCatalogManager() throws Exception { } + @Test + public void testVariantFileReload() throws Exception { + try { + toolRunner.execute(VariantIndexOperationTool.class, STUDY, + new VariantIndexParams() + .setForceReload(false) + .setFile(file.getId()), + Paths.get(opencga.createTmpOutdir()), "index_reload", token); + fail("Should have thrown an exception"); + } catch (ToolException e) { + assertEquals(StorageEngineException.class, e.getCause().getClass()); + assertEquals("We can only INDEX VCF files not transformed, the status is READY", e.getCause().getMessage()); + } + + toolRunner.execute(VariantIndexOperationTool.class, STUDY, + new VariantIndexParams() + .setForceReload(true) + .setFile(file.getId()), + Paths.get(opencga.createTmpOutdir()), "index_reload", token); + + } + @Test public void testVariantSecondaryAnnotationIndex() throws Exception { @@ -375,6 +413,50 @@ public void testVariantSecondarySampleIndex() throws Exception { } } + @Test + public void testVariantSecondarySampleIndexPartialFamily() throws Exception { + Assume.assumeThat(storageEngine, anyOf( +// is(DummyVariantStorageEngine.STORAGE_ENGINE_ID), + is(HadoopVariantStorageEngine.STORAGE_ENGINE_ID) + )); + for (String sample : samples) { + SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); + assertEquals(sample, IndexStatus.READY, sampleIndex.getStatus().getId()); + assertEquals(sample, IndexStatus.NONE, sampleIndex.getFamilyStatus().getId()); + assertEquals(sample, 1, sampleIndex.getVersion().intValue()); + } + + Phenotype phenotype = new Phenotype("phenotype", "phenotype", ""); + Disorder disorder = new Disorder("disorder", "disorder", "", "", Collections.singletonList(phenotype), Collections.emptyMap()); + + catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList("f1"), null, token); + catalogManager.getIndividualManager().update(STUDY, daughter, new IndividualUpdateParams() + .setMother(new IndividualReferenceParam(null, null)), null, token); + catalogManager.getFamilyManager().create( + STUDY, + new Family("f2", "f2", Collections.singletonList(phenotype), Collections.singletonList(disorder), null, null, 2, null, null), + Arrays.asList(father, daughter), new QueryOptions(), + token); + + // Run family index. The family index status should be READY on offspring + toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, STUDY, + new VariantSecondarySampleIndexParams() + .setFamilyIndex(true) + .setSample(Arrays.asList(daughter)), + Paths.get(opencga.createTmpOutdir()), "index", token); + + for (String sample : samples) { + SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); + assertEquals(sample, IndexStatus.READY, sampleIndex.getStatus().getId()); + if (sample.equals(daughter)) { + assertEquals(sample, IndexStatus.READY, sampleIndex.getFamilyStatus().getId()); + } else { + assertEquals(sample, IndexStatus.NONE, sampleIndex.getFamilyStatus().getId()); + } + assertEquals(sample, 1, sampleIndex.getVersion().intValue()); + } + } + @Test public void testGwasIndex() throws Exception { // Variant scores can not be loaded in mongodb nor dummy @@ -408,6 +490,28 @@ public void testGwasIndex() throws Exception { }); } + @Test + public void testCellbaseConfigure() throws Exception { + String project = "Project_test_cellbase_configure"; + catalogManager.getProjectManager().create(new ProjectCreateParams(project, project, "", "", "", new ProjectOrganism("hsapiens", "GRCh38"), null, null), QueryOptions.empty(), token); + + CellBaseUtils cellBaseUtils = variantStorageManager.getVariantStorageEngineByProject(project, null, token).getCellBaseUtils(); + assertEquals(ParamConstants.CELLBASE_URL, cellBaseUtils.getURL()); + assertEquals(ParamConstants.CELLBASE_VERSION, cellBaseUtils.getVersion()); + assertEquals("hsapiens", cellBaseUtils.getSpecies()); + assertEquals("GRCh38", cellBaseUtils.getAssembly()); + + String newCellbase = "https://uk.ws.zettagenomics.com/cellbase/"; + String newCellbaseVersion = "v5.2"; + + assertNotEquals(newCellbase, cellBaseUtils.getURL()); + assertNotEquals(newCellbaseVersion, cellBaseUtils.getVersion()); + + variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, "1", ""), false, null, token); + CellBaseConfiguration cellbaseConfiguration = catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getCellbase(); +// assertTrue(family.getPedigreeGraph() != null); + } + public void checkExecutionResult(ExecutionResult er) { checkExecutionResult(er, true); } diff --git a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile index 5558259e3f5..ff1298c64d0 100644 --- a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile +++ b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile @@ -17,14 +17,15 @@ RUN apt-get update -y && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt ## Installation dependencies using R install.packages() is slower than apt-get but final size is 400GB smaller. R -e "install.packages(c('BiocManager', 'RCircos', 'nnls', 'ggplot2', 'jsonlite', 'optparse', 'knitr', 'configr', 'dplyr', 'rmarkdown', 'tidyr', 'httr', 'kinship2', 'limSolve'))" && \ R -e "BiocManager::install('BiocStyle')" && \ + R -e "BiocManager::install('BSgenome.Hsapiens.UCSC.hg38')" && \ ## signature.tools.lib installation \ - R -e 'install.packages(c("devtools", "getopt"), repos="https://www.stats.bris.ac.uk/R/")' && \ + R -e 'install.packages(c("devtools", "getopt"), repos=c("http://cran.rstudio.com/", "https://www.stats.bris.ac.uk/R/"))' && \ git clone https://github.com/Nik-Zainal-Group/signature.tools.lib.git /opt/opencga/signature.tools.lib WORKDIR /opt/opencga/signature.tools.lib RUN git fetch origin --tags && \ - git checkout tags/v2.4.1 && \ + git checkout tags/v2.4.2 && \ sed -i '/Mmusculus/d' DESCRIPTION && \ sed -i '/Cfamiliaris/d' DESCRIPTION && \ sed -i '/1000genomes/d' DESCRIPTION && \ diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index 4bb045cb874..7d3f514a745 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -380,6 +380,7 @@ private void index() throws ToolException { cliOptions.genericVariantIndexOptions.family, cliOptions.genericVariantIndexOptions.somatic, cliOptions.genericVariantIndexOptions.load, + cliOptions.genericVariantIndexOptions.forceReload, cliOptions.genericVariantIndexOptions.loadSplitData, cliOptions.genericVariantIndexOptions.loadMultiFileData, cliOptions.genericVariantIndexOptions.loadSampleIndex, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index c820242e234..4c794443f98 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD * Copyright 2015-2023-09-21 OpenCB +======= +* Copyright 2015-2023-10-20 OpenCB +>>>>>>> develop * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index 8cd4a5a92c5..046a7adbb9e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD * Copyright 2015-2023-09-21 OpenCB +======= +* Copyright 2015-2023-10-20 OpenCB +>>>>>>> develop * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java index 719d9999d96..39baa4b29e5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java @@ -17,7 +17,6 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.app.cli.session.SessionManager; -import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.rest.OpenCGAClient; import org.opencb.opencga.core.models.file.File; @@ -36,10 +35,10 @@ public CustomFilesCommandExecutor(ObjectMap options, String token, ClientConfigu super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse upload() throws Exception { + public RestResponse upload(CustomFilesCommandOptions.UploadCommandOptions commandOptions) throws Exception { // ObjectMap params = new ObjectMap() - options.append("fileFormat", ParamUtils.defaultString(String.valueOf(options.get("fileFormat")), File.Format.UNKNOWN.toString())) - .append("bioformat", ParamUtils.defaultString(String.valueOf(options.get("bioformat")), File.Bioformat.UNKNOWN.toString())); + options.append("fileFormat", options.getString("fileFormat", File.Format.UNKNOWN.toString())) + .append("bioformat", options.getString("bioformat", File.Bioformat.UNKNOWN.toString())); // //If the DEPRECATED parameter fileFormat has set we only override it if the new parameter format is also set // params.append("fileFormat", ParamUtils.defaultString(commandOptions.format, params.getString("fileFormat"))); // params.putIfNotEmpty("study", commandOptions.study); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java index 8d5742e09ec..dffedddcab4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java @@ -17,6 +17,8 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; +import org.opencb.opencga.app.cli.main.options.JobsCommandOptions; +import org.opencb.opencga.app.cli.main.utils.JobsLog; import org.opencb.opencga.app.cli.main.utils.JobsTopManager; import org.opencb.opencga.app.cli.session.SessionManager; import org.opencb.opencga.catalog.db.api.JobDBAdaptor; @@ -40,27 +42,26 @@ public CustomJobsCommandExecutor(ObjectMap options, String token, ClientConfigur super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse top() throws Exception { + public RestResponse top(CustomJobsCommandOptions.TopCommandOptions c) throws Exception { Query query = new Query(); - query.putIfNotEmpty(JobDBAdaptor.QueryParams.STUDY.key(), String.valueOf(options.get("study"))); - query.putIfNotEmpty(ParamConstants.JOB_TOOL_ID_PARAM, String.valueOf(options.get("toolId"))); - query.putIfNotEmpty(ParamConstants.INTERNAL_STATUS_PARAM, String.valueOf(options.get("internalStatus"))); - query.putIfNotEmpty(ParamConstants.JOB_USER_PARAM, String.valueOf(options.get("userId"))); - query.putIfNotEmpty(ParamConstants.JOB_PRIORITY_PARAM, String.valueOf(options.get("priority"))); - query.putAll(options); + query.putIfNotEmpty(JobDBAdaptor.QueryParams.STUDY.key(), c.study); + query.putIfNotEmpty(ParamConstants.JOB_TOOL_ID_PARAM, c.toolId); + query.putIfNotEmpty(ParamConstants.INTERNAL_STATUS_PARAM, c.internalStatus); + query.putIfNotEmpty(ParamConstants.JOB_USER_PARAM, c.userId); + query.putIfNotEmpty(ParamConstants.JOB_PRIORITY_PARAM, c.priority); + query.putAll(c.commonOptions.params); - new JobsTopManager(openCGAClient, query, 2, 20, 2, false).run(); + new JobsTopManager(openCGAClient, query, c.iterations, c.jobsLimit, c.delay, c.plain, c.columns).run(); RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); return res; } -/* + public RestResponse log(JobsCommandOptions.LogCommandOptions c) throws Exception { - // JobsCommandOptions.LogCommandOptions c = jobsCommandOptions.logCommandOptions; new JobsLog(openCGAClient, c, System.out).run(); RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); return res; } -*/ + } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java index 102a2b99255..a7ffdc3633b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java @@ -51,29 +51,25 @@ public CustomStudiesCommandExecutor(ObjectMap options, String token, ClientConfi super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse runTemplates() throws Exception { + public RestResponse runTemplates(CustomStudiesCommandOptions.RunTemplatesCommandOptions c) throws Exception { logger.debug("Run template"); - // StudiesCommandOptions.RunTemplatesCommandOptions c = studiesCommandOptions.runTemplatesCommandOptions; - String study = getSingleValidStudy(String.valueOf(options.get("study"))); - TemplateParams templateParams = new TemplateParams(String.valueOf(options.get("id")), - Boolean.parseBoolean(String.valueOf(options.get("overwrite"))), - Boolean.parseBoolean(String.valueOf(options.get("resume")))); + String study = getSingleValidStudy(c.study); + TemplateParams templateParams = new TemplateParams(c.id, c.overwrite, c.resume); ObjectMap params = new ObjectMap(); return openCGAClient.getStudyClient().runTemplates(study, templateParams, params); } - public RestResponse uploadTemplates() throws Exception { + public RestResponse uploadTemplates(CustomStudiesCommandOptions.UploadTemplatesCommandOptions c) throws Exception { logger.debug("Upload template file"); - // StudiesCommandOptions.UploadTemplatesCommandOptions c = studiesCommandOptions.uploadTemplatesCommandOptions; ObjectMap params = new ObjectMap(); - String study = getSingleValidStudy(String.valueOf(options.get("study"))); - Path path = Paths.get(String.valueOf(options.get("inputFile"))); + String study = c.study = getSingleValidStudy(c.study); + Path path = Paths.get(c.file); if (!path.toFile().exists()) { - throw new CatalogException("File '" + options.get("inputFile") + "' not found"); + throw new CatalogException("File '" + path + "' not found"); } IOManagerFactory ioManagerFactory = new IOManagerFactory(); IOManager ioManager = ioManagerFactory.get(path.toUri()); @@ -106,10 +102,10 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws logger.debug("Compressing file in '" + manifestPath + "' before uploading"); ioManager.zip(fileList, manifestPath.toFile()); params.put("file", manifestPath.toString()); - } else if (String.valueOf(options.get("inputFile")).endsWith("zip")) { - params.put("file", String.valueOf(options.get("inputFile"))); + } else if (c.file.endsWith("zip")) { + params.put("file", c.file); } else { - throw new CatalogException("File '" + options.get("inputFile") + "' is not a zip file"); + throw new CatalogException("File '" + c.file + "' is not a zip file"); } RestResponse uploadResponse = openCGAClient.getStudyClient().uploadTemplates(study, params); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java index eab222ebc45..7cac620fdc3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java @@ -50,18 +50,19 @@ public CustomUsersCommandExecutor(ObjectMap options, String token, ClientConfigu super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse login() throws Exception { + public RestResponse login(CustomUsersCommandOptions.LoginCommandOptions commandOptions) throws Exception { logger.debug("Login"); RestResponse res = new RestResponse<>(); try { - String user = String.valueOf(options.get("user")); - String password = String.valueOf(options.get("password")); + String user = commandOptions.user; + String password = commandOptions.password; if (StringUtils.isNotEmpty(user) && StringUtils.isNotEmpty(password)) { AuthenticationResponse response = null; try { response = openCGAClient.login(user, password); } catch (Exception e) { + logger.debug("Login error", e); Event event = new Event(); event.setMessage(e.getMessage()); event.setType(Event.Type.ERROR); @@ -73,16 +74,16 @@ public RestResponse login() throws Exception { res = session.saveSession(user, response, openCGAClient); println(getKeyValueAsFormattedString(LOGIN_OK, user)); } else { - String sessionId = String.valueOf(options.get("token")); + String token = session.getSession().getToken(); String errorMsg = "Missing password. "; - if (StringUtils.isNotEmpty(sessionId)) { + if (StringUtils.isNotEmpty(token)) { errorMsg += "Active token detected "; } CommandLineUtils.error(errorMsg); } } catch (Exception e) { CommandLineUtils.error(LOGIN_ERROR, e); - e.printStackTrace(); + logger.debug("Login error", e); Event event = new Event(); event.setMessage(LOGIN_ERROR + e.getMessage()); res.setType(QueryType.VOID); @@ -93,7 +94,7 @@ public RestResponse login() throws Exception { } - public RestResponse logout() throws IOException { + public RestResponse logout(CustomUsersCommandOptions.LogoutCommandOptions commandOptions) throws IOException { logger.debug("Logout"); RestResponse res = new RestResponse<>(); try { @@ -106,6 +107,7 @@ public RestResponse logout() throws IOException { res.setType(QueryType.VOID); } catch (Exception e) { CommandLineUtils.error("Logout fail", e); + logger.debug("Logout error", e); } return res; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java index a16efb4df47..19efed930b2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java @@ -1,25 +1,19 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import java.util.Map; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; import org.opencb.opencga.app.cli.main.options.AdminCommandOptions; - -import java.util.Map; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.admin.GroupSyncParams; import org.opencb.opencga.core.models.admin.InstallationParams; import org.opencb.opencga.core.models.admin.JWTParams; @@ -31,6 +25,8 @@ import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index 328a83daeb4..c1e5a5e0b7c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -1,26 +1,20 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.AnalysisAlignmentCommandOptions; - +import java.util.List; import org.ga4gh.models.ReadAlignment; import org.opencb.biodata.models.alignment.GeneCoverageStats; import org.opencb.biodata.models.alignment.RegionCoverage; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.AnalysisAlignmentCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.alignment.AlignmentGeneCoverageStatsParams; import org.opencb.opencga.core.models.alignment.AlignmentIndexParams; import org.opencb.opencga.core.models.alignment.AlignmentQcParams; @@ -31,6 +25,8 @@ import org.opencb.opencga.core.models.alignment.PicardWrapperParams; import org.opencb.opencga.core.models.alignment.SamtoolsWrapperParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index c8830ea76f3..bc73028ac99 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -1,34 +1,27 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.AnalysisClinicalCommandOptions; - +import java.util.List; import org.opencb.biodata.models.clinical.ClinicalDiscussion; import org.opencb.biodata.models.clinical.ClinicalProperty; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.AnalysisClinicalCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveReplaceAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.SaveInterpretationAs; import org.opencb.opencga.catalog.utils.ParamUtils.UpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByGeneSummary; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividualSummary; @@ -64,6 +57,8 @@ import org.opencb.opencga.core.models.study.configuration.ClinicalConsentAnnotationParam; import org.opencb.opencga.core.models.study.configuration.ClinicalConsentConfiguration; import org.opencb.opencga.core.models.study.configuration.InterpretationStudyConfiguration; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 601ea3739bf..2f670f539e8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -1,23 +1,9 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.AnalysisVariantCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.ClinicalProperty.ModeOfInheritance; import org.opencb.biodata.models.clinical.ClinicalProperty.Penetrance; @@ -29,7 +15,15 @@ import org.opencb.biodata.models.variant.metadata.VariantMetadata; import org.opencb.biodata.models.variant.metadata.VariantSetStats; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryResponse; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.AnalysisVariantCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByGene; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; @@ -58,6 +52,8 @@ import org.opencb.opencga.core.models.variant.VariantExportParams; import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.models.variant.VariantStatsAnalysisParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; import org.opencb.oskar.analysis.variant.gwas.GwasConfiguration; @@ -862,6 +858,7 @@ private RestResponse runIndex() throws Exception { putNestedIfNotNull(beanParams, "family",commandOptions.family, true); putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); putNestedIfNotNull(beanParams, "load",commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java index 92677f27fa4..54d3c8c3704 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java @@ -1,28 +1,22 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.CohortsCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.CohortsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortAclEntryList; import org.opencb.opencga.core.models.cohort.CohortAclUpdateParams; @@ -33,6 +27,8 @@ import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java index 3e05c22aa5f..fcac8b23c88 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java @@ -1,25 +1,19 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; import org.opencb.opencga.app.cli.main.options.DiseasePanelsCommandOptions; - -import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelAclEntryList; @@ -27,6 +21,8 @@ import org.opencb.opencga.core.models.panel.PanelCreateParams; import org.opencb.opencga.core.models.panel.PanelImportParams; import org.opencb.opencga.core.models.panel.PanelUpdateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -210,7 +206,7 @@ private RestResponse distinct() throws Exception { } private RestResponse importPanels() throws Exception { - logger.debug("Executing import in Disease Panels command line"); + logger.debug("Executing importPanels in Disease Panels command line"); DiseasePanelsCommandOptions.ImportCommandOptions commandOptions = diseasePanelsCommandOptions.importCommandOptions; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java index 35e1596aba4..b7521c2c790 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java @@ -1,28 +1,22 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.FamiliesCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.FamiliesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.family.Family; @@ -33,6 +27,8 @@ import org.opencb.opencga.core.models.family.FamilyQualityControl; import org.opencb.opencga.core.models.family.FamilyUpdateParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index 039e0d536e3..62796417c62 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -1,66 +1,34 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.FilesCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; import java.io.DataInputStream; import java.io.InputStream; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.interpretation.Software; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; +import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandOptions; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.FilesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.alignment.AlignmentFileQualityControl; import org.opencb.opencga.core.models.alignment.CoverageFileQualityControl; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; +import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.File.Bioformat; import org.opencb.opencga.core.models.file.File.Format; -import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileAclEntryList; import org.opencb.opencga.core.models.file.FileAclUpdateParams; import org.opencb.opencga.core.models.file.FileContent; @@ -79,6 +47,8 @@ import org.opencb.opencga.core.models.file.SmallFileInternal; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.variant.VariantFileQualityControl; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -637,7 +607,7 @@ private RestResponse upload() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomFilesCommandExecutor customFilesCommandExecutor = new CustomFilesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customFilesCommandExecutor.upload(); + return customFilesCommandExecutor.upload(commandOptions); } private RestResponse acl() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 86fcc2f8d58..1516be90a6c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -1,32 +1,26 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.IndividualsCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.qc.SampleRelatednessReport; import org.opencb.biodata.models.core.OntologyTermAnnotation; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.IndividualsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.individual.Individual; @@ -39,6 +33,8 @@ import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.individual.Location; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index 93723bdc05e..17b78b206b7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -1,41 +1,22 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; +import java.util.Date; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.JobsCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; +import org.opencb.opencga.app.cli.main.*; import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import java.util.Date; -import org.opencb.commons.datastore.core.FacetField; -import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.JobsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.FileContent; import org.opencb.opencga.core.models.job.Job; @@ -46,6 +27,8 @@ import org.opencb.opencga.core.models.job.JobTop; import org.opencb.opencga.core.models.job.JobUpdateParams; import org.opencb.opencga.core.models.job.ToolInfo; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.core.tools.result.ExecutorInfo; @@ -123,6 +106,12 @@ public void execute() throws Exception { case "log-tail": queryResponse = tailLog(); break; + case "log": + CustomJobsCommandExecutor customJobsCommandExecutor = new CustomJobsCommandExecutor( + new ObjectMap(jobsCommandOptions.commonCommandOptions.params), token, + clientConfiguration, getSessionManager(), appHome, getLogger()); + queryResponse = customJobsCommandExecutor.log(jobsCommandOptions.logCommandOptions); + break; default: logger.error("Subcommand not valid"); break; @@ -364,7 +353,7 @@ private RestResponse top() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomJobsCommandExecutor customJobsCommandExecutor = new CustomJobsCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customJobsCommandExecutor.top(); + return customJobsCommandExecutor.top(commandOptions); } private RestResponse acl() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java index 3f8448898be..e1d64ae2d4a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java @@ -1,25 +1,20 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.MetaCommandOptions; - import java.util.List; import java.util.Map; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.MetaCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 78ed32b92dc..494067498b1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -1,25 +1,18 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.OperationsVariantStorageCommandOptions; - +import java.util.List; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.OperationsVariantStorageCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; @@ -44,6 +37,8 @@ import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -530,6 +525,7 @@ private RestResponse indexVariant() throws Exception { putNestedIfNotNull(beanParams, "family",commandOptions.family, true); putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); putNestedIfNotNull(beanParams, "load",commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); @@ -600,6 +596,7 @@ private RestResponse launcherVariantIndex() throws Exception { putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true); putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true); putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true); + putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true); putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true); putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true); putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java index 2ed3593e612..e1757e59f09 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java @@ -1,30 +1,26 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; import org.opencb.opencga.app.cli.main.options.ProjectsCommandOptions; - -import org.opencb.commons.datastore.core.FacetField; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.project.ProjectUpdateParams; import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java index a88c8330a26..e3bd0666741 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java @@ -1,29 +1,23 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.SamplesCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.biodata.models.core.OntologyTermAnnotation; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.SamplesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.ExternalSource; import org.opencb.opencga.core.models.common.RgaIndex.Status; import org.opencb.opencga.core.models.common.StatusParams; @@ -38,6 +32,8 @@ import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.sample.SampleVariantQualityControlMetrics; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 944262e77f1..e183fcae20d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -1,54 +1,29 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.StudiesCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; import java.io.InputStream; import java.lang.Object; import java.net.URL; +import java.util.HashMap; +import java.util.List; import java.util.Set; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; +import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandOptions; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.StudiesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveForceRemoveAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.audit.AuditRecord.Status.Result; import org.opencb.opencga.core.models.audit.AuditRecord; @@ -73,6 +48,8 @@ import org.opencb.opencga.core.models.study.Variable; import org.opencb.opencga.core.models.study.VariableSet; import org.opencb.opencga.core.models.study.VariableSetCreateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -464,7 +441,7 @@ private RestResponse runTemplates() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomStudiesCommandExecutor customStudiesCommandExecutor = new CustomStudiesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customStudiesCommandExecutor.runTemplates(); + return customStudiesCommandExecutor.runTemplates(commandOptions); } private RestResponse uploadTemplates() throws Exception { @@ -478,7 +455,7 @@ private RestResponse uploadTemplates() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomStudiesCommandExecutor customStudiesCommandExecutor = new CustomStudiesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customStudiesCommandExecutor.uploadTemplates(); + return customStudiesCommandExecutor.uploadTemplates(commandOptions); } private RestResponse deleteTemplates() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index b2cca1037ba..0c19edfe709 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -1,39 +1,23 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.UsersCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; +import java.util.List; import java.util.Map; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; +import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandOptions; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.UsersCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.user.AuthenticationResponse; @@ -44,6 +28,8 @@ import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.models.user.UserUpdateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -105,6 +91,12 @@ public void execute() throws Exception { case "update": queryResponse = update(); break; + case "logout": + CustomUsersCommandExecutor customUsersCommandExecutor = new CustomUsersCommandExecutor( + new ObjectMap(usersCommandOptions.commonCommandOptions.params), token, + clientConfiguration, getSessionManager(), appHome, getLogger()); + queryResponse = customUsersCommandExecutor.logout(usersCommandOptions.logoutCommandOptions); + break; default: logger.error("Subcommand not valid"); break; @@ -123,7 +115,7 @@ private RestResponse login() throws Exception { queryParams.putIfNotEmpty("password", commandOptions.password); queryParams.putIfNotEmpty("refreshToken", commandOptions.refreshToken); CustomUsersCommandExecutor customUsersCommandExecutor = new CustomUsersCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customUsersCommandExecutor.login(); + return customUsersCommandExecutor.login(commandOptions); } private RestResponse password() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 0b37bec1c1c..c69a1700845 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -1055,91 +1055,94 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--outdir"}, description = "Output directory", required = false, arity = 1) public String outdir; - @Parameter(names = {"--transform"}, description = "The body web service transform parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean transform = false; - @Parameter(names = {"--gvcf"}, description = "The body web service gvcf parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean gvcf = false; - @Parameter(names = {"--normalization-skip"}, description = "The body web service normalizationSkip parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean normalizationSkip = false; - @Parameter(names = {"--reference-genome"}, description = "The body web service referenceGenome parameter", required = false, arity = 1) + @Parameter(names = {"--reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String referenceGenome; - @Parameter(names = {"--fail-on-malformed-lines"}, description = "The body web service failOnMalformedLines parameter", required = false, arity = 1) + @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean family = false; - @Parameter(names = {"--somatic"}, description = "The body web service somatic parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean somatic = false; - @Parameter(names = {"--load"}, description = "The body web service load parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean load = false; - @Parameter(names = {"--load-split-data"}, description = "The body web service loadSplitData parameter", required = false, arity = 1) + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) + public boolean forceReload = false; + + @Parameter(names = {"--load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String loadSplitData; - @Parameter(names = {"--load-multi-file-data"}, description = "The body web service loadMultiFileData parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean loadMultiFileData = false; - @Parameter(names = {"--load-sample-index"}, description = "The body web service loadSampleIndex parameter", required = false, arity = 1) + @Parameter(names = {"--load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String loadSampleIndex; - @Parameter(names = {"--load-archive"}, description = "The body web service loadArchive parameter", required = false, arity = 1) + @Parameter(names = {"--load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String loadArchive; - @Parameter(names = {"--load-hom-ref"}, description = "The body web service loadHomRef parameter", required = false, arity = 1) + @Parameter(names = {"--load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String loadHomRef; - @Parameter(names = {"--post-load-check"}, description = "The body web service postLoadCheck parameter", required = false, arity = 1) + @Parameter(names = {"--post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String postLoadCheck; - @Parameter(names = {"--include-genotypes"}, description = "The body web service includeGenotypes parameter", required = false, arity = 1) + @Parameter(names = {"--include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String includeGenotypes; - @Parameter(names = {"--include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) + @Parameter(names = {"--include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String includeSampleData; - @Parameter(names = {"--merge"}, description = "The body web service merge parameter", required = false, arity = 1) + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String merge; - @Parameter(names = {"--deduplication-policy"}, description = "The body web service deduplicationPolicy parameter", required = false, arity = 1) + @Parameter(names = {"--deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String deduplicationPolicy; - @Parameter(names = {"--calculate-stats"}, description = "The body web service calculateStats parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean calculateStats = false; - @Parameter(names = {"--aggregated"}, description = "The body web service aggregated parameter", required = false, arity = 1) + @Parameter(names = {"--aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String aggregated; - @Parameter(names = {"--aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) + @Parameter(names = {"--aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String aggregationMappingFile; - @Parameter(names = {"--annotate"}, description = "The body web service annotate parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean annotate = false; - @Parameter(names = {"--annotator"}, description = "The body web service annotator parameter", required = false, arity = 1) + @Parameter(names = {"--annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String annotator; - @Parameter(names = {"--overwrite-annotations"}, description = "The body web service overwriteAnnotations parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean overwriteAnnotations = false; - @Parameter(names = {"--index-search"}, description = "The body web service indexSearch parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexSearch = false; - @Parameter(names = {"--skip-indexed-files"}, description = "The body web service skipIndexedFiles parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean skipIndexedFiles = false; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index d0b8a26d019..a9b95be294a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -449,91 +449,94 @@ public class IndexVariantCommandOptions { @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--outdir"}, description = "Output directory", required = false, arity = 1) public String outdir; - @Parameter(names = {"--transform"}, description = "The body web service transform parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean transform = false; - @Parameter(names = {"--gvcf"}, description = "The body web service gvcf parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean gvcf = false; - @Parameter(names = {"--normalization-skip"}, description = "The body web service normalizationSkip parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean normalizationSkip = false; - @Parameter(names = {"--reference-genome"}, description = "The body web service referenceGenome parameter", required = false, arity = 1) + @Parameter(names = {"--reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String referenceGenome; - @Parameter(names = {"--fail-on-malformed-lines"}, description = "The body web service failOnMalformedLines parameter", required = false, arity = 1) + @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean family = false; - @Parameter(names = {"--somatic"}, description = "The body web service somatic parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean somatic = false; - @Parameter(names = {"--load"}, description = "The body web service load parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean load = false; - @Parameter(names = {"--load-split-data"}, description = "The body web service loadSplitData parameter", required = false, arity = 1) + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) + public boolean forceReload = false; + + @Parameter(names = {"--load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String loadSplitData; - @Parameter(names = {"--load-multi-file-data"}, description = "The body web service loadMultiFileData parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean loadMultiFileData = false; - @Parameter(names = {"--load-sample-index"}, description = "The body web service loadSampleIndex parameter", required = false, arity = 1) + @Parameter(names = {"--load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String loadSampleIndex; - @Parameter(names = {"--load-archive"}, description = "The body web service loadArchive parameter", required = false, arity = 1) + @Parameter(names = {"--load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String loadArchive; - @Parameter(names = {"--load-hom-ref"}, description = "The body web service loadHomRef parameter", required = false, arity = 1) + @Parameter(names = {"--load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String loadHomRef; - @Parameter(names = {"--post-load-check"}, description = "The body web service postLoadCheck parameter", required = false, arity = 1) + @Parameter(names = {"--post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String postLoadCheck; - @Parameter(names = {"--include-genotypes"}, description = "The body web service includeGenotypes parameter", required = false, arity = 1) + @Parameter(names = {"--include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String includeGenotypes; - @Parameter(names = {"--include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) + @Parameter(names = {"--include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String includeSampleData; - @Parameter(names = {"--merge"}, description = "The body web service merge parameter", required = false, arity = 1) + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String merge; - @Parameter(names = {"--deduplication-policy"}, description = "The body web service deduplicationPolicy parameter", required = false, arity = 1) + @Parameter(names = {"--deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String deduplicationPolicy; - @Parameter(names = {"--calculate-stats"}, description = "The body web service calculateStats parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean calculateStats = false; - @Parameter(names = {"--aggregated"}, description = "Enum param allowed values: NONE, BASIC, EVS, EXAC", required = false, arity = 1) + @Parameter(names = {"--aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String aggregated; - @Parameter(names = {"--aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) + @Parameter(names = {"--aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String aggregationMappingFile; - @Parameter(names = {"--annotate"}, description = "The body web service annotate parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean annotate = false; - @Parameter(names = {"--annotator"}, description = "The body web service annotator parameter", required = false, arity = 1) + @Parameter(names = {"--annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String annotator; - @Parameter(names = {"--overwrite-annotations"}, description = "The body web service overwriteAnnotations parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean overwriteAnnotations = false; - @Parameter(names = {"--index-search"}, description = "The body web service indexSearch parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexSearch = false; - @Parameter(names = {"--skip-indexed-files"}, description = "The body web service skipIndexedFiles parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean skipIndexedFiles = false; } @@ -580,91 +583,94 @@ public class LauncherVariantIndexCommandOptions { @Parameter(names = {"--max-jobs"}, description = "The body web service maxJobs parameter", required = false, arity = 1) public Integer maxJobs; - @Parameter(names = {"--index-params-file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-file"}, description = "List of files to be indexed.", required = false, arity = 1) public String indexParamsFile; - @Parameter(names = {"--index-params-resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean indexParamsResume = false; - @Parameter(names = {"--index-params-outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-outdir"}, description = "Output directory", required = false, arity = 1) public String indexParamsOutdir; - @Parameter(names = {"--index-params-transform"}, description = "The body web service transform parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean indexParamsTransform = false; - @Parameter(names = {"--index-params-gvcf"}, description = "The body web service gvcf parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean indexParamsGvcf = false; - @Parameter(names = {"--index-params-normalization-skip"}, description = "The body web service normalizationSkip parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean indexParamsNormalizationSkip = false; - @Parameter(names = {"--index-params-reference-genome"}, description = "The body web service referenceGenome parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String indexParamsReferenceGenome; - @Parameter(names = {"--index-params-fail-on-malformed-lines"}, description = "The body web service failOnMalformedLines parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsFailOnMalformedLines; - @Parameter(names = {"--index-params-family"}, description = "The body web service family parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean indexParamsFamily = false; - @Parameter(names = {"--index-params-somatic"}, description = "The body web service somatic parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean indexParamsSomatic = false; - @Parameter(names = {"--index-params-load"}, description = "The body web service load parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean indexParamsLoad = false; - @Parameter(names = {"--index-params-load-split-data"}, description = "The body web service loadSplitData parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) + public boolean indexParamsForceReload = false; + + @Parameter(names = {"--index-params-load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String indexParamsLoadSplitData; - @Parameter(names = {"--index-params-load-multi-file-data"}, description = "The body web service loadMultiFileData parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean indexParamsLoadMultiFileData = false; - @Parameter(names = {"--index-params-load-sample-index"}, description = "The body web service loadSampleIndex parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsLoadSampleIndex; - @Parameter(names = {"--index-params-load-archive"}, description = "The body web service loadArchive parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsLoadArchive; - @Parameter(names = {"--index-params-load-hom-ref"}, description = "The body web service loadHomRef parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsLoadHomRef; - @Parameter(names = {"--index-params-post-load-check"}, description = "The body web service postLoadCheck parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsPostLoadCheck; - @Parameter(names = {"--index-params-include-genotypes"}, description = "The body web service includeGenotypes parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsIncludeGenotypes; - @Parameter(names = {"--index-params-include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String indexParamsIncludeSampleData; - @Parameter(names = {"--index-params-merge"}, description = "The body web service merge parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String indexParamsMerge; - @Parameter(names = {"--index-params-deduplication-policy"}, description = "The body web service deduplicationPolicy parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String indexParamsDeduplicationPolicy; - @Parameter(names = {"--index-params-calculate-stats"}, description = "The body web service calculateStats parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean indexParamsCalculateStats = false; - @Parameter(names = {"--index-params-aggregated"}, description = "Enum param allowed values: NONE, BASIC, EVS, EXAC", required = false, arity = 1) + @Parameter(names = {"--index-params-aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String indexParamsAggregated; - @Parameter(names = {"--index-params-aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String indexParamsAggregationMappingFile; - @Parameter(names = {"--index-params-annotate"}, description = "The body web service annotate parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean indexParamsAnnotate = false; - @Parameter(names = {"--index-params-annotator"}, description = "The body web service annotator parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String indexParamsAnnotator; - @Parameter(names = {"--index-params-overwrite-annotations"}, description = "The body web service overwriteAnnotations parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean indexParamsOverwriteAnnotations = false; - @Parameter(names = {"--index-params-index-search"}, description = "The body web service indexSearch parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexParamsIndexSearch = false; - @Parameter(names = {"--index-params-skip-indexed-files"}, description = "The body web service skipIndexedFiles parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean indexParamsSkipIndexedFiles = false; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java index 418f6ed3152..6ff2cc5bff5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java @@ -34,14 +34,28 @@ public class CalculatePedigreeGraphMigration extends MigrationTool { protected void run() throws Exception { MigrationRun migrationRun = getMigrationRun(); + // Map study studyFqn -> job Map jobs = new HashMap<>(); for (JobReferenceParam jobReference : migrationRun.getJobs()) { Job job = catalogManager.getJobManager().get(jobReference.getStudyId(), jobReference.getId(), new QueryOptions(), token) .first(); - logger.info("Registering job {} for study {} to migrate", job.getId(), job.getStudy().getId()); + logger.info("Reading already executed job '{}' for study '{}' with status '{}'", + job.getId(), + job.getStudy().getId(), + job.getInternal().getStatus().getId()); jobs.put(job.getStudy().getId(), job); } - for (String study : getStudies()) { + + Set studies = new LinkedHashSet<>(getStudies()); + logger.info("Study IDs (num. total = {}) to initialize pedigree graphs: {}", studies.size(), StringUtils.join(studies, ", ")); + + // Ensure that studies with already executed jobs are included in the migration run + getMigrationRun().getJobs().forEach(j -> studies.add(j.getStudyId())); + + logger.info("Study IDs (num. total = {}) after adding studies from migration jobs: {}", studies.size(), + StringUtils.join(studies, ", ")); + + for (String study : studies) { Job job = jobs.get(study); if (job != null) { String status = job.getInternal().getStatus().getId(); @@ -61,8 +75,9 @@ protected void run() throws Exception { logger.info("Adding new job to migrate/initialize pedigree graph for study {}", study); ObjectMap params = new ObjectMap() .append(ParamConstants.STUDY_PARAM, study); - getMigrationRun().addJob(catalogManager.getJobManager().submit(study, PedigreeGraphInitAnalysis.ID, Enums.Priority.MEDIUM, - params, null, null, null, new ArrayList<>(), token).first()); + Job newJob = catalogManager.getJobManager().submit(study, PedigreeGraphInitAnalysis.ID, Enums.Priority.MEDIUM, + params, null, null, null, new ArrayList<>(), token).first(); + getMigrationRun().addJob(newJob); } } @@ -73,7 +88,7 @@ public List getStudies() throws CatalogException { for (Project project : catalogManager.getProjectManager().search(new Query(), projectOptions, token).getResults()) { if (CollectionUtils.isNotEmpty(project.getStudies())) { for (Study study : project.getStudies()) { - String id = project.getId() + ":" + study.getId(); + String id = study.getFqn(); for (Family family : catalogManager.getFamilyManager().search(id, new Query(), familyOptions, token).getResults()) { if (PedigreeGraphUtils.hasMinTwoGenerations(family) && (family.getPedigreeGraph() == null || StringUtils.isEmpty(family.getPedigreeGraph().getBase64()))) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java new file mode 100644 index 00000000000..593eb6a15a0 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java @@ -0,0 +1,43 @@ +package org.opencb.opencga.app.migrations.v2_8_6.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; + +import java.util.Map; + +@Migration(id = "mark_unknown_largest_variant_length" , + description = "Mark as unknown largest variant length", + version = "2.8.6", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + patch = 1, + date = 20230927 +) +public class MarkUnknownLargestVariantLength extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + for (String project : getVariantStorageProjects()) { + try (VariantStorageEngine engine = getVariantStorageEngineByProject(project)) { + for (Map.Entry entry : engine.getMetadataManager().getStudies().entrySet()) { + String studyName = entry.getKey(); + Integer studyId = entry.getValue(); + logger.info("Process study '" + studyName + "' (" + studyId + ")"); + // Check for indexed samples with undefined largest variant length + for (SampleMetadata sampleMetadata : engine.getMetadataManager().sampleMetadataIterable(studyId)) { + if (sampleMetadata.isIndexed()) { + if (!sampleMetadata.getAttributes().containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH)) { + logger.info("Mark unknown largest variant length for sample '" + sampleMetadata.getName() + "'"); + engine.getMetadataManager().updateSampleMetadata(studyId, sampleMetadata.getId(), + sm -> sm.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, true)); + } + } + } + } + } + } + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 1a074a17331..6437a0293b2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -644,7 +644,6 @@ private void updateClinicalAnalysisIndividualReferences(ClientSession clientSess */ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document individual) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - String individualId = individual.getString(QueryParams.ID.key()); long individualUid = individual.getLong(PRIVATE_UID); long studyUid = individual.getLong(PRIVATE_STUDY_UID); @@ -653,8 +652,7 @@ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document .append(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), individualUid); OpenCGAResult count = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().count(clientSession, query); if (count.getNumMatches() > 0) { - throw new CatalogDBException("Could not delete individual '" + individualId + "'. Individual is in use in " - + count.getNumMatches() + " cases"); + throw new CatalogDBException("Individual is in use in " + count.getNumMatches() + " cases"); } } @@ -940,7 +938,7 @@ public OpenCGAResult delete(Individual individual) throws CatalogDBException, Ca return runTransaction(clientSession -> privateDelete(clientSession, result.first())); } catch (CatalogDBException e) { logger.error("Could not delete individual {}: {}", individual.getId(), e.getMessage(), e); - throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e.getCause()); + throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e); } } @@ -976,7 +974,7 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document indivi checkInUseInClinicalAnalysis(clientSession, individualDocument); logger.debug("Deleting individual {} ({})", individualId, individualUid); - // Look for all the different family versions + // Look for all the different individual versions Query individualQuery = new Query() .append(QueryParams.UID.key(), individualUid) .append(QueryParams.STUDY_UID.key(), studyUid); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index dde62bd87d3..6d7bd764c57 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -500,7 +500,6 @@ void updateIndividualIdFromSamples(ClientSession clientSession, long studyUid, S */ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document sample) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - String sampleId = sample.getString(QueryParams.ID.key()); long sampleUid = sample.getLong(QueryParams.UID.key()); long studyUid = sample.getLong(QueryParams.STUDY_UID.key()); @@ -509,8 +508,7 @@ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document .append(ClinicalAnalysisDBAdaptor.QueryParams.SAMPLE.key(), sampleUid); OpenCGAResult count = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().count(clientSession, query); if (count.getNumMatches() > 0) { - throw new CatalogDBException("Could not delete sample '" + sampleId + "'. Sample is in use in " - + count.getNumMatches() + " cases"); + throw new CatalogDBException("Sample is in use in " + count.getNumMatches() + " cases"); } } @@ -860,7 +858,7 @@ public OpenCGAResult delete(Sample sample) throws CatalogDBException, CatalogPar return runTransaction(clientSession -> privateDelete(clientSession, result.first())); } catch (CatalogDBException e) { logger.error("Could not delete sample {}: {}", sample.getId(), e.getMessage(), e); - throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e.getCause()); + throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java index d7183adabae..c135e390744 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java @@ -530,7 +530,8 @@ private void checkCanBeDeleted(Study study, Family family) throws CatalogExcepti OpenCGAResult result = clinicalDBAdaptor.get(query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); if (result.getNumResults() > 0) { String clinicalIds = result.getResults().stream().map(ClinicalAnalysis::getId).collect(Collectors.joining(", ")); - throw new CatalogException("Family {" + family.getId() + "} in use in Clinical Analyses: {" + clinicalIds + "}"); + throw new CatalogException("Could not delete family '" + family.getId() + "'. Family is in use in Clinical Analyses: '" + + clinicalIds + "'"); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java index 07d03a295ce..97057677b0d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java @@ -710,10 +710,8 @@ private OpenCGAResult delete(Study study, Individual individual, ObjectMap param // Check if the individual can be deleted if (!params.getBoolean(Constants.FORCE, false)) { if (familyDataResult.getNumResults() > 0) { - throw new CatalogException("Individual found in the families: " + familyDataResult.getResults() - .stream() - .map(Family::getId) - .collect(Collectors.joining(", "))); + throw new CatalogException("Could not delete individual '" + individual.getId() + "'. Individual found in the families: " + + familyDataResult.getResults().stream().map(Family::getId).collect(Collectors.joining(", "))); } } else { logger.info("Forcing deletion of individuals belonging to families"); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java index 5ab592050d2..580bc1c2c2c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java @@ -815,6 +815,7 @@ public OpenCGAResult resetAnnotations(String studyStr, String sampleStr, } private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) throws CatalogException { + String msg = "Could not delete sample '" + sample.getId() + "'. "; // Look for files related with the sample Query query = new Query() .append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sample.getId()) @@ -834,9 +835,9 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) } if (!errorFiles.isEmpty()) { if (force) { - throw new CatalogException("Associated files are used in storage: " + StringUtils.join(errorFiles, ", ")); + throw new CatalogException(msg + "Associated files are used in storage: " + StringUtils.join(errorFiles, ", ")); } else { - throw new CatalogException("Sample associated to the files: " + StringUtils.join(errorFiles, ", ")); + throw new CatalogException(msg + "Sample associated to the files: " + StringUtils.join(errorFiles, ", ")); } } @@ -865,14 +866,14 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) } } if (associatedToDefaultCohort) { - throw new CatalogException("Sample in cohort " + StudyEntry.DEFAULT_COHORT); + throw new CatalogException(msg + "Sample in cohort " + StudyEntry.DEFAULT_COHORT); } if (!errorCohorts.isEmpty()) { if (force) { - throw new CatalogException("Sample present in cohorts in the process of calculating the stats: " + throw new CatalogException(msg + "Sample present in cohorts in the process of calculating the stats: " + StringUtils.join(errorCohorts, ", ")); } else { - throw new CatalogException("Sample present in cohorts: " + StringUtils.join(errorCohorts, ", ")); + throw new CatalogException(msg + "Sample present in cohorts: " + StringUtils.join(errorCohorts, ", ")); } } @@ -884,8 +885,7 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) OpenCGAResult individualDataResult = individualDBAdaptor.get(query, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.UID.key(), IndividualDBAdaptor.QueryParams.ID.key()))); if (individualDataResult.getNumResults() > 0) { - throw new CatalogException("Sample from individual " + individualDataResult.first().getName() + "(" - + individualDataResult.first().getUid() + ")"); + throw new CatalogException(msg + "Sample is associated with individual '" + individualDataResult.first().getId() + "'."); } } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java index 51c78472790..e9134475dd7 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java @@ -20,8 +20,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.RandomStringUtils; import org.bson.Document; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.internal.matchers.ThrowableMessageMatcher; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; import org.opencb.biodata.models.clinical.qc.SampleQcVariantStats; @@ -2776,8 +2779,9 @@ public void deleteInUseInCATest() throws CatalogException { try { catalogManager.getSampleManager().delete(studyFqn, Collections.singletonList(case1.getProband().getSamples().get(0).getId()), new QueryOptions(ParamConstants.FORCE, true), token); + fail("Expected CatalogException with message containing 'in use in 3 cases'"); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("in use in 3 cases")); + MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("in use in 3 cases"))); } // unlock case3 @@ -2786,9 +2790,10 @@ public void deleteInUseInCATest() throws CatalogException { try { catalogManager.getSampleManager().delete(studyFqn, Collections.singletonList(case1.getProband().getSamples().get(0).getId()), - new QueryOptions(ParamConstants.FORCE, true), token); + new QueryOptions(ParamConstants.FORCE, false), token); + fail("Expected CatalogException with message containing 'associated with individual'."); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("in use in 3 cases")); + MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("associated with individual"))); } } diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index fb0eca89883..612d5fd7f51 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index f2d94f8d87e..9777dbfccb6 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index ee079be8a55..0dd45611bfb 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,6 +1,10 @@ # ############################################################################## ## UserClient +<<<<<<< HEAD setGeneric("userClient", function(OpencgaR, filterId, users, user, endpointName, params=NULL, ...) +======= +setGeneric("userClient", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("userClient")) # ############################################################################## @@ -10,12 +14,20 @@ setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, # ############################################################################## ## StudyClient +<<<<<<< HEAD setGeneric("studyClient", function(OpencgaR, variableSet, templateId, members, study, studies, group, endpointName, params=NULL, ...) +======= +setGeneric("studyClient", function(OpencgaR, members, templateId, study, variableSet, group, studies, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("studyClient")) # ############################################################################## ## FileClient +<<<<<<< HEAD setGeneric("fileClient", function(OpencgaR, folder, annotationSet, file, members, files, endpointName, params=NULL, ...) +======= +setGeneric("fileClient", function(OpencgaR, members, folder, annotationSet, files, file, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("fileClient")) # ############################################################################## @@ -25,22 +37,38 @@ setGeneric("jobClient", function(OpencgaR, jobs, members, job, endpointName, par # ############################################################################## ## SampleClient +<<<<<<< HEAD setGeneric("sampleClient", function(OpencgaR, samples, annotationSet, sample, members, endpointName, params=NULL, ...) +======= +setGeneric("sampleClient", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient +<<<<<<< HEAD setGeneric("individualClient", function(OpencgaR, individual, annotationSet, members, individuals, endpointName, params=NULL, ...) +======= +setGeneric("individualClient", function(OpencgaR, members, individuals, individual, annotationSet, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("individualClient")) # ############################################################################## ## FamilyClient +<<<<<<< HEAD setGeneric("familyClient", function(OpencgaR, families, members, family, annotationSet, endpointName, params=NULL, ...) +======= +setGeneric("familyClient", function(OpencgaR, members, families, family, annotationSet, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("familyClient")) # ############################################################################## ## CohortClient +<<<<<<< HEAD setGeneric("cohortClient", function(OpencgaR, cohort, cohorts, members, annotationSet, endpointName, params=NULL, ...) +======= +setGeneric("cohortClient", function(OpencgaR, members, cohorts, cohort, annotationSet, endpointName, params=NULL, ...) +>>>>>>> develop standardGeneric("cohortClient")) # ############################################################################## @@ -60,7 +88,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, interpretation, interpretations, clinicalAnalysis, clinicalAnalyses, members, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, interpretation, members, interpretations, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 9153616b670..4b7b2f51b8f 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -59,7 +63,7 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, interpretation, interpretations, clinicalAnalysis, clinicalAnalyses, members, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, interpretation, members, interpretations, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index 15fdb9a0e6b..d2e324b7c9b 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +43,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("cohortClient", "OpencgaR", function(OpencgaR, cohort, cohorts, members, annotationSet, endpointName, params=NULL, ...) { +======= +setMethod("cohortClient", "OpencgaR", function(OpencgaR, members, cohorts, cohort, annotationSet, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index 8439bc5eed1..4442581dfde 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +42,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("familyClient", "OpencgaR", function(OpencgaR, families, members, family, annotationSet, endpointName, params=NULL, ...) { +======= +setMethod("familyClient", "OpencgaR", function(OpencgaR, members, families, family, annotationSet, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index c3163af8bc1..920d5c7c749 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +58,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("fileClient", "OpencgaR", function(OpencgaR, folder, annotationSet, file, members, files, endpointName, params=NULL, ...) { +======= +setMethod("fileClient", "OpencgaR", function(OpencgaR, members, folder, annotationSet, files, file, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index 91e50ab9a75..07cfde767ef 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index 84cbf3c13c8..d4424703aa2 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +43,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("individualClient", "OpencgaR", function(OpencgaR, individual, annotationSet, members, individuals, endpointName, params=NULL, ...) { +======= +setMethod("individualClient", "OpencgaR", function(OpencgaR, members, individuals, individual, annotationSet, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index 5f0ea55bf3a..fab9d0bfba1 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index 17c8aec917c..ef2ec571c8d 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 73008a6dc58..85972e5fa70 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index 4f7e71e90dc..e9a0416afbe 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index 4ab8c70d386..3f08b058bd0 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index f23d51b52ad..537c4960bd4 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +43,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("sampleClient", "OpencgaR", function(OpencgaR, samples, annotationSet, sample, members, endpointName, params=NULL, ...) { +======= +setMethod("sampleClient", "OpencgaR", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index 054e4bbcac4..b0b0073520f 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +50,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("studyClient", "OpencgaR", function(OpencgaR, variableSet, templateId, members, study, studies, group, endpointName, params=NULL, ...) { +======= +setMethod("studyClient", "OpencgaR", function(OpencgaR, members, templateId, study, variableSet, group, studies, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index 1584c0e075a..99ed56568b3 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +42,11 @@ #' [*]: Required parameter #' @export +<<<<<<< HEAD setMethod("userClient", "OpencgaR", function(OpencgaR, filterId, users, user, endpointName, params=NULL, ...) { +======= +setMethod("userClient", "OpencgaR", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) { +>>>>>>> develop switch(endpointName, #' @section Endpoint /{apiVersion}/users/login: diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index 58fa54e96f8..2b45a633adc 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,11 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. +<<<<<<< HEAD # Autogenerated on: 2023-09-21 +======= +# Autogenerated on: 2023-10-20 +>>>>>>> develop # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java index 58d37b4caf6..007c5f7f2e5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java @@ -456,6 +456,7 @@ protected RestResponse callUploadRest(WebTarget path, Map client.property(ClientProperties.READ_TIMEOUT, DEFAULT_UPLOAD_TIMEOUT); path.register(MultiPartFeature.class); path.property(ClientProperties.REQUEST_ENTITY_PROCESSING, RequestEntityProcessing.CHUNKED); + final FileDataBodyPart filePart = new FileDataBodyPart("file", new File(filePath)); FormDataMultiPart formDataMultiPart = new FormDataMultiPart(); // Add the rest of the parameters to the form diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 410029a185c..f35b2fe758f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -36,7 +36,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index cb76d509e42..90ed67e0a6c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -40,7 +40,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index d2a34debe8b..94b5981fdd0 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -52,7 +52,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index 464bccf9f3c..f252f231f62 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -37,7 +37,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index 28509381a9b..fb4b5649f9b 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -35,7 +35,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index fc41fdc86b6..bc708898e98 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -36,7 +36,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index f813fd06120..f198e84498f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -43,7 +43,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index 13f517d8fbc..500db60dfec 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -27,7 +27,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index 7241abf1571..e4f578c8cca 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -36,7 +36,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index 741ce9b9f4c..98ce8574285 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -37,7 +37,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index 1f53e93943d..a20d0a4d11e 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -28,7 +28,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 7fb04407c13..421feae2f3f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -32,7 +32,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index d8e4df075da..d007ee204bd 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -36,7 +36,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 6c0ccc7405b..5cece772beb 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -45,7 +45,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index de64fc0bbf3..c3e6d3e7480 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -36,7 +36,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index c8c64eb994a..e7dcfc64c5f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -62,7 +62,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index 25d8baaccdb..43fd9f4caa5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -50,7 +50,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= +* Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index d2360169f0a..77366a82440 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index bc89db681be..7a874e1460b 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index e2f78ab24e8..c4d69a64193 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index dc60589a027..fb6dc3ce084 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index 7110b11e7aa..5890f01e9cf 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index ba2b0388341..4390c4238bc 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index 911ef949bfb..b1edba7171f 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index 27512d50b5c..85bfc2ba14a 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index f74942a95bd..871f6b58d15 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index 1025b47d6c1..d19d635261e 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index 01db0c77a91..200aa8d662c 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index cbb8f11c136..d07fffce7de 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index 8946f10ceb7..8187f920561 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index fca98d52c02..7bade16e1c2 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 35915ec5f1b..19b84806f08 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index d9439f7c98d..79616899a57 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index b9098424bfe..1fa108ef909 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -12,7 +12,11 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. +<<<<<<< HEAD * Autogenerated on: 2023-09-21 +======= + * Autogenerated on: 2023-10-20 +>>>>>>> develop * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index f31cdfdbe0a..b416ce91a5f 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index 9bc67c570d9..fb4e77ec382 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index 42726068e16..541fe3467a3 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index 56a6bc14438..d6b66993e13 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index a1726e8c0b1..aff54b0ca43 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index b60403516bf..89d35f329d4 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index fad895ae19c..06fbf35dffe 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index 642f1991404..0b09ebf5147 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index f5e40232abc..d283d7c78fc 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 5e89ae5ab93..9d50da8043a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index 9a6b7d564d2..03991cb2f22 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index 4ab51325764..bfd131998aa 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index be639101daf..c0c4ff7102d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 07b4dea33f6..4a70bfc9ae2 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index ce4d480c293..0796d30f325 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index 1e36904b911..061c05a7984 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index 40fc3e805f2..44930e5ce30 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,11 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. +<<<<<<< HEAD Autogenerated on: 2023-09-21 +======= + Autogenerated on: 2023-10-20 +>>>>>>> develop Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index edf5e3e50bc..6d7eafd1503 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -11,17 +11,20 @@ import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.VersionUtils; + import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; + public class CellBaseValidator { private static Logger logger = LoggerFactory.getLogger(CellBaseValidator.class); @@ -193,6 +196,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException } } } + String apiKey = getApiKey(); if (StringUtils.isEmpty(apiKey)) { cellBaseConfiguration.setApiKey(null); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java index aea655f6d9b..1e0cd7dac49 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java @@ -17,6 +17,8 @@ package org.opencb.opencga.core.models.variant; import org.opencb.biodata.models.variant.metadata.Aggregation; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.tools.ToolParams; public class VariantIndexParams extends ToolParams { @@ -29,11 +31,14 @@ public VariantIndexParams(String file, boolean resume, String outdir, boolean transform, boolean gvcf, - boolean normalizationSkip, String referenceGenome, + boolean normalizationSkip, + String referenceGenome, String failOnMalformedLines, boolean family, boolean somatic, - boolean load, String loadSplitData, boolean loadMultiFileData, + boolean load, + boolean forceReload, + String loadSplitData, boolean loadMultiFileData, String loadSampleIndex, String loadArchive, String loadHomRef, @@ -53,6 +58,7 @@ public VariantIndexParams(String file, this.family = family; this.somatic = somatic; this.load = load; + this.forceReload = forceReload; this.loadSplitData = loadSplitData; this.loadMultiFileData = loadMultiFileData; this.loadSampleIndex = loadSampleIndex; @@ -73,43 +79,82 @@ public VariantIndexParams(String file, this.skipIndexedFiles = skipIndexedFiles; } + @DataField(description = "List of files to be indexed.") private String file; + @DataField(description = "Resume a previously failed index operation") private boolean resume; + @DataField(description = "Output directory") private String outdir; + @DataField(description = "If present it only runs the transform stage, no load is executed") private boolean transform; + @DataField(description = "Hint to indicate that the input file is in gVCF format.") private boolean gvcf; + @DataField(description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base") private boolean normalizationSkip; + @DataField(description = "Reference genome in FASTA format used during the normalization step " + + "for a complete left alignment") private String referenceGenome; + @DataField(description = "Fail when encountering malformed lines. (yes, no, auto) [auto]") private String failOnMalformedLines; + @DataField(description = "Indicate that the files to be loaded are part of a family. " + + "This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards") private boolean family; + @DataField(description = "Indicate that the files to be loaded contain somatic samples. " + + "This will set 'load-hom-ref' to YES if it was in AUTO.") private boolean somatic; + @DataField(description = "If present only the load stage is executed, transformation is skipped") private boolean load; + @DataField(description = "If the file is already loaded, force a file reload") + private boolean forceReload; + @DataField(description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.") private String loadSplitData; + @DataField(description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.") private boolean loadMultiFileData; + @DataField(description = "Build sample index while loading. (yes, no, auto) [auto]") private String loadSampleIndex; + @DataField(description = "Load archive data. (yes, no, auto) [auto]") private String loadArchive; + @DataField(description = "Load HOM_REF genotypes. (yes, no, auto) [auto]") private String loadHomRef; + @DataField(description = "Execute post load checks over the database. (yes, no, auto) [auto]") private String postLoadCheck; + @DataField(description = "Load the genotype data for the current file. " + + "This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. " + + "Use this parameter skip load data when the GT field is not reliable, or its only value across the file is \"./.\". " + + "If \"auto\", genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. " + + "(yes, no, auto) [auto]") private String includeGenotypes; + @DataField(description = "Index including other sample data fields (i.e. FORMAT fields)." + + " Use \"" + ParamConstants.ALL + "\", \"" + ParamConstants.NONE + "\", or CSV with the fields to load.") private String includeSampleData; + @DataField(deprecated = true, description = "Currently two levels of merge are supported: \"basic\" mode merge genotypes of the same variants while \"advanced\" merge multiallelic and overlapping variants.") private String merge; + @DataField(description = "Specify how duplicated variants should be handled. Available policies: \"discard\", \"maxQual\"") private String deduplicationPolicy; + @DataField(description = "Calculate indexed variants statistics after the load step") private boolean calculateStats; + @DataField(description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC") private Aggregation aggregated; + @DataField(description = "File containing population names mapping in an aggregated VCF file") private String aggregationMappingFile; + @DataField(description = "Annotate indexed variants after the load step") private boolean annotate; + @DataField(description = "Annotation source {cellbase_rest, cellbase_db_adaptor}") private String annotator; + @DataField(description = "Overwrite annotations in variants already present") private boolean overwriteAnnotations; + @DataField(description = "Add files to the secondary search index") private boolean indexSearch; + @DataField(description = "Do not fail if any of the input files was already indexed.") private boolean skipIndexedFiles; public String getFile() { @@ -211,6 +256,15 @@ public VariantIndexParams setLoad(boolean load) { return this; } + public boolean isForceReload() { + return forceReload; + } + + public VariantIndexParams setForceReload(boolean forceReload) { + this.forceReload = forceReload; + return this; + } + public String getLoadSplitData() { return loadSplitData; } diff --git a/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java b/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java index ada81158f44..cab404b3c34 100644 --- a/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java +++ b/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java @@ -112,4 +112,5 @@ public void testUnsignedApiKey() throws IOException { CellBaseConfiguration validated = CellBaseValidator.validate(new CellBaseConfiguration(ParamConstants.CELLBASE_URL, "v5.4", null, apiKey), "hsapiens", "grch38", true); Assert.assertNotNull(validated.getApiKey()); } + } \ No newline at end of file diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/OpenCGAHealthCheckMonitor.java b/opencga-server/src/main/java/org/opencb/opencga/server/OpenCGAHealthCheckMonitor.java new file mode 100644 index 00000000000..fa8c0a1b95f --- /dev/null +++ b/opencga-server/src/main/java/org/opencb/opencga/server/OpenCGAHealthCheckMonitor.java @@ -0,0 +1,283 @@ +package org.opencb.opencga.server; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang3.time.StopWatch; +import org.opencb.commons.datastore.core.Event; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.storage.core.StorageEngineFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +public class OpenCGAHealthCheckMonitor { + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final AtomicReference cachedResult = new AtomicReference<>(); + + private final Configuration configuration; + private final CatalogManager catalogManager; + private final StorageEngineFactory storageEngineFactory; + private final VariantStorageManager variantManager; + private final ExecutorService executorService; + + public OpenCGAHealthCheckMonitor(Configuration configuration, CatalogManager catalogManager, + StorageEngineFactory storageEngineFactory, + VariantStorageManager variantManager) { + this.configuration = configuration; + this.catalogManager = catalogManager; + this.storageEngineFactory = storageEngineFactory; + this.variantManager = variantManager; + executorService = Executors.newCachedThreadPool(); + } + + public static class HealthCheckStatus { + + enum Status { + OK, + KO, + NA + } + + @JsonProperty("CatalogMongoDB") + private Status catalogMongoDbStatus = null; + @JsonProperty("Solr") + private Status solrStatus = null; + @JsonProperty("VariantStorage") + private Status variantStorageStatus = null; + + @JsonProperty("VariantStorageId") + private String variantStorageId = ""; + + @JsonIgnore + private String errorMessage = null; + @JsonIgnore + private LocalDateTime creationDate; + @JsonIgnore + private boolean healthy; + + public HealthCheckStatus() { + } + + public Status getCatalogMongoDbStatus() { + return catalogMongoDbStatus; + } + + public HealthCheckStatus setCatalogMongoDbStatus(Status catalogMongoDbStatus) { + this.catalogMongoDbStatus = catalogMongoDbStatus; + return this; + } + + public Status getSolrStatus() { + return solrStatus; + } + + public HealthCheckStatus setSolrStatus(Status solrStatus) { + this.solrStatus = solrStatus; + return this; + } + + public Status getVariantStorageStatus() { + return variantStorageStatus; + } + + public HealthCheckStatus setVariantStorageStatus(Status variantStorageStatus) { + this.variantStorageStatus = variantStorageStatus; + return this; + } + + public String getVariantStorageId() { + return variantStorageId; + } + + public HealthCheckStatus setVariantStorageId(String variantStorageId) { + this.variantStorageId = variantStorageId; + return this; + } + + public boolean isHealthy() { + return healthy; + } + + public HealthCheckStatus setHealthy(boolean healthy) { + this.healthy = healthy; + return this; + } + + public String getErrorMessage() { + return errorMessage; + } + + public HealthCheckStatus setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + return this; + } + + public LocalDateTime getCreationDate() { + return creationDate; + } + + public HealthCheckStatus setCreationDate(LocalDateTime creationDate) { + this.creationDate = creationDate; + return this; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HealthCheckStatus{"); + sb.append("catalogMongoDbStatus='").append(catalogMongoDbStatus).append('\''); + sb.append(", solrStatus='").append(solrStatus).append('\''); + sb.append(", variantStorageStatus='").append(variantStorageStatus).append('\''); + sb.append(", variantStorageId='").append(variantStorageId).append('\''); + sb.append(", errorMessage='").append(errorMessage).append('\''); + sb.append(", creationDate=").append(creationDate); + sb.append(", healthy=").append(healthy); + sb.append('}'); + return sb.toString(); + } + } + + public void asyncUpdate() { + if (shouldUpdateStatus()) { + executorService.submit(() -> { + logger.debug("Update HealthCheck cache status"); + updateHealthCheck(); + }); + } + } + + public OpenCGAResult getStatus() { + + OpenCGAResult queryResult = new OpenCGAResult<>(); + StopWatch stopWatch = StopWatch.createStarted(); + + if (shouldUpdateStatus()) { + logger.debug("Update HealthCheck cache status"); + updateHealthCheck(); + } else { + HealthCheckStatus status = cachedResult.get(); + String msg = "HealthCheck results from cache at " + status.getCreationDate().format(DateTimeFormatter.ofPattern("HH:mm:ss")); + queryResult.setEvents(Collections.singletonList(new Event(Event.Type.INFO, msg))); + logger.debug(msg); + } + + queryResult.setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); + queryResult.setResults(Collections.singletonList(cachedResult.get())); + return queryResult; + } + + private boolean shouldUpdateStatus() { + HealthCheckStatus status = cachedResult.get(); + if (status == null || !status.isHealthy()) { + // Always update if not healthy or undefined + return true; + } + // If healthy, only update every "healthCheck.interval" seconds + long elapsedTime = Duration.between(status.getCreationDate(), LocalDateTime.now()).getSeconds(); + return elapsedTime > configuration.getHealthCheck().getInterval(); + } + + private void updateHealthCheck() { + updateHealthCheck(StopWatch.createStarted()); + } + + private synchronized void updateHealthCheck(StopWatch totalTime) { + if (!shouldUpdateStatus()) { + if (totalTime.getTime(TimeUnit.SECONDS) > 5) { + logger.warn("Slow OpenCGA status. Synchronized time wait: {} . Skip update.", + totalTime.getTime(TimeUnit.MILLISECONDS) / 1000.0 + ); + } + // Skip update! + return; + } + StringBuilder errorMsg = new StringBuilder(); + boolean healthy = true; + + HealthCheckStatus status = new HealthCheckStatus(); + + StopWatch catalogMongoDBTime = StopWatch.createStarted(); + try { + if (catalogManager.getCatalogDatabaseStatus()) { + status.setCatalogMongoDbStatus(HealthCheckStatus.Status.OK); + } else { + status.setCatalogMongoDbStatus(HealthCheckStatus.Status.KO); + healthy = false; + } + } catch (Exception e) { + status.setCatalogMongoDbStatus(HealthCheckStatus.Status.KO); + healthy = false; + errorMsg.append(e.getMessage()); + logger.error("Error reading catalog status", e); + } + catalogMongoDBTime.stop(); + + StopWatch storageTime = StopWatch.createStarted(); + try { + storageEngineFactory.getVariantStorageEngine(null, configuration.getDatabasePrefix() + "_test_connection", "test_connection") + .testConnection(); + status.setVariantStorageId(storageEngineFactory.getVariantStorageEngine().getStorageEngineId()); + status.setVariantStorageStatus(HealthCheckStatus.Status.OK); + } catch (Exception e) { + status.setVariantStorageStatus(HealthCheckStatus.Status.KO); + healthy = false; + errorMsg.append(e.getMessage()); + logger.error("Error reading variant storage status", e); + } + storageTime.stop(); + + StopWatch solrEngineTime = StopWatch.createStarted(); + if (storageEngineFactory.getStorageConfiguration().getSearch().isActive()) { + try { + if (variantManager.isSolrAvailable()) { + status.setSolrStatus(HealthCheckStatus.Status.OK); + } else { + errorMsg.append(", unable to connect with solr, "); + status.setSolrStatus(HealthCheckStatus.Status.KO); + healthy = false; + } + } catch (Exception e) { + status.setSolrStatus(HealthCheckStatus.Status.KO); + healthy = false; + errorMsg.append(e.getMessage()); + logger.error("Error reading solr status", e); + } + } else { + status.setSolrStatus(HealthCheckStatus.Status.NA); + } + solrEngineTime.stop(); + totalTime.stop(); + + if (totalTime.getTime(TimeUnit.SECONDS) > 5) { + logger.warn("Slow OpenCGA status: Updated time: {}. Catalog: {} , Storage: {} , Solr: {}", + totalTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, + catalogMongoDBTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, + storageTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, + solrEngineTime.getTime(TimeUnit.MILLISECONDS) / 1000.0 + ); + } + + if (errorMsg.length() == 0) { + status.setErrorMessage(null); + } else { + status.setErrorMessage(errorMsg.toString()); + } + + status.setCreationDate(LocalDateTime.now()); + status.setHealthy(healthy); + cachedResult.set(status); + } + +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java index d814ac76f1b..74874049837 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java @@ -70,6 +70,7 @@ public static void main(String[] args) throws URISyntaxException { clientsGenerator.cli(flatRestApi); } catch (Exception e) { logger.error(e.getMessage(), e); + System.exit(1); } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java index 96b13ca4292..ea7f6fe917b 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java @@ -1,6 +1,8 @@ package org.opencb.opencga.server.generator.config; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; public class CategoryConfig { @@ -186,6 +188,21 @@ public boolean isAvailableCommand(String commandName) { return true; } + public Command getCommandOrFail(String commandName) { + Command command = getCommand(commandName); + if (command == null) { + List availableCommands; + if (commands == null) { + availableCommands = Collections.emptyList(); + } else { + availableCommands = commands.stream().map(Command::getName).collect(Collectors.toList()); + } + throw new IllegalArgumentException("Command '" + commandName + "' not found. Given category '" + name + "'" + + ", available sub-commands are: " + availableCommands + "."); + } + return command; + } + public Command getCommand(String commandName) { if (commands != null) { for (Command cmd : commands) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java index 72857bc2aec..86f1bcda615 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.server.generator.config.CategoryConfig; +import org.opencb.opencga.server.generator.config.Command; import org.opencb.opencga.server.generator.config.CommandLineConfiguration; import org.opencb.opencga.server.generator.models.RestApi; import org.opencb.opencga.server.generator.models.RestCategory; @@ -186,8 +187,6 @@ public static String getCommandName(RestCategory restCategory, RestEndpoint rest } protected static String getMethodName(RestCategory restCategory, RestEndpoint restEndpoint) { - - String methodName = ""; String subpath = restEndpoint.getPath().replace(restCategory.getPath() + "/", ""); return getMethodName(subpath); } @@ -298,4 +297,61 @@ public boolean isValidMap(RestParameter bodyRestParameter) { || "java.util.Map".equals(bodyRestParameter.getGenericType()) || "java.util.Map".equals(bodyRestParameter.getGenericType()); } + + protected String getExecutorVarName(RestCategory restCategory) { + return "custom" + getAsClassName(restCategory.getName()) + "CommandExecutor"; + } + + protected String getExecutorClassName(RestCategory restCategory, CategoryConfig categoryConfig, String commandName) { + String executorClassName; + Command command = categoryConfig.getCommandOrFail(commandName); + if (StringUtils.isNotEmpty(command.getExecutorExtendedClassName())) { + executorClassName = command.getExecutorExtendedClassName(); + } else { + executorClassName = "Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor"; + } + return executorClassName; + } + + protected String getCommandOptionsClassName(RestCategory restCategory, CategoryConfig categoryConfig, String commandName) { + String commandOptionsClassName; + if (categoryConfig.isExtendedOptionCommand(commandName)) { + commandOptionsClassName = "Custom" + getAsClassName(restCategory.getName()) + "CommandOptions"; + } else { + commandOptionsClassName = getAsClassName(restCategory.getName()) + "CommandOptions"; + } + return commandOptionsClassName; + } + + protected String getCommandOptionsClassName(RestCategory restCategory) { + return getAsClassName(restCategory.getName()) + "CommandOptions"; + } + + protected String getSubCommandOptionsClassName(CategoryConfig categoryConfig, String commandName) { +// String methodName = getJavaMethodName(categoryConfig, commandName); + String methodName = getAsCamelCase(commandName); + String subCommandOptionsClassName = getAsClassName(methodName + "CommandOptions"); + return subCommandOptionsClassName; + } + + protected String getCommandOptionsVarName(RestCategory restCategory) { + return getAsVariableName(getAsCamelCase(restCategory.getName())) + "CommandOptions"; + } + + protected String getSubCommandOptionsVarName(CategoryConfig categoryConfig, String commandName) { +// String methodName = getJavaMethodName(categoryConfig, commandName); + String methodName = getAsCamelCase(commandName); + return methodName + "CommandOptions"; + } + + protected String getJavaMethodName(CategoryConfig config, String commandName) { + Command command = config.getCommand(commandName); + String commandMethod; + if (command != null && StringUtils.isNotEmpty(command.getRename())) { + commandMethod = command.getRename(); + } else { + commandMethod = getAsCamelCase(commandName); + } + return commandMethod; + } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java index f75776a7dfa..d0c830abd3d 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.text.SimpleDateFormat; import java.util.*; public class ExecutorsCliRestApiWriter extends ParentClientRestApiWriter { @@ -44,48 +43,48 @@ public ExecutorsCliRestApiWriter(RestApi restApi, CommandLineConfiguration confi @Override protected String getClassImports(String key) { - StringBuilder sb = new StringBuilder(); RestCategory restCategory = availableCategories.get(key); CategoryConfig categoryConfig = availableCategoryConfigs.get(key); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - sb.append("package ").append(config.getOptions().getExecutorsPackage()).append(";\n\n"); - sb.append("import com.fasterxml.jackson.databind.DeserializationFeature;\n"); +// SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Set imports = new TreeSet<>(); + imports.add("com.fasterxml.jackson.databind.DeserializationFeature"); if (StringUtils.isEmpty(config.getApiConfig().getExecutorsParentClass())) { - sb.append("import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor;\n"); + imports.add("org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor"); } else { - sb.append("import " + config.getApiConfig().getExecutorsParentClass() + ";\n"); + imports.add(config.getApiConfig().getExecutorsParentClass()); } - sb.append("import org.opencb.opencga.app.cli.main.*;\n"); - sb.append("import org.opencb.opencga.core.response.RestResponse;\n"); - sb.append("import org.opencb.opencga.client.exceptions.ClientException;\n"); - sb.append("import org.opencb.commons.datastore.core.ObjectMap;\n\n"); - sb.append("import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException;\n"); - sb.append("import org.opencb.opencga.core.common.JacksonUtils;\n\n"); - sb.append("import com.fasterxml.jackson.databind.ObjectMapper;\n"); - sb.append("import java.util.List;\n"); - sb.append("import java.util.HashMap;\n"); - sb.append("import org.opencb.opencga.core.response.QueryType;\n"); - sb.append("import org.opencb.commons.utils.PrintUtils;\n\n"); + imports.add("com.fasterxml.jackson.databind.DeserializationFeature"); + imports.add("org.opencb.opencga.app.cli.main.*"); + imports.add("org.opencb.opencga.core.response.RestResponse"); + imports.add("org.opencb.opencga.client.exceptions.ClientException"); + imports.add("org.opencb.commons.datastore.core.ObjectMap"); + imports.add("org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException"); + imports.add("org.opencb.opencga.core.common.JacksonUtils"); + imports.add("com.fasterxml.jackson.databind.ObjectMapper"); + imports.add("java.util.List"); + imports.add("java.util.HashMap"); + imports.add("org.opencb.opencga.core.response.QueryType"); + imports.add("org.opencb.commons.utils.PrintUtils"); + // Add custom parent class - sb.append("import " + config.getOptions().getOptionsPackage() + "." + getAsClassName(restCategory.getName()) + "CommandOptions;\n\n"); + imports.add(config.getOptions().getOptionsPackage() + "." + getCommandOptionsClassName(restCategory)); if (categoryConfig.isExecutorExtended()) { for (RestEndpoint restEndpoint : restCategory.getEndpoints()) { String commandName = getCommandName(restCategory, restEndpoint); - if (categoryConfig.getCommand(commandName) != null && StringUtils.isNotEmpty(categoryConfig.getCommand(commandName).getExecutorExtendedClassName())) { - sb.append("import " + categoryConfig.getCommand(commandName).getExecutorExtendedClassName() + ";\n"); + Command command = categoryConfig.getCommand(commandName); + if (command != null && StringUtils.isNotEmpty(command.getExecutorExtendedClassName())) { + imports.add(command.getExecutorExtendedClassName()); } else { - sb.append("import org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor;\n"); + imports.add("org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor"); } if (categoryConfig.isExtendedOptionCommand(commandName)) { - sb.append("import org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandOptions;\n"); - + imports.add("org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandOptions"); } } } - Set imports = new TreeSet<>(); for (RestEndpoint restEndpoint : restCategory.getEndpoints()) { if (isValidImport(restEndpoint.getResponseClass())) { imports.add(restEndpoint.getResponseClass().replaceAll("\\$", "\\.")); @@ -99,7 +98,7 @@ protected String getClassImports(String key) { if (bodyParam.isComplex() && !bodyParam.isCollection()) { if (bodyParam.getTypeClass() != null) { if (bodyParam.getTypeClass().contains("$")) { - imports.add(bodyParam.getTypeClass().substring(0, bodyParam.getTypeClass().lastIndexOf("$")) + ";"); + imports.add(bodyParam.getTypeClass().substring(0, bodyParam.getTypeClass().lastIndexOf("$"))); } else { imports.add(bodyParam.getTypeClass().replaceAll("\\$", "\\.")); } @@ -115,8 +114,17 @@ protected String getClassImports(String key) { } } + StringBuilder sb = new StringBuilder(); + sb.append("package ").append(config.getOptions().getExecutorsPackage()).append(";\n\n"); + for (String string : imports) { - sb.append("import ").append(string).append("\n"); + if (string.endsWith(";")) { + string = string.substring(0, string.length() - 1); + if (imports.contains(string)) { + continue; + } + } + sb.append("import ").append(string).append(";\n"); } sb.append("\n"); @@ -170,8 +178,7 @@ protected String getClassHeader(String key) { CategoryConfig config = availableCategoryConfigs.get(key); sb.append("public class " + getAsClassName(restCategory.getName()) + "CommandExecutor extends " + getExtendedClass() + " {\n\n"); - sb.append(" public " + getAsClassName(restCategory.getName()) + "CommandOptions " - + getAsVariableName(getAsCamelCase(restCategory.getName())) + "CommandOptions;\n\n"); + sb.append(" public " + getCommandOptionsClassName(restCategory) + " " + getCommandOptionsVarName(restCategory) + ";\n\n"); sb.append(" public " + getAsClassName(restCategory.getName()) + "CommandExecutor(" + getAsClassName(restCategory.getName()) + "CommandOptions " + getAsVariableName(getAsCamelCase(restCategory.getName())) + "CommandOptions) throws CatalogAuthenticationException {\n"); @@ -203,19 +210,24 @@ private String methodExecute(RestCategory restCategory, CategoryConfig categoryC } // } } - /* if (CollectionUtils.isNotEmpty(categoryConfig.getAddedMethods())) { - for (String methodName : categoryConfig.getAddedMethods()) { - sb.append(" case \"" + methodName + "\":\n"); - if (categoryConfig.getCommand(methodName) != null && StringUtils.isNotEmpty(categoryConfig.getCommand(methodName).getExecutorExtendedClassName())) { - sb.append(" " + categoryConfig.getCommand(methodName).getExecutorExtendedClassName() + " custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new " + categoryConfig.getCommand(methodName).getExecutorExtendedClassName() + "();\n"); - } else { - sb.append(" Parent" + getAsClassName(restCategory.getName()) + "CommandExecutor custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new Parent" + getAsClassName(restCategory.getName()) + "CommandExecutor();\n"); + if (CollectionUtils.isNotEmpty(categoryConfig.getAddedMethods())) { + for (String commandName : categoryConfig.getAddedMethods()) { - } - sb.append(" queryResponse = custom" + getAsClassName(restCategory.getName()) + "CommandExecutor." + getAsCamelCase(methodName) + "();\n"); + String executorClassName = getExecutorClassName(restCategory, categoryConfig, commandName); + String executorVarName = getExecutorVarName(restCategory); + String javaMethodName = getJavaMethodName(categoryConfig, commandName); + + String commandOptions = getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(categoryConfig, commandName); + String commonOptionsParams = getCommandOptionsVarName(restCategory) + ".commonCommandOptions.params"; + + sb.append(" case \"" + commandName + "\":\n"); + sb.append(" " + executorClassName + " " + executorVarName + " = new " + executorClassName + "(\n"); + sb.append(" new ObjectMap(" + commonOptionsParams + "), token,\n"); + sb.append(" clientConfiguration, getSessionManager(), appHome, getLogger());\n"); + sb.append(" queryResponse = " + executorVarName + "." + javaMethodName + "(" + commandOptions + ");\n"); sb.append(" break;\n"); } - }*/ + } sb.append(" default:\n"); sb.append(" logger.error(\"Subcommand not valid\");\n"); sb.append(" break;\n"); @@ -243,19 +255,14 @@ protected String getClassMethods(String key) { sb.append("\n"); sb.append(" private RestResponse<" + getValidResponseNames(restEndpoint.getResponse()) + "> " + getJavaMethodName(categoryConfig, commandName) + "() throws Exception {\n"); - sb.append(" logger.debug(\"Executing " + getAsCamelCase(commandName) + " in " + sb.append(" logger.debug(\"Executing " + getJavaMethodName(categoryConfig, commandName) + " in " + restCategory.getName() + " command line\");\n\n"); - if (categoryConfig.isExtendedOptionCommand(commandName)) { - sb.append(" Custom" + getAsClassName(restCategory.getName()) + "CommandOptions." + getAsClassName(getAsCamelCase(commandName)) - + "CommandOptions commandOptions = " + getAsVariableName(getAsCamelCase(restCategory.getName())) + - "CommandOptions." - + getAsCamelCase(commandName) + "CommandOptions;\n"); - } else { - sb.append(" " + getAsClassName(restCategory.getName()) + "CommandOptions." + getAsClassName(getAsCamelCase(commandName)) - + "CommandOptions commandOptions = " + getAsVariableName(getAsCamelCase(restCategory.getName())) + - "CommandOptions." - + getAsCamelCase(commandName) + "CommandOptions;\n"); - } + + String commandOptionsClassName = getCommandOptionsClassName(restCategory, categoryConfig, commandName) + "." + getSubCommandOptionsClassName(categoryConfig, commandName); + String commandOptionsLocalVarName = "commandOptions"; + String commandOptionsVarName = getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(categoryConfig, commandName); + + sb.append(" " + commandOptionsClassName + " " + commandOptionsLocalVarName + " = " + commandOptionsVarName + ";\n"); if (categoryConfig.isExecutorExtendedCommand(commandName)) { List body = new ArrayList<>(); for (RestParameter restParameter : restEndpoint.getParameters()) { @@ -306,13 +313,11 @@ protected String getClassMethods(String key) { } } - if (StringUtils.isNotEmpty(categoryConfig.getCommand(commandName).getExecutorExtendedClassName())) { - sb.append(" " + categoryConfig.getCommand(commandName).getExecutorExtendedClassName() + " custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new " + categoryConfig.getCommand(commandName).getExecutorExtendedClassName() + "(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());\n"); - } else { - sb.append(" Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());\n"); - } + String executorClassName = getExecutorClassName(restCategory, categoryConfig, commandName); + String executorVarName = getExecutorVarName(restCategory); - sb.append(" return custom" + getAsClassName(restCategory.getName()) + "CommandExecutor." + getAsCamelCase(commandName) + "();\n"); + sb.append(" " + executorClassName + " " + executorVarName + " = new " + executorClassName + "(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());\n"); + sb.append(" return " + executorVarName + "." + getJavaMethodName(categoryConfig, commandName) + "(commandOptions);\n"); } else { sb.append(getQueryParams(restEndpoint, categoryConfig, commandName)); sb.append(getBodyParams(restCategory, restEndpoint, categoryConfig, commandName)); @@ -351,15 +356,6 @@ private String getReturn(RestCategory restCategory, RestEndpoint restEndpoint, C return res; } - private String getJavaMethodName(CategoryConfig config, String commandName) { - Command command = config.getCommand(commandName); - String commandMethod = getAsCamelCase(commandName); - if (command != null && StringUtils.isNotEmpty(command.getRename())) { - commandMethod = command.getRename(); - } - return commandMethod; - } - private String getBodyParams(RestCategory restCategory, RestEndpoint restEndpoint, CategoryConfig config, String commandName) { StringBuilder sb = new StringBuilder(); String bodyClassName = restEndpoint.getBodyClassName(); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java index 80c9a5fcdd7..bfa784319b7 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java @@ -114,15 +114,16 @@ protected String getClassHeader(String key) { String commandName = getCommandName(restCategory, restEndpoint); // if ("POST".equals(restEndpoint.getMethod()) || restEndpoint.hasParameters()) { if (config.isAvailableCommand(commandName)) { - sb.append(" public " + getAsClassName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions " - + getAsVariableName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions;\n"); + String className = getSubCommandOptionsClassName(config, commandName); + String varName = getSubCommandOptionsVarName(config, commandName); + sb.append(" public " + className + " " + varName + ";\n"); } // } } sb.append("\n"); sb.append("\n"); - sb.append(" public " + getAsClassName(restCategory.getName()) + "CommandOptions(CommonCommandOptions commonCommandOptions, " + + sb.append(" public " + getCommandOptionsClassName(restCategory) + "(CommonCommandOptions commonCommandOptions, " + "JCommander jCommander) {\n"); sb.append(" \n"); if (config.isOptionExtended()) { @@ -135,9 +136,9 @@ protected String getClassHeader(String key) { String commandName = getCommandName(restCategory, restEndpoint); // if ("POST".equals(restEndpoint.getMethod()) || restEndpoint.hasParameters()) { if (config.isAvailableCommand(commandName)) { - sb.append(" this." + getAsVariableName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions = " + - "new " - + getAsClassName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions();\n"); + String varName = getSubCommandOptionsVarName(config, commandName); + String className = getSubCommandOptionsClassName(config, commandName); + sb.append(" this." + varName + " = " + "new " + className + "();\n"); } // } } @@ -158,7 +159,7 @@ protected String getClassMethods(String key) { if (config.isAvailableCommand(commandName) && !config.isExtendedOptionCommand(commandName)) { sb.append(" @Parameters(commandNames = {\"" + reverseCommandName(commandName) + "\"}, commandDescription =\"" + restEndpoint.getDescription().replaceAll("\"", "'") + "\")\n"); - sb.append(" public class " + getAsClassName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions " + + sb.append(" public class " + getSubCommandOptionsClassName(config, commandName) + " " + "{\n"); sb.append(" \n"); sb.append(" @ParametersDelegate\n"); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java index 828d12be889..dd4717b0fee 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java @@ -116,17 +116,15 @@ protected String getClassHeader(String key) { // if ("POST".equals(restEndpoint.getMethod()) || restEndpoint.hasParameters()) { if (config.isAvailableCommand(commandName)) { sb.append(" " + getAsVariableName(restCategory.getName()) + "SubCommands.addCommand(\"" + reverseCommandName(commandName) + "\", " - + getAsVariableName(restCategory.getName()) + "CommandOptions." + getAsCamelCase(commandName) + - "CommandOptions);\n"); + + getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(config, commandName) + ");\n"); } // } } if (CollectionUtils.isNotEmpty(config.getAddedMethods())) { - for (String methodName : config.getAddedMethods()) { - sb.append(" " + getAsVariableName(restCategory.getName()) + "SubCommands.addCommand(\"" + methodName + "\", " - + getAsVariableName(restCategory.getName()) + "CommandOptions." + getAsCamelCase(methodName) + - "CommandOptions);\n"); + for (String commandName : config.getAddedMethods()) { + sb.append(" " + getAsVariableName(restCategory.getName()) + "SubCommands.addCommand(\"" + commandName + "\", " + + getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(config, commandName) + ");\n"); } } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java index 528853faeb6..edb1ab223c1 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java @@ -17,8 +17,6 @@ package org.opencb.opencga.server.rest; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.Event; import org.opencb.commons.utils.DataModelsUtils; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.exceptions.VersionException; @@ -26,6 +24,7 @@ import org.opencb.opencga.core.tools.annotations.Api; import org.opencb.opencga.core.tools.annotations.ApiOperation; import org.opencb.opencga.core.tools.annotations.ApiParam; +import org.opencb.opencga.server.OpenCGAHealthCheckMonitor; import org.opencb.opencga.server.generator.RestApiParser; import org.opencb.opencga.server.generator.models.RestApi; import org.opencb.opencga.server.rest.admin.AdminWSServer; @@ -44,13 +43,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.io.IOException; -import java.time.Duration; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; /** * Created by pfurio on 05/05/17. @@ -60,15 +53,6 @@ @Api(value = "Meta", description = "Meta RESTful Web Services API") public class MetaWSServer extends OpenCGAWSServer { - private static final AtomicReference healthCheckErrorMessage = new AtomicReference<>(); - private static final AtomicReference lastAccess = new AtomicReference<>(LocalTime.now()); - private static final Map healthCheckResults = new ConcurrentHashMap<>(); - private final String OKAY = "OK"; - private final String NOT_OKAY = "KO"; - private final String SOLR = "Solr"; - private final String VARIANT_STORAGE = "VariantStorage"; - private final String CATALOG_MONGO_DB = "CatalogMongoDB"; - public MetaWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { super(uriInfo, httpServletRequest, httpHeaders); @@ -111,116 +95,16 @@ public Response fail() { @Path("/status") @ApiOperation(httpMethod = "GET", value = "Database status.", response = Map.class) public Response status() { + OpenCGAResult queryResult = healthCheckMonitor.getStatus(); + OpenCGAHealthCheckMonitor.HealthCheckStatus status = queryResult.first(); - OpenCGAResult> queryResult = new OpenCGAResult<>(); - StopWatch stopWatch = StopWatch.createStarted(); - - if (shouldUpdateStatus()) { - logger.debug("Update HealthCheck cache status"); - updateHealthCheck(); - } else { - logger.debug("HealthCheck results from cache at " + lastAccess.get().format(DateTimeFormatter.ofPattern("HH:mm:ss"))); - queryResult.setEvents(Collections.singletonList(new Event(Event.Type.INFO, "HealthCheck results from cache at " - + lastAccess.get().format(DateTimeFormatter.ofPattern("HH:mm:ss"))))); - } - - queryResult.setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); - queryResult.setResults(Collections.singletonList(healthCheckResults)); - - if (isHealthy()) { - logger.debug("HealthCheck : " + healthCheckResults.toString()); + if (status.isHealthy()) { + logger.debug("HealthCheck : " + status); return createOkResponse(queryResult); } else { - logger.error("HealthCheck : " + healthCheckResults.toString()); - return createErrorResponse(healthCheckErrorMessage.get(), queryResult); - } - } - - private boolean shouldUpdateStatus() { - if (!isHealthy()) { - // Always update if not healthy - return true; - } - // If healthy, only update every "healthCheck.interval" seconds - long elapsedTime = Duration.between(lastAccess.get(), LocalTime.now()).getSeconds(); - return elapsedTime > configuration.getHealthCheck().getInterval(); - } - - private synchronized void updateHealthCheck() { - if (!shouldUpdateStatus()) { - // Skip update! - return; - } - StringBuilder errorMsg = new StringBuilder(); - - Map newHealthCheckResults = new HashMap<>(); - newHealthCheckResults.put(CATALOG_MONGO_DB, ""); - newHealthCheckResults.put(VARIANT_STORAGE, ""); - newHealthCheckResults.put(SOLR, ""); - - StopWatch totalTime = StopWatch.createStarted(); - StopWatch catalogMongoDBTime = StopWatch.createStarted(); - try { - if (catalogManager.getCatalogDatabaseStatus()) { - newHealthCheckResults.put(CATALOG_MONGO_DB, OKAY); - } else { - newHealthCheckResults.put(CATALOG_MONGO_DB, NOT_OKAY); - } - } catch (Exception e) { - newHealthCheckResults.put(CATALOG_MONGO_DB, NOT_OKAY); - errorMsg.append(e.getMessage()); - } - catalogMongoDBTime.stop(); - - StopWatch storageTime = StopWatch.createStarted(); - try { - storageEngineFactory.getVariantStorageEngine(null, configuration.getDatabasePrefix() + "_test_connection", "test_connection") - .testConnection(); - newHealthCheckResults.put("VariantStorageId", storageEngineFactory.getVariantStorageEngine().getStorageEngineId()); - newHealthCheckResults.put(VARIANT_STORAGE, OKAY); - } catch (Exception e) { - newHealthCheckResults.put(VARIANT_STORAGE, NOT_OKAY); - errorMsg.append(e.getMessage()); -// errorMsg.append(" No storageEngineId is set in configuration or Unable to initiate storage Engine, ").append(e.getMessage() -// ).append(", "); - } - storageTime.stop(); - - StopWatch solrEngineTime = StopWatch.createStarted(); - if (storageEngineFactory.getStorageConfiguration().getSearch().isActive()) { - try { - if (variantManager.isSolrAvailable()) { - newHealthCheckResults.put(SOLR, OKAY); - } else { - errorMsg.append(", unable to connect with solr, "); - newHealthCheckResults.put(SOLR, NOT_OKAY); - } - } catch (Exception e) { - newHealthCheckResults.put(SOLR, NOT_OKAY); - errorMsg.append(e.getMessage()); - } - } else { - newHealthCheckResults.put(SOLR, "solr not active in storage-configuration!"); + logger.error("HealthCheck : " + status); + return createErrorResponse(status.getErrorMessage(), queryResult); } - solrEngineTime.stop(); - - if (totalTime.getTime(TimeUnit.SECONDS) > 5) { - logger.warn("Slow OpenCGA status: Updated time: {}. Catalog: {} , Storage: {} , Solr: {}", - totalTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, - catalogMongoDBTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, - storageTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, - solrEngineTime.getTime(TimeUnit.MILLISECONDS) / 1000.0 - ); - } - - if (errorMsg.length() == 0) { - healthCheckErrorMessage.set(null); - } else { - healthCheckErrorMessage.set(errorMsg.toString()); - } - - healthCheckResults.putAll(newHealthCheckResults); - lastAccess.set(LocalTime.now()); } @GET @@ -270,8 +154,4 @@ public Response api(@ApiParam(value = "List of categories to get API from") @Que RestApi restApi = new RestApiParser().parse(classes, summary); return createOkResponse(new OpenCGAResult<>(0, Collections.emptyList(), 1, Collections.singletonList(restApi.getCategories()), 1)); } - - private boolean isHealthy() { - return healthCheckResults.isEmpty() ? false : !healthCheckResults.values().stream().anyMatch(x -> x.equals(NOT_OKAY)); - } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java index 03a39102d9e..761c0a3bd52 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java @@ -53,6 +53,7 @@ import org.opencb.opencga.core.response.RestResponse; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.core.tools.annotations.ApiParam; +import org.opencb.opencga.server.OpenCGAHealthCheckMonitor; import org.opencb.opencga.server.WebServiceException; import org.opencb.opencga.server.rest.analysis.ClinicalWebService; import org.opencb.opencga.storage.core.StorageEngineFactory; @@ -137,6 +138,8 @@ public static Configuration getConfiguration() { protected static StorageEngineFactory storageEngineFactory; protected static VariantStorageManager variantManager; + protected static OpenCGAHealthCheckMonitor healthCheckMonitor; + private static final int DEFAULT_LIMIT = AbstractManager.DEFAULT_LIMIT; private static final int MAX_LIMIT = AbstractManager.MAX_LIMIT; private static final int MAX_ID_SIZE = 100; @@ -294,6 +297,8 @@ private static void initOpenCGAObjects() { catalogManager = new CatalogManager(configuration); storageEngineFactory = StorageEngineFactory.get(storageConfiguration); variantManager = new VariantStorageManager(catalogManager, storageEngineFactory); + healthCheckMonitor = new OpenCGAHealthCheckMonitor(configuration, catalogManager, storageEngineFactory, variantManager); + healthCheckMonitor.asyncUpdate(); MigrationSummary migrationSummary = catalogManager.getMigrationManager().getMigrationSummary(); if (migrationSummary.getMigrationsToBeApplied() > 0) { @@ -304,7 +309,6 @@ private static void initOpenCGAObjects() { } } } - } catch (Exception e) { errorMessage = e.getMessage(); // e.printStackTrace(); diff --git a/opencga-server/src/main/resources/cli-config.yaml b/opencga-server/src/main/resources/cli-config.yaml index b3a3c302120..4e5c24c5e9d 100644 --- a/opencga-server/src/main/resources/cli-config.yaml +++ b/opencga-server/src/main/resources/cli-config.yaml @@ -97,6 +97,9 @@ apiConfig: - name: top executorExtended: True optionExtended: True + - name: log + executorExtended: True + optionExtended: True addedMethods: - log - name: samples @@ -147,6 +150,7 @@ apiConfig: key: DiseasePanel commands: - name: import + ## Rename needed as `import` is a reserved word in Java rename: importPanels - name: analysisAlignment commandName: alignments diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java index 2550d6f0592..398b7f775cd 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java @@ -94,6 +94,9 @@ public static class GenericVariantIndexOptions { @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped") public boolean load; + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload") + public boolean forceReload; + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: \"basic\" mode merge genotypes of the same variants while \"advanced\" merge multiallelic and overlapping variants.") public String merge; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java index 3c193134518..536d8795d70 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java @@ -18,6 +18,13 @@ */ public class SampleMetadata extends StudyResourceMetadata { + /** + * List of fileIds that contains the sample. + * This list is sorted by the order files were loaded. + * Only files of type {@link FileMetadata.Type#NORMAL} and {@link FileMetadata.Type#VIRTUAL} will be present in this list. + * If the sample was part of a set of {@link FileMetadata.Type#PARTIAL} files, only the + * {@link FileMetadata.Type#VIRTUAL} file will appear in this list. + */ private List files; private Set cohorts; // Prepared to have more than one secondary index per sample. diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java index 07b111f24bf..75a7c58b1d8 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java @@ -10,7 +10,7 @@ * * @author Jacobo Coll <jacobo167@gmail.com> */ -public abstract class StudyResourceMetadata { +public abstract class StudyResourceMetadata> { private int studyId; private int id; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java index 2c12a0021e0..82c85135b74 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java @@ -6,21 +6,42 @@ import java.util.List; import java.util.Objects; +/** + * This class represents a family trio. + * + * - Father and mother can be null + * - Child cannot be null + * - All samples must be unique + * - All samples must be different to NA (dash) + * + */ public class Trio { + public static final String NA = "-"; private final String id; private final String father; private final String mother; private final String child; + public Trio(String trio) { + String[] split = trio.split(","); + if (split.length != 3) { + throw new IllegalArgumentException("Expected three samples in trio '" + trio + "'"); + } + this.id = null; + this.child = split[0]; + this.father = NA.equals(split[1]) ? null : split[1]; + this.mother = NA.equals(split[2]) ? null : split[2]; + } + public Trio(List trio) { this(null, trio); } public Trio(String id, List trio) { this.id = id; - this.father = trio.get(1); - this.mother = trio.get(2); this.child = trio.get(0); + this.father = NA.equals(trio.get(1)) ? null : trio.get(1); + this.mother = NA.equals(trio.get(2)) ? null : trio.get(2); } public Trio(String father, String mother, String child) { @@ -29,8 +50,8 @@ public Trio(String father, String mother, String child) { public Trio(String id, String father, String mother, String child) { this.id = id; - this.father = father; - this.mother = mother; + this.father = NA.equals(father) ? null : father; + this.mother = NA.equals(mother) ? null : mother; this.child = child; } @@ -50,6 +71,11 @@ public String getChild() { return child; } + /** + * Returns a list with the non-null samples in the trio. + * + * @return List of samples + */ public List toList() { ArrayList list = new ArrayList<>(3); list.add(getChild()); @@ -82,6 +108,31 @@ public int hashCode() { return Objects.hash(id, father, mother, child); } + /** + * Serialize the trio into a string contain the three samples separated by commas. + * order: child, father, mother. + * If the father or mother are null, they will be replaced by {@link #NA}. + * + * Can be deserialized using {@link #Trio(String)}. + * + * @return String + */ + public String serialize() { + ArrayList list = new ArrayList<>(3); + list.add(getChild()); + if (getFather() == null) { + list.add(NA); + } else { + list.add(getFather()); + } + if (getMother() == null) { + list.add(NA); + } else { + list.add(getMother()); + } + return Strings.join(list, ','); + } + @Override public String toString() { return Strings.join(toList(), ','); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index a4f560e49f2..a3ab7e539b5 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -140,11 +140,11 @@ public enum SplitData { REGION, MULTI; - public static boolean isPartial(ObjectMap options) { - return isPartial(from(options)); + public static boolean isPartialSplit(ObjectMap options) { + return isPartialSplit(from(options)); } - public static boolean isPartial(SplitData splitData) { + public static boolean isPartialSplit(SplitData splitData) { return splitData == CHROMOSOME || splitData == REGION; } @@ -582,45 +582,52 @@ public void sampleIndexAnnotate(String study, List samples, ObjectMap op * The Family Index is used alongside with the SampleIndex to speed up queries involving children and parents. * * @param study Study - * @param trios List of trios "father, mother, child". - * Missing parents in trios are specified with "-", - * If a family has two children, two trios should be defined. + * @param trios Trios to index. If a family has two children, two trios should be defined. * @param options Other options * @throws StorageEngineException in an error occurs * @return List of trios used to index. Empty if there was nothing to do. */ - public DataResult> familyIndex(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult familyIndex(String study, List trios, ObjectMap options) throws StorageEngineException { throw new UnsupportedOperationException("Unsupported familyIndex"); } - public DataResult> familyIndexUpdate(String study, ObjectMap options) throws StorageEngineException { + /** + * Update the family index. + * The Family Index is used alongside with the SampleIndex to speed up queries involving children and parents. + * + * @param study Study + * @param options Other options + * @throws StorageEngineException in an error occurs + * @return List of trios used to index. Empty if there was nothing to do. + */ + public DataResult familyIndexUpdate(String study, ObjectMap options) throws StorageEngineException { StudyMetadata studyMetadata = getMetadataManager().getStudyMetadata(study); int studyId = studyMetadata.getId(); int version = studyMetadata.getSampleIndexConfigurationLatest().getVersion(); - List> trios = new LinkedList<>(); + List trios = new LinkedList<>(); for (SampleMetadata sampleMetadata : getMetadataManager().sampleMetadataIterable(studyId)) { if (sampleMetadata.isFamilyIndexDefined()) { if (sampleMetadata.getFamilyIndexStatus(version) != TaskMetadata.Status.READY) { // This sample's family index needs to be updated String father; if (sampleMetadata.getFather() == null) { - father = "-"; + father = null; } else { father = getMetadataManager().getSampleName(studyId, sampleMetadata.getFather()); } String mother; if (sampleMetadata.getMother() == null) { - mother = "-"; + mother = null; } else { mother = getMetadataManager().getSampleName(studyId, sampleMetadata.getMother()); } - trios.add(Arrays.asList(father, mother, sampleMetadata.getName())); + trios.add(new Trio(father, mother, sampleMetadata.getName())); } } } if (trios.isEmpty()) { logger.info("Nothing to do!"); - return new DataResult>().setEvents(Collections.singletonList(new Event(Event.Type.INFO, "Nothing to do"))); + return new DataResult().setEvents(Collections.singletonList(new Event(Event.Type.INFO, "Nothing to do"))); } else { return familyIndex(study, trios, options); } @@ -1025,6 +1032,7 @@ protected void postRemoveFiles(String study, List fileIds, List fields, QueryOptions options } public DataResult count(Query query) throws StorageEngineException { - VariantQueryResult result = get(query, new QueryOptions(QueryOptions.INCLUDE, VariantField.ID) - .append(QueryOptions.LIMIT, 1) - .append(QueryOptions.COUNT, true)); + VariantQueryResult result = get(VariantQueryUtils.copy(query), + new QueryOptions(QueryOptions.INCLUDE, VariantField.ID) + .append(QueryOptions.LIMIT, 1) + .append(QueryOptions.COUNT, true)); return new DataResult<>( result.getTime(), result.getEvents(), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 44c063cf17b..19b0631659f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -52,9 +52,9 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.common.YesNoAuto; +import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.storage.core.StoragePipeline; -import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.io.plain.StringDataReader; @@ -172,7 +172,17 @@ public URI preTransform(URI input) throws StorageEngineException, IOException, F } return existingStudyMetadata; }); - if (VariantStorageEngine.SplitData.isPartial(options) + if (options.getBoolean(FORCE.key())) { + Integer fileId = getMetadataManager().getFileId(studyMetadata.getId(), fileName, true); + if (fileId != null) { + // File is indexed. Mark as non indexed. + getMetadataManager().updateFileMetadata(studyMetadata.getId(), fileId, fileMetadata -> { + fileMetadata.setIndexStatus(TaskMetadata.Status.NONE); + }); + logger.info("File '{}' already loaded. Force reload!", fileName); + } + } + if (VariantStorageEngine.SplitData.isPartialSplit(options) && !options.getString(LOAD_VIRTUAL_FILE.key(), "").isEmpty()) { setFileId(smm.registerPartialFile(studyMetadata.getId(), input.getPath())); } else { @@ -549,7 +559,7 @@ protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata f final int fileId; String virtualFile = options.getString(LOAD_VIRTUAL_FILE.key()); - if (VariantStorageEngine.SplitData.isPartial(options)) { + if (VariantStorageEngine.SplitData.isPartialSplit(options)) { if (StringUtils.isEmpty(virtualFile)) { fileId = getMetadataManager().registerFile(studyId, fileMetadata); // throw new StorageEngineException("Unable to load file with 'split-data'. Missing virtual file belonging! " diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java index e7ceeee7970..fe5313bdc80 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java @@ -17,8 +17,11 @@ package org.opencb.opencga.storage.core.variant; import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Test; import org.junit.*; import org.junit.rules.ExpectedException; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.opencb.biodata.formats.io.FileFormatException; import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.ObjectMap; @@ -37,6 +40,7 @@ import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; @@ -115,6 +119,10 @@ public abstract class VariantStorageBaseTest extends GenericTest implements Vari @Rule public ExpectedException thrown = ExpectedException.none(); + @ClassRule + public static TestClassNameWatcher testClassNameWatcher = new TestClassNameWatcher(); + + @BeforeClass public static void _beforeClass() throws Exception { // System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "debug"); @@ -191,7 +199,7 @@ public static URI getPlatinumFile(int fileId) throws IOException { private static void newRootDir() throws IOException { - rootDir = Paths.get("target/test-data", "junit-opencga-storage-" + TimeUtils.getTimeMillis() + "_" + RandomStringUtils.randomAlphabetic(3)); + rootDir = Paths.get("target/test-data", "junit-" + testClassNameWatcher.getTestClassSimpleName() + "-" + TimeUtils.getTimeMillis() + "_" + RandomStringUtils.randomAlphabetic(3)); Files.createDirectories(rootDir); } @@ -199,21 +207,33 @@ public static void setRootDir(Path rootDir) { VariantStorageBaseTest.rootDir = rootDir; } - public static URI newOutputUri() throws IOException { - return newOutputUri(1, outputUri); - } - - public static URI newOutputUri(int extraCalls) throws IOException { - return newOutputUri(1 + extraCalls, outputUri); - } - - public static URI newOutputUri(int extraCalls, URI outputUri) throws IOException { + protected static URI newOutputUri() throws IOException { + String dirName = null; StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - // stackTrace[0] = "Thread.currentThread" - // stackTrace[1] = "newOutputUri" - // stackTrace[2] = caller method - String testName = stackTrace[2 + extraCalls].getMethodName(); - return newOutputUri(testName, outputUri); + for (int i = 0; i < stackTrace.length; i++) { + StackTraceElement element = stackTrace[i]; + try { + Class aClass = Class.forName(element.getClassName()); + Method method = aClass.getMethod(element.getMethodName()); + if (method.isAnnotationPresent(Test.class) + || method.isAnnotationPresent(Before.class) + || method.isAnnotationPresent(BeforeClass.class) + || method.isAnnotationPresent(After.class) + || method.isAnnotationPresent(AfterClass.class)) { + dirName = element.getMethodName(); + break; + } + } catch (ClassNotFoundException | NoSuchMethodException ignore) { + } + } + if (dirName == null) { + // stackTrace[0] = "Thread.currentThread" + // stackTrace[1] = "newOutputUri" + // stackTrace[2] = caller method + dirName = stackTrace[2].getMethodName(); + } + + return newOutputUri(dirName, outputUri); } protected static URI newOutputUri(String testName) throws IOException { @@ -434,4 +454,18 @@ public void printActiveThreads() { } System.out.println("========================================="); } + + private static class TestClassNameWatcher extends TestWatcher { + + private String testClass; + + @Override + protected void starting(Description d) { + this.testClass = d.getTestClass().getSimpleName(); + } + + public String getTestClassSimpleName() { + return this.testClass; + } + } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java index c81c7d21831..55866e24160 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java @@ -28,6 +28,7 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantImporter; @@ -102,7 +103,7 @@ public DummyVariantStoragePipeline newStoragePipeline(boolean connected) throws } @Override - public DataResult> familyIndex(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult familyIndex(String study, List trios, ObjectMap options) throws StorageEngineException { logger.info("Running family index!"); VariantStorageMetadataManager metadataManager = getMetadataManager(); StudyMetadata studyMetadata = metadataManager.getStudyMetadata(study); @@ -121,7 +122,7 @@ public DataResult> familyIndex(String study, List> tri } }); } - return new DataResult>().setResults(trios); + return new DataResult().setResults(trios); } @Override diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java index 031360d3298..7ae1991221d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java @@ -21,20 +21,22 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.core.NodeConfig; -import org.apache.solr.core.SolrResourceLoader; import org.junit.rules.ExternalResource; import org.opencb.commons.datastore.solr.SolrManager; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.DB_NAME; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; /** * + @@ -61,18 +63,19 @@ public VariantSolrExternalResource(boolean embeded) { protected void before() throws Throwable { super.before(); - Path rootDir = getTmpRootDir(); + Path rootDir = Paths.get("target/test-data", "junit-variant-solr-" + TimeUtils.getTimeMillis()); + Files.createDirectories(rootDir); String configSet = "opencga-variant-configset-" + GitRepositoryState.getInstance().getBuildVersion(); // Copy configuration - getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml"); - getResourceUri("managed-schema", "configsets/" + configSet + "/managed-schema"); - getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json"); - getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt"); - getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt"); - getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt"); - getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt"); + getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml", rootDir); + getResourceUri("managed-schema", "configsets/" + configSet + "/managed-schema", rootDir); + getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json", rootDir); + getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt", rootDir); + getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt", rootDir); String solrHome = rootDir.resolve("solr").toString(); diff --git a/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz new file mode 100644 index 00000000000..798a4bf531f Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz new file mode 100644 index 00000000000..2206202e390 Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz new file mode 100644 index 00000000000..a2557925cdd Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz differ diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java index 063219f7654..5c3d0e7a0d1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java @@ -513,49 +513,44 @@ public URI postLoad(URI input, URI output) throws StorageEngineException { metadataManager.setStatus(getStudyId(), taskId, Status.READY); boolean loadSampleIndex = YesNoAuto.parse(getOptions(), LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); + int updatedSamples = 0; for (Integer sampleId : metadataManager.getSampleIdsFromFileId(getStudyId(), getFileId())) { // Worth to check first to avoid too many updates in scenarios like 1000G SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), sampleId); boolean updateSampleIndexStatus = loadSampleIndex && sampleMetadata.getSampleIndexStatus(sampleIndexVersion) != Status.READY; int actualLargestVariantLength = sampleMetadata.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH); - boolean alreadyLoadedFiles = false; - if (sampleMetadata.getFiles().size() > 1) { - int loadedFiles = 0; - for (Integer fileId : sampleMetadata.getFiles()) { - if (metadataManager.isFileIndexed(getStudyId(), fileId)) { - loadedFiles++; - } - } - if (loadedFiles > 1) { - alreadyLoadedFiles = true; - } - metadataManager.getFileIdsFromSampleId(1, 1, true); - } + boolean isLargestVariantLengthDefined = sampleMetadata.getAttributes() + .containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH); + boolean unknownLargestVariantLength = sampleMetadata.getAttributes() + .getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH); + boolean updateLargestVariantLength; - if (alreadyLoadedFiles) { - if (actualLargestVariantLength > 0) { - // Already loaded files, with a valid value. Update if needed. - updateLargestVariantLength = largestVariantLength > actualLargestVariantLength; - } else { - // Already loaded files without a valid value. Do not set a value, as it might be smaller than previous files. - updateLargestVariantLength = false; - } + if (isLargestVariantLengthDefined) { + // Update only if the new value is bigger than the current one + updateLargestVariantLength = largestVariantLength > actualLargestVariantLength; + } else if (unknownLargestVariantLength) { + // Already loaded files with unknown largest variant length. Do not update value. + updateLargestVariantLength = false; } else { // First file loaded. Update value updateLargestVariantLength = true; } if (updateSampleIndexStatus || updateLargestVariantLength) { + updatedSamples++; metadataManager.updateSampleMetadata(getStudyId(), sampleId, s -> { if (updateSampleIndexStatus) { s.setSampleIndexStatus(Status.READY, sampleIndexVersion); } if (updateLargestVariantLength) { - s.getAttributes().put(SampleIndexSchema.LARGEST_VARIANT_LENGTH, largestVariantLength); + int current = s.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, largestVariantLength); + s.getAttributes().put(SampleIndexSchema.LARGEST_VARIANT_LENGTH, Math.max(current, largestVariantLength)); } }); } } + getLoadStats().put("updatedSampleMetadata", updatedSamples); + logger.info("Updated status of {} samples", updatedSamples); boolean loadArchive = YesNoAuto.parse(getOptions(), LOAD_ARCHIVE.key()).orYes().booleanValue(); if (loadArchive) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index c3afc1f5040..917b679cf1c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -46,10 +46,7 @@ import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.metadata.VariantMetadataFactory; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -400,7 +397,7 @@ public void sampleIndexAnnotate(String study, List samples, ObjectMap op @Override - public DataResult> familyIndex(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult familyIndex(String study, List trios, ObjectMap options) throws StorageEngineException { options = getMergedOptions(options); return new FamilyIndexLoader(getSampleIndexDBAdaptor(), getDBAdaptor(), getMRExecutor()) .load(study, trios, options); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java index a5a6e56ef14..4d3718a6250 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java @@ -376,7 +376,7 @@ public boolean getOverlappingVariants(Variant variant, int fileId, List> overlappingRecords) { String chromosome = vcfSlice.getChromosome(); int position = vcfSlice.getPosition(); - Integer resetPosition = null; + Integer resetIteratorIndex = null; boolean isAlreadyPresent = false; int firstIndex = iterator.nextIndex(); // Assume sorted VcfRecords @@ -388,8 +388,8 @@ public boolean getOverlappingVariants(Variant variant, int fileId, String alternate = vcfRecord.getAlternate(); // If the VcfRecord starts after the variant, stop looking for variants if (overlapsWith(variant, chromosome, start, end)) { - if (resetPosition == null) { - resetPosition = Math.max(iterator.previousIndex() - 1, firstIndex); + if (resetIteratorIndex == null) { + resetIteratorIndex = Math.max(iterator.previousIndex() - 1, firstIndex); } // if (skipReferenceVariants && hasAllReferenceGenotype(vcfSlice, vcfRecord)) { @@ -406,8 +406,8 @@ public boolean getOverlappingVariants(Variant variant, int fileId, overlappingRecords.add(ImmutablePair.of(vcfSlice, vcfRecord)); } else if (isRegionAfterVariantStart(start, end, variant)) { - if (resetPosition == null) { - resetPosition = Math.max(iterator.previousIndex() - 1, firstIndex); + if (resetIteratorIndex == null) { + resetIteratorIndex = Math.max(iterator.previousIndex() - 1, firstIndex); } // Shouldn't happen that the first VcfRecord from the iterator is beyond the variant to process, // and is not the first VcfRecord from the slice. @@ -429,14 +429,15 @@ public boolean getOverlappingVariants(Variant variant, int fileId, } } } - if (resetPosition == null && !iterator.hasNext()) { + if (resetIteratorIndex == null && !iterator.hasNext()) { // If the iterator reaches the end without finding any point, reset the iterator - resetPosition = firstIndex; + resetIteratorIndex = firstIndex; } // Send back the iterator - if (resetPosition != null) { -// logger.info("Reset from " + iterator.nextIndex() + " to " + resetPosition + ". fileId : " + fileId + " variant " + variant); - while (iterator.nextIndex() > resetPosition) { + if (resetIteratorIndex != null) { +// logger.info("Reset from " + iterator.nextIndex() + " to " + resetIteratorIndex +// + ". fileId : " + fileId + " variant " + variant); + while (iterator.nextIndex() > resetIteratorIndex) { iterator.previous(); } } @@ -506,10 +507,19 @@ protected LinkedHashMap getSamplePosition(Integer fileId) { }); } + /** + * Checks if the given region is entirely or partially (overlapping) after the variant start. + * + * @param start Start of a region + * @param end End of a region (might be before the start if the region represented an insertion) + * @param variant Variant to check + * @return if the region is after the variant + */ public static boolean isRegionAfterVariantStart(int start, int end, Variant variant) { - int pos = Math.min(start, end); - int variantPos = Math.min(variant.getStart(), variant.getEnd()); - return pos > variantPos; + // Get region max position. In the region represents an insertion, the start might be after the end + int regionMaxPos = Math.max(start, end); + int variantMinPos = Math.min(variant.getStart(), variant.getEnd()); + return regionMaxPos > variantMinPos; } public static boolean overlapsWith(Variant variant, String chromosome, int start, int end) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 5829a9580a8..510085420dd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -248,11 +248,7 @@ private boolean isIncludeCovered(SampleIndexQuery sampleIndexQuery, Query inputQ return false; } - List sampleDataKeys = VariantQueryUtils.getIncludeSampleData(inputQuery); - if (sampleDataKeys == null) { - // Undefined, get default sampleDataKeys - sampleDataKeys = HBaseToVariantConverter.getFixedFormat(study.getStudyMetadata()); - } + List sampleDataKeys = getSampleDataKeys(inputQuery, study); if (sampleDataKeys.size() != 1) { // One and only one sampledatakey @@ -490,8 +486,9 @@ private class AddMissingDataTask implements Task { for (Integer fileId : fileIds) { filesFromSample.add(metadataManager.getFileName(studyId, fileId)); } - List includeSampleData = VariantQueryUtils.getIncludeSampleData(parsedQuery.getInputQuery()); - gtIdx = includeSampleData.indexOf("GT"); + + List sampleDataKeys = getSampleDataKeys(parsedQuery.getInputQuery(), parsedQuery.getProjection().getStudy(studyId)); + gtIdx = sampleDataKeys.indexOf("GT"); } @Override @@ -657,4 +654,13 @@ private void mergeFileEntries(StudyEntry studyEntry, List newFileEntr } } } + + private List getSampleDataKeys(Query parsedQuery, VariantQueryProjection.StudyVariantQueryProjection parsedQuery1) { + List sampleDataKeys = VariantQueryUtils.getIncludeSampleData(parsedQuery); + if (sampleDataKeys == null) { + // Undefined, get default sampleDataKeys + sampleDataKeys = HBaseToVariantConverter.getFixedFormat(parsedQuery1.getStudyMetadata()); + } + return sampleDataKeys; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java index 36f17f5f3a1..ada52270419 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java @@ -17,6 +17,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.hadoop.variant.AbstractVariantsTableDriver; import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; @@ -98,28 +99,28 @@ protected void parseAndValidateParameters() throws IOException { if (StringUtils.isNotEmpty(triosStr)) { String[] trios = triosStr.split(";"); List trioList = new ArrayList<>(3); - for (String trio : trios) { - for (String sample : trio.split(",")) { - Integer sampleId; - if (sample.equals("-")) { - sampleId = MISSING_SAMPLE; - } else { - sampleId = metadataManager.getSampleId(getStudyId(), sample); - if (sampleId == null) { - throw new IllegalArgumentException("Sample '" + sample + "' not found."); - } - } - trioList.add(sampleId); + for (String trioString : trios) { + Trio trio = new Trio(Arrays.asList(trioString.split(","))); + + if (trio.getFather() == null) { + trioList.add(MISSING_SAMPLE); + } else { + trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getFather())); } - if (trioList.size() != 3) { - throw new IllegalArgumentException("Found trio with " + trioList.size() + " members, instead of 3: " + trioList); + if (trio.getMother() == null) { + trioList.add(MISSING_SAMPLE); + } else { + trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getMother())); } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), trioList.get(2)); + int childId = metadataManager.getSampleIdOrFail(getStudyId(), trio.getChild()); + trioList.add(childId); + + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), childId); if (!overwrite && sampleMetadata.getFamilyIndexStatus(sampleIndexVersion) == TaskMetadata.Status.READY) { LOGGER.info("Skip sample " + sampleMetadata.getName() + ". Already precomputed!"); } else { sampleIds.addAll(trioList); - LOGGER.info("Trio: " + trio + " -> " + trioList); + LOGGER.info("Trio: " + trioString + " -> " + trioList); } trioList.clear(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java index 244945ce71e..17083348c45 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java @@ -9,6 +9,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; @@ -17,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -38,9 +38,9 @@ public FamilyIndexLoader(SampleIndexDBAdaptor sampleIndexDBAdaptor, VariantHadoo this.mrExecutor = mrExecutor; } - public DataResult> load(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult load(String study, List trios, ObjectMap options) throws StorageEngineException { trios = new LinkedList<>(trios); - DataResult> dr = new DataResult<>(); + DataResult dr = new DataResult<>(); dr.setResults(trios); dr.setEvents(new LinkedList<>()); @@ -55,35 +55,34 @@ public DataResult> load(String study, List> trios, Obj options.put(FamilyIndexDriver.SAMPLE_INDEX_VERSION, version); options.put(FamilyIndexDriver.OUTPUT, sampleIndexDBAdaptor.getSampleIndexTableName(studyId, version)); - Iterator> iterator = trios.iterator(); + Iterator iterator = trios.iterator(); while (iterator.hasNext()) { - List trioIds = new ArrayList<>(3); - List trio = iterator.next(); - for (String sample : trio) { - Integer sampleId; - if (sample.equals("-")) { - sampleId = -1; - } else { - sampleId = metadataManager.getSampleId(studyId, sample); - if (sampleId == null) { - throw new IllegalArgumentException("Sample '" + sample + "' not found."); - } - } - trioIds.add(sampleId); + Trio trio = iterator.next(); + + final Integer fatherId; + final Integer motherId; + final Integer childId; + + childId = metadataManager.getSampleId(studyId, trio.getChild()); + if (trio.getFather() == null) { + fatherId = -1; + } else { + fatherId = metadataManager.getSampleIdOrFail(studyId, trio.getFather()); } - if (trioIds.size() != 3) { - throw new IllegalArgumentException("Found trio with " + trioIds.size() + " members, instead of 3: " + trioIds); + if (trio.getMother() == null) { + motherId = -1; + } else { + motherId = metadataManager.getSampleIdOrFail(studyId, trio.getMother()); } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, trioIds.get(2)); + + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, childId); if (!overwrite && sampleMetadata.getFamilyIndexStatus(version) == TaskMetadata.Status.READY) { String msg = "Skip sample " + sampleMetadata.getName() + ". Already precomputed!"; logger.info(msg); dr.getEvents().add(new Event(Event.Type.INFO, msg)); iterator.remove(); } else { - Integer fatherId = trioIds.get(0); boolean fatherDefined = fatherId != -1; - Integer motherId = trioIds.get(1); boolean motherDefined = motherId != -1; if (fatherDefined && !fatherId.equals(sampleMetadata.getFather()) || motherDefined && !motherId.equals(sampleMetadata.getMother())) { @@ -106,14 +105,14 @@ public DataResult> load(String study, List> trios, Obj int batchSize = options.getInt(HadoopVariantStorageOptions.SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR.key(), HadoopVariantStorageOptions.SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR.defaultValue()); - List>> batches = BatchUtils.splitBatches(trios, batchSize); + List> batches = BatchUtils.splitBatches(trios, batchSize); if (batches.size() == 1) { runBatch(study, trios, options, studyId); } else { logger.warn("Unable to run family index in one single MapReduce operation."); logger.info("Split in {} jobs of {} samples each.", batches, batches.get(0).size()); for (int i = 0; i < batches.size(); i++) { - List> batch = batches.get(i); + List batch = batches.get(i); logger.info("Running MapReduce {}/{} over {} trios", i + 1, batches, batch.size()); runBatch(study, batch, options, studyId); } @@ -122,12 +121,14 @@ public DataResult> load(String study, List> trios, Obj return dr; } - private void runBatch(String study, List> trios, ObjectMap options, int studyId) throws StorageEngineException { + private void runBatch(String study, List trios, ObjectMap options, int studyId) throws StorageEngineException { if (trios.size() < 500) { - options.put(FamilyIndexDriver.TRIOS, trios.stream().map(trio -> String.join(",", trio)).collect(Collectors.joining(";"))); + options.put(FamilyIndexDriver.TRIOS, trios.stream() + .map(Trio::serialize) + .collect(Collectors.joining(";"))); } else { CohortMetadata cohortMetadata = metadataManager.registerTemporaryCohort(study, "pendingFamilyIndexSamples", - trios.stream().map(t -> t.get(2)).collect(Collectors.toList())); + trios.stream().map(Trio::getChild).collect(Collectors.toList())); options.put(FamilyIndexDriver.TRIOS_COHORT, cohortMetadata.getName()); options.put(FamilyIndexDriver.TRIOS_COHORT_DELETE, true); @@ -137,7 +138,7 @@ private void runBatch(String study, List> trios, ObjectMap options, tableNameGenerator.getArchiveTableName(studyId), tableNameGenerator.getVariantTableName(), studyId, null, options), - "Precompute mendelian errors for " + (trios.size() == 1 ? "trio " + trios.get(0) : trios.size() + " trios")); + "Precompute mendelian errors for " + (trios.size() == 1 ? "trio " + trios.get(0).serialize() : trios.size() + " trios")); } public void postIndex(int studyId, int version) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 3731d3ff6f4..2856385c571 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -535,8 +535,12 @@ public SampleIndexQuery parse(Query query) { int extendedFilteringRegion = 0; for (String sample : sampleGenotypeQuery.keySet()) { SampleMetadata sampleMetadata = getSampleMetadata(sampleMetadatas, sample, studyId); - extendedFilteringRegion = Math.max(extendedFilteringRegion, sampleMetadata.getAttributes() - .getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + if (sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)) { + extendedFilteringRegion = extendedFilteringRegionDefault; + } else { + extendedFilteringRegion = Math.max(extendedFilteringRegion, sampleMetadata.getAttributes() + .getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + } } if (extendedFilteringRegion <= 0) { extendedFilteringRegion = extendedFilteringRegionDefault; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java index d050421a160..273ad40ae0d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java @@ -118,6 +118,12 @@ public final class SampleIndexSchema { static final String ANNOTATION_CLINICAL_PREFIX = META_PREFIX + "CL_"; static final byte[] ANNOTATION_CLINICAL_PREFIX_BYTES = Bytes.toBytes(ANNOTATION_CLINICAL_PREFIX); + /** + * The largestVariantLength might be unknown if the sample was loaded before this field was added. + * In that case, a default value for the largestVariantLength will be used. + * See HadoopVariantStorageOptions.SAMPLE_INDEX_QUERY_EXTENDED_REGION_FILTER + */ + public static final String UNKNOWN_LARGEST_VARIANT_LENGTH = "unknownLargestVariantLength"; public static final String LARGEST_VARIANT_LENGTH = "largestVariantLength"; private final int version; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java index 8285823ef2e..aea720d356a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java @@ -19,7 +19,7 @@ public class HadoopVariantStorageEngineBNDTest extends VariantStorageEngineBNDTe @Override protected void loadFiles() throws Exception { super.loadFiles(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java index f17dedc0b8f..0082e869319 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java @@ -15,6 +15,6 @@ public class HadoopVariantStorageEngineDuplicatedTest extends VariantStorageEngi public void testDuplicatedVariant() throws Exception { super.testDuplicatedVariant(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java index 5d77e7dc375..f5ebbbe8fd6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java @@ -14,6 +14,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.StoragePipelineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -31,6 +32,7 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexEntry; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; import java.net.URI; import java.nio.file.Paths; @@ -134,16 +136,33 @@ public void testMultiChromosomeSplitDataVirtualFile() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_VIRTUAL_FILE.key(), "virtual-variant-test-file.vcf"); - variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr20.variant-test-file.vcf.gz")), - outputUri, true, true, true); + int studyId = variantStorageEngine.getMetadataManager().createStudy(STUDY_NAME).getId(); + int sampleIdMock = variantStorageEngine.getMetadataManager().registerSamples(studyId, Collections.singletonList("NA19660")).get(0); + // Mark one random sample as having unknown largest variant size + // Ensure that the largest variant size is not updated + variantStorageEngine.getMetadataManager().updateSampleMetadata(studyId, sampleIdMock, sampleMetadata -> { + sampleMetadata.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, true); + }); + + StoragePipelineResult result = variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr20.variant-test-file.vcf.gz")), + outputUri, true, true, true).get(0); + + // All samples expected to be updated + assertEquals(4, result.getLoadStats().getInt("updatedSampleMetadata")); VariantStorageMetadataManager mm = variantStorageEngine.getMetadataManager(); - int studyId = mm.getStudyId(STUDY_NAME); for (String sample : SAMPLES) { SampleMetadata sampleMetadata = mm.getSampleMetadata(studyId, mm.getSampleId(studyId, sample)); assertEquals(TaskMetadata.Status.READY, sampleMetadata.getIndexStatus()); assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getAnnotationStatus()); assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getSampleIndexAnnotationStatus(1)); + if (sampleIdMock == sampleMetadata.getId()) { + assertFalse(sample, sampleMetadata.getAttributes().containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH)); + assertTrue(sample, sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)); + } else { + assertNotEquals(sample, -1, sampleMetadata.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + assertFalse(sample, sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)); + } } variantStorageEngine.annotate(outputUri, new QueryOptions()); @@ -157,8 +176,11 @@ public void testMultiChromosomeSplitDataVirtualFile() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_VIRTUAL_FILE.key(), "virtual-variant-test-file.vcf"); - variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), - outputUri, true, true, true); + result = variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), + outputUri, true, true, true).get(0); + + // No sample expected to be updated + assertEquals(0, result.getLoadStats().getInt("updatedSampleMetadata")); for (String sample : SAMPLES) { SampleMetadata sampleMetadata = mm.getSampleMetadata(studyId, mm.getSampleId(studyId, sample)); @@ -174,12 +196,29 @@ public void testMultiChromosomeSplitDataVirtualFile() throws Exception { assertEquals(TaskMetadata.Status.READY, sampleMetadata.getSampleIndexAnnotationStatus(1)); } + // Revert the unknown largest variant size + // Ensure that the largest variant size is now updated + variantStorageEngine.getMetadataManager().updateSampleMetadata(studyId, sampleIdMock, sampleMetadata -> { + sampleMetadata.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, false); + }); variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_VIRTUAL_FILE.key(), "virtual-variant-test-file.vcf"); - variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr22.variant-test-file.vcf.gz")), - outputUri); + result = variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr22.variant-test-file.vcf.gz")), + outputUri).get(0); + // One sample (sampleIdMock) expected to be updated + assertEquals(1, result.getLoadStats().getInt("updatedSampleMetadata")); + + for (String sample : SAMPLES) { + SampleMetadata sampleMetadata = mm.getSampleMetadata(studyId, mm.getSampleId(studyId, sample)); + assertEquals(TaskMetadata.Status.READY, sampleMetadata.getIndexStatus()); + assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getAnnotationStatus()); + assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getSampleIndexAnnotationStatus(1)); + + assertNotEquals(sample, -1, sampleMetadata.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + assertFalse(sample, sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)); + } for (Variant variant : variantStorageEngine.iterable(new Query(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL), null)) { String expectedFile = "virtual-variant-test-file.vcf"; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java index 673fe944ba7..e9e99cdb884 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java @@ -384,7 +384,7 @@ public void testMultipleFilesConcurrent(ObjectMap extraParams) throws Exception } studyMetadata = dbAdaptor.getMetadataManager().getStudyMetadata(studyMetadata.getId()); - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); // checkLoadedVariants(expectedVariants, dbAdaptor, PLATINUM_SKIP_VARIANTS); @@ -406,7 +406,7 @@ public void testMultipleFilesConcurrent(ObjectMap extraParams) throws Exception assertNotNull(variantFileMetadata); } - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); } @Test @@ -708,7 +708,7 @@ public StudyMetadata testPlatinumFilesOneByOne(ObjectMap otherParams, int maxFil } } - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); for (Variant variant : dbAdaptor) { System.out.println("variant = " + variant); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java index 884f45f39ff..be47dd08890 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java @@ -43,11 +43,11 @@ public void before() throws Exception { boolean wasLoaded = loaded; super.before(); if (loaded && !wasLoaded) { - VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri(getClass().getSimpleName())); + VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri()); // for (String study : variantStorageEngine.getDBAdaptor().getStudyConfigurationManager().getStudies(null).keySet()) { // variantStorageEngine.fillMissing(study, new ObjectMap(), false); // } -// VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri(getClass().getSimpleName())); +// VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java index b083abf3488..0052e805798 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java @@ -39,7 +39,7 @@ public class FisherTestDriverTest extends VariantStorageBaseTest implements Hado @Before public void setUp() throws Exception { - localOut = newOutputUri(getClass().getSimpleName()); + localOut = newOutputUri(); } @After diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java index 18692c8f77a..3d9db73719e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java @@ -159,18 +159,31 @@ public void testOverlapsWith() { @Test public void testIsRegionAfterVariantStart() { - assertTrue(FillGapsTask.isRegionAfterVariantStart(100, 100, new Variant("1:100:-:T"))); - assertFalse(FillGapsTask.isRegionAfterVariantStart(100, 100, new Variant("1:100:A:T"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(101, 101, new Variant("1:100:A:T"))); - - assertFalse(FillGapsTask.isRegionAfterVariantStart(99, 99, new Variant("1:100:AAA:GGG"))); - assertFalse(FillGapsTask.isRegionAfterVariantStart(100, 100, new Variant("1:100:AAA:GGG"))); - assertFalse(FillGapsTask.isRegionAfterVariantStart(101, 100, new Variant("1:100:AAA:GGG"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(101, 101, new Variant("1:100:AAA:GGG"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(102, 102, new Variant("1:100:AAA:GGG"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(103, 103, new Variant("1:100:AAA:GGG"))); + isRegionAfterVariantStart(100, 100, "1:100:-:T", true, false); + isRegionAfterVariantStart(101, 101, "1:100:A:T", true, false); + + isRegionAfterVariantStart(99, 99, "1:100:AAA:GGG", false, false); + isRegionAfterVariantStart(100, 100, "1:100:AAA:GGG", false, true); + isRegionAfterVariantStart(101, 100, "1:100:AAA:GGG", true, true); + isRegionAfterVariantStart(101, 101, "1:100:AAA:GGG", true, true); + isRegionAfterVariantStart(102, 102, "1:100:AAA:GGG", true, true); + isRegionAfterVariantStart(103, 103, "1:100:AAA:GGG", true, false); + + isRegionAfterVariantStart(100, 100, "1:101:-:TTT", false, false); + isRegionAfterVariantStart(101, 100, "1:100:A:T", true, false); + + // Overlapping positions + isRegionAfterVariantStart(100, 110, "1:105:A:T", true, true); + isRegionAfterVariantStart(100, 100, "1:100:A:T", false, true); } + private void isRegionAfterVariantStart(int start, int end, String variantString, boolean afterVariant, boolean overlaps) { + Variant variant = new Variant(variantString); + assertEquals("isRegionAfterVariantStart", afterVariant, FillGapsTask.isRegionAfterVariantStart(start, end, variant)); + assertEquals("overlapsWith", overlaps, FillGapsTask.overlapsWith(variant, variant.getChromosome(), start, end)); + } + + @Test public void fillGapsAlreadyPresent() { FillGapsTask task = new FillGapsTask(metadataManager, this.studyMetadata, true, false, GenotypeClass.UNKNOWN_GENOTYPE); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java index b23ca96b304..3ae2a36bfbc 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java @@ -1,6 +1,8 @@ package org.opencb.opencga.storage.hadoop.variant.gaps; import com.google.common.collect.BiMap; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -26,10 +28,13 @@ import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; +import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; @@ -79,7 +84,7 @@ public void testFillGapsVcfFiles() throws Exception { inputFiles.add(getResourceUri(fileName)); } - StudyMetadata studyMetadata = load(new ObjectMap(VariantStorageOptions.GVCF.key(), false), inputFiles, newOutputUri(1)); + StudyMetadata studyMetadata = load(new ObjectMap(VariantStorageOptions.GVCF.key(), false), inputFiles, newOutputUri()); // Do not check new missing genotypes, as the input files does not have missing genotype regions testFillGapsPlatinumFiles(studyMetadata, false); @@ -217,8 +222,10 @@ public void checkConflictingFiles(StudyMetadata studyMetadata) throws Exception sampleIds.sort(Integer::compareTo); List samples = sampleIds.stream().map(samplesMap::get).collect(Collectors.toList()); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); + fillGaps(variantStorageEngine, studyMetadata, samples); - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); checkFillGaps(studyMetadata, dbAdaptor, samples, Collections.singleton("1:10020:A:T")); checkSampleIndexTable(dbAdaptor); @@ -317,6 +324,62 @@ public void testFillGapsCorpasome() throws Exception { .setSamples(Arrays.asList("ISDBM322015", "ISDBM322016", "ISDBM322017", "ISDBM322018")), new ObjectMap()); printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); + load(new ObjectMap(VariantStorageOptions.STUDY.key(), STUDY_NAME_2), + Collections.singletonList(getResourceUri("quartet.variants.annotated.partial.vcf.gz"))); + + checkInputValuesAreUnmodified(STUDY_NAME, STUDY_NAME_2); + } + + @Test + public void testFillGapsGiabChinesse() throws Exception { + ObjectMap options = new ObjectMap(VariantStorageOptions.GVCF.key(), true) + .append(HadoopVariantStorageOptions.ARCHIVE_CHUNK_SIZE.key(), 1000000) + .append(VariantStorageOptions.STUDY.key(), STUDY_NAME); + loadGIAB(options); + printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); + + variantStorageEngine.aggregateFamily(STUDY_NAME, new VariantAggregateFamilyParams() + .setSamples(Arrays.asList("HG005", "HG006", "HG007")), new ObjectMap()); + printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); + + options.put(VariantStorageOptions.STUDY.key(), STUDY_NAME_2); + loadGIAB(options); + + checkInputValuesAreUnmodified(STUDY_NAME, STUDY_NAME_2); + } + + private void checkInputValuesAreUnmodified(String aggregatedStudy, String referenceStudy) throws Exception { + VariantHadoopDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); + + int studyId1 = dbAdaptor.getMetadataManager().getStudyId(aggregatedStudy); + int studyId2 = dbAdaptor.getMetadataManager().getStudyId(referenceStudy); + + dbAdaptor.getHBaseManager().act(dbAdaptor.getVariantTable(), table -> { + table.getScanner(new Scan()).iterator().forEachRemaining(r -> { + byte[] row = r.getRow(); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(row); + + NavigableMap cells = r.getFamilyMap(GenomeHelper.COLUMN_FAMILY_BYTES); + for (Map.Entry entry : cells.entrySet()) { + String columnKey = Bytes.toString(entry.getKey()); + Integer studyId = VariantPhoenixSchema.extractStudyId(columnKey, false); + if (studyId != null && studyId == studyId2) { + String otherColumnKey = columnKey.replaceFirst(VariantPhoenixSchema.buildStudyColumnsPrefix(studyId2), + VariantPhoenixSchema.buildStudyColumnsPrefix(studyId1)); + byte[] thisCell = entry.getValue(); + byte[] otherCell = cells.get(Bytes.toBytes(otherColumnKey)); + assertArrayEquals(variant.toString() + " study1ColumnKey " + otherColumnKey + ", study2ColumnKey " + columnKey, + thisCell, otherCell); + } + } + }); + }); + } + + private void loadGIAB(ObjectMap options) throws Exception { + load(options, Collections.singletonList(getResourceUri("giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz")), newOutputUri(), false); + load(options, Collections.singletonList(getResourceUri("giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz")), newOutputUri(), false); + load(options, Collections.singletonList(getResourceUri("giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz")), newOutputUri(), false); } @Test @@ -325,12 +388,12 @@ public void testFillGapsImpact() throws Exception { ObjectMap extraParams = new ObjectMap(VariantStorageOptions.LOAD_HOM_REF.key(), true); // extraParams.append(VariantStorageOptions.TRANSFORM_FORMAT.key(), "proto"); // extraParams.append(VariantStorageOptions.GVCF.key(), true); - StudyMetadata study = load(extraParams, Collections.singletonList(getResourceUri("impact/HG005_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); + load(extraParams, Collections.singletonList(getResourceUri("impact/HG005_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); load(extraParams, Collections.singletonList(getResourceUri("impact/HG006_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); load(extraParams, Collections.singletonList(getResourceUri("impact/HG007_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); - variantStorageEngine.aggregateFamily(study.getName(), new VariantAggregateFamilyParams() + variantStorageEngine.aggregateFamily(STUDY_NAME, new VariantAggregateFamilyParams() .setSamples(Arrays.asList("HG005", "HG006", "HG007")), new ObjectMap()); printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); @@ -373,11 +436,11 @@ private StudyMetadata loadPlatinum(ObjectMap extraParams, int from, int to) thro inputFiles.add(getResourceUri(fileName)); } - return load(extraParams, inputFiles, newOutputUri(1)); + return load(extraParams, inputFiles, newOutputUri()); } private StudyMetadata load(ObjectMap extraParams, List inputFiles) throws Exception { - return load(extraParams, inputFiles, newOutputUri(1)); + return load(extraParams, inputFiles, newOutputUri()); } private StudyMetadata load(ObjectMap extraParams, List inputFiles, URI outputUri) throws Exception { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index 630ddb4960d..034e676c1b4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -18,6 +18,7 @@ import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -80,12 +81,13 @@ public void before() throws Exception { runETL(variantStorageEngine, getResourceUri("variant-test-me.vcf"), outputUri, params, true, true, true); - List family = Arrays.asList(father, mother, child); + Trio family = new Trio(father, mother, child); + Trio family2 = new Trio("FATHER", "MOTHER", "PROBAND"); - variantStorageEngine.aggregateFamily(study, new VariantAggregateFamilyParams(family, false), new ObjectMap()); + variantStorageEngine.aggregateFamily(study, new VariantAggregateFamilyParams(family.toList(), false), new ObjectMap()); variantStorageEngine.familyIndex(study, Collections.singletonList(family), new ObjectMap()); - variantStorageEngine.familyIndex(study, Collections.singletonList(Arrays.asList("FATHER", "MOTHER", "PROBAND")), new ObjectMap()); + variantStorageEngine.familyIndex(study, Collections.singletonList(family2), new ObjectMap()); variantStorageEngine.annotate(outputUri, new ObjectMap()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 0319c2bc43b..3e6ab96edcd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -31,6 +31,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -111,12 +112,12 @@ public class SampleIndexTest extends VariantStorageBaseTest implements HadoopVar // Arrays.asList("NA19661", "NA19685", "NA19600"), // Arrays.asList("NA19685", "NA19600", "NA19660") // ); - private static List> trios = Arrays.asList( - Arrays.asList("NA19660", "NA19661", "NA19685"), - Arrays.asList("NA19660", "NA19661", "NA19600") + private static List trios = Arrays.asList( + new Trio("NA19660", "NA19661", "NA19685"), + new Trio("NA19660", "NA19661", "NA19600") ); - private static List> triosPlatinum = Arrays.asList( - Arrays.asList("NA12877", "-", "NA12878") + private static List triosPlatinum = Arrays.asList( + new Trio("NA12877", null, "NA12878") ); @Before @@ -341,8 +342,8 @@ public void regenerateSampleIndex() throws Exception { studyId, Collections.emptySet(), options), ""); - if (sampleNames.get(study).containsAll(trios.get(0))) { - options.put(FamilyIndexDriver.TRIOS, trios.stream().map(trio -> String.join(",", trio)).collect(Collectors.joining(";"))); + if (sampleNames.get(study).containsAll(trios.get(0).toList())) { + options.put(FamilyIndexDriver.TRIOS, trios.stream().map(Trio::serialize).collect(Collectors.joining(";"))); options.put(FamilyIndexDriver.OVERWRITE, true); new TestMRExecutor().run(FamilyIndexDriver.class, FamilyIndexDriver.buildArgs( dbAdaptor.getArchiveTableName(studyId), @@ -350,7 +351,7 @@ public void regenerateSampleIndex() throws Exception { studyId, Collections.emptySet(), options), ""); } else if (study.equals(STUDY_NAME_3)) { - options.put(FamilyIndexDriver.TRIOS, triosPlatinum.stream().map(trio -> String.join(",", trio)).collect(Collectors.joining(";"))); + options.put(FamilyIndexDriver.TRIOS, triosPlatinum.stream().map(Trio::serialize).collect(Collectors.joining(";"))); options.put(FamilyIndexDriver.OVERWRITE, true); new TestMRExecutor().run(FamilyIndexDriver.class, FamilyIndexDriver.buildArgs( dbAdaptor.getArchiveTableName(studyId), @@ -1036,8 +1037,8 @@ public void testApproximateCount() { @Test public void testFamilyIndexQueryCount() { - List trio = trios.get(0); - String proband = trio.get(2); + Trio trio = trios.get(0); + String proband = trio.getChild(); VariantQueryResult result = variantStorageEngine.get( new Query() .append(STUDY.key(), STUDY_NAME) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java index d0c7fcfc95c..ddb7e993549 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java @@ -235,7 +235,7 @@ public void exportFromSearchIndex() throws Exception { public URI getOutputUri(String fileName) throws IOException { if (exportToLocal) { - return newOutputUri(1).resolve(fileName); + return newOutputUri().resolve(fileName); } else { return URI.create("hdfs:///" + fileName); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java index d63e4c9b6cc..3cce6445d52 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java @@ -51,7 +51,7 @@ public void tearDown() throws Exception { public void calculateStatsMultiCohortsAfterFillMissingTest() throws Exception { VariantStorageEngine storageEngine = getVariantStorageEngine(); storageEngine.aggregate(studyMetadata.getName(), new VariantAggregateParams(false, false), new ObjectMap()); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); calculateStatsMultiCohortsTest(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java index abebe1e78ab..c98505eeafa 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java @@ -82,14 +82,14 @@ public void tearDown() throws Exception { public void calculateStatsMultiCohortsTest() throws Exception { super.calculateStatsMultiCohortsTest(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); //delete all stats List cohorts = new ArrayList<>(); metadataManager.getCalculatedCohorts(studyMetadata.getId()).forEach(c -> cohorts.add(c.getName())); variantStorageEngine.deleteStats(studyMetadata.getName(), cohorts, new ObjectMap()); checkCohorts(dbAdaptor, studyMetadata); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); super.calculateStatsMultiCohortsTest(); } @@ -98,14 +98,14 @@ public void calculateStatsMultiCohortsTest() throws Exception { public void calculateStatsSeparatedCohortsTest() throws Exception { super.calculateStatsSeparatedCohortsTest(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); //delete all stats List cohorts = new ArrayList<>(); metadataManager.getCalculatedCohorts(studyMetadata.getId()).forEach(c -> cohorts.add(c.getName())); variantStorageEngine.deleteStats(studyMetadata.getName(), cohorts, new ObjectMap()); checkCohorts(dbAdaptor, studyMetadata); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); super.calculateStatsSeparatedCohortsTest(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java index 1fa042eed74..5a3a0a1ea54 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java @@ -69,7 +69,7 @@ public void before() throws Exception { engine.annotate(outputUri, new ObjectMap()); engine.aggregateFamily(study, new VariantAggregateFamilyParams(family, false), new ObjectMap()); - VariantHbaseTestUtils.printVariants(engine.getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(engine.getDBAdaptor(), newOutputUri()); stats = computeSampleVariantStatsDirectly(); }