diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index e79ba77ade7..00f6aa9fa53 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -13,23 +13,24 @@ jobs: with: maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip - test: - uses: ./.github/workflows/test-analysis.yml - needs: build - with: - test_profile: runShortTests,runMediumTests - secrets: inherit +## TASK-4970: remove the test job from the develop.yml workflow because it is not needed anymore. +# test: +# uses: ./.github/workflows/test-analysis.yml +# needs: build +# with: +# test_profile: runShortTests,runMediumTests +# secrets: inherit deploy-maven: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop - needs: test + needs: build with: maven_opts: -P hdp3.1 -Dopencga.war.name=opencga secrets: inherit deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop - needs: test + needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init secrets: inherit diff --git a/.github/workflows/docker-aws-emr.yml b/.github/workflows/docker-aws-emr.yml deleted file mode 100644 index 392514a9a9b..00000000000 --- a/.github/workflows/docker-aws-emr.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Release new AWS EMR OpenCGA version - -on: - push: - tags: - - '*' - workflow_dispatch: - -jobs: - build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop - with: - maven_opts: -Pemr6.1 -Dopencga.war.name=opencga - - deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop - needs: build - with: - cli: python3 ./build/cloud/docker/docker-build.py push --images base,init - secrets: inherit diff --git a/.github/workflows/long-test-analysis.yml b/.github/workflows/long-test-analysis.yml index e0c14b79175..10fdee5a0c3 100644 --- a/.github/workflows/long-test-analysis.yml +++ b/.github/workflows/long-test-analysis.yml @@ -14,8 +14,8 @@ jobs: matrix: hadoop: [ "hdp3.1", "hdi5.1", "emr6.1", "emr6.13" ] uses: ./.github/workflows/test-analysis.yml - secrets: inherit with: test_profile: runShortTests,runMediumTests,runLongTests hadoop: ${{ matrix.hadoop }} + secrets: inherit diff --git a/.github/workflows/manual-deploy-docker.yml b/.github/workflows/manual-deploy-docker.yml index 69472345e83..3ee65923466 100644 --- a/.github/workflows/manual-deploy-docker.yml +++ b/.github/workflows/manual-deploy-docker.yml @@ -51,10 +51,9 @@ jobs: name: build-folder path: build - deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ inputs.tag }} - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml index 63e61914ea0..b614a520caf 100644 --- a/.github/workflows/manual-deploy-ext-tools.yml +++ b/.github/workflows/manual-deploy-ext-tools.yml @@ -46,11 +46,9 @@ jobs: name: build-folder path: build - deploy-docker-ext-tools: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }} secrets: inherit - diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml new file mode 100644 index 00000000000..753466dd67a --- /dev/null +++ b/.github/workflows/pull-request-approved.yml @@ -0,0 +1,19 @@ +name: Pull request approve workflow + +on: + pull_request_review: + types: [submitted] + +jobs: + build: + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + with: + maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip + + test: + name: "Run all tests before merging, ie. short, medium and long tests." + uses: ./.github/workflows/test-analysis.yml + needs: build + with: + test_profile: runShortTests,runMediumTests,runLongTests + secrets: inherit diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merged.yml similarity index 100% rename from .github/workflows/pull-request-merge.yml rename to .github/workflows/pull-request-merged.yml diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index f27e7ce3b7f..cc8470b747a 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -17,9 +17,9 @@ jobs: test: uses: ./.github/workflows/test-analysis.yml needs: build - secrets: inherit with: test_profile: runShortTests + secrets: inherit deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index bb48ccd4b40..554e1f72520 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: '10' - name: Set up JDK 11 uses: actions/setup-java@v4 with: @@ -65,7 +65,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: '10' - name: Set up JDK 8 uses: actions/setup-java@v4 with: @@ -90,8 +90,8 @@ jobs: - name: Maven build run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }} - name: Run Junit tests - run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }} - - name: Publish Test Report + run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }} + - name: Publish Test Report on GitHub uses: scacap/action-surefire-report@v1 env: NODE_OPTIONS: '--max_old_space_size=4096' diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 3b96f1b246e..1d127746157 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml @@ -357,6 +357,17 @@ + + + ../opencga-core/src/test/resources + + log4j2-test.xml + + + + src/test/resources + + org.apache.maven.plugins diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java index 2712d22b8f3..ce7b82a7dba 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java @@ -54,6 +54,10 @@ protected void check() throws Exception { super.check(); // Sanity check + if (StringUtils.isEmpty(getJobId())) { + throw new ToolException("Missing job ID"); + } + if (StringUtils.isEmpty(getStudy())) { throw new ToolException("Missing study when computing alignment coverage"); } @@ -168,20 +172,22 @@ protected void run() throws Exception { + ") was not create, please, check log files."); } - // Try to copy the BW file into the BAM file directory + // Try to move the BW file into the BAM file directory + boolean moveSuccessful = false; Path targetPath = Paths.get(bamCatalogFile.getUri()).getParent().resolve(bwPath.getFileName()); try { - Files.move(bwPath, targetPath); + Path movedPath = Files.move(bwPath, targetPath); + moveSuccessful = targetPath.equals(movedPath); } catch (Exception e) { - // Do nothing - logger.info("Moving from {} to {}: {}", bwPath, targetPath, e.getMessage()); + // Log message + logger.info("Error moving the coverage file into the BAM folder {} to {}", bwPath, targetPath, e); } - if (targetPath.toFile().exists()) { + if (moveSuccessful) { bwPath = targetPath; - logger.info("Coverage file was copied into the BAM folder: {}", bwPath); + logger.info("Coverage file was moved into the BAM folder: {}", bwPath); } else { - logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}", + logger.info("Couldn't move the coverage file into the BAM folder. The coverage file is in the job folder instead: {}", bwPath); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java index f21986c2174..fa537f5a973 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java @@ -16,29 +16,34 @@ package org.opencb.opencga.analysis.alignment; +import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.tools.alignment.BamManager; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaTool; +import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.alignment.AlignmentIndexParams; +import org.opencb.opencga.core.models.alignment.CoverageIndexParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; +import org.opencb.opencga.core.tools.annotations.ToolParams; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @Tool(id = AlignmentIndexOperation.ID, resource = Enums.Resource.ALIGNMENT, description = "Index alignment.") -public class AlignmentIndexOperation extends OpenCgaTool { +public class AlignmentIndexOperation extends OpenCgaToolScopeStudy { public static final String ID = "alignment-index-run"; public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file"; - private String study; - private String inputFile; + @ToolParams + protected final AlignmentIndexParams indexParams = new AlignmentIndexParams(); private File inputCatalogFile; private Path inputPath; @@ -48,14 +53,23 @@ public class AlignmentIndexOperation extends OpenCgaTool { protected void check() throws Exception { super.check(); + // Sanity check + if (StringUtils.isEmpty(getJobId())) { + throw new ToolException("Missing job ID"); + } + + if (StringUtils.isEmpty(getStudy())) { + throw new ToolException("Missing study when computing alignment index"); + } + OpenCGAResult fileResult; try { - fileResult = catalogManager.getFileManager().get(getStudy(), inputFile, QueryOptions.empty(), token); + fileResult = catalogManager.getFileManager().get(getStudy(), indexParams.getFileId(), QueryOptions.empty(), token); } catch (CatalogException e) { - throw new ToolException("Error accessing file '" + inputFile + "' of the study " + study + "'", e); + throw new ToolException("Error accessing file '" + indexParams.getFileId() + "' of the study " + study + "'", e); } if (fileResult.getNumResults() <= 0) { - throw new ToolException("File '" + inputFile + "' not found in study '" + study + "'"); + throw new ToolException("File '" + indexParams.getFileId() + "' not found in study '" + study + "'"); } inputCatalogFile = fileResult.getResults().get(0); @@ -64,7 +78,7 @@ protected void check() throws Exception { // Check if the input file is .bam or .cram if (!filename.endsWith(AlignmentConstants.BAM_EXTENSION) && !filename.endsWith(AlignmentConstants.CRAM_EXTENSION)) { - throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format"); + throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format"); } outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION) @@ -73,6 +87,9 @@ protected void check() throws Exception { @Override protected void run() throws Exception { + setUpStorageEngineExecutor(study); + + logger.info("Running with parameters {}", indexParams); step(ID, () -> { // Compute index if necessary @@ -82,28 +99,31 @@ protected void run() throws Exception { bamManager.close(); if (!outputPath.toFile().exists()) { - throw new ToolException("Something wrong happened when computing index file for '" + inputFile + "'"); + throw new ToolException("Something wrong happened when computing index file for '" + indexParams.getFileId() + "'"); } - // Try to copy the BAI file into the BAM file directory + // Try to move the BAI file into the BAM file directory + boolean moveSuccessful = false; Path targetPath = inputPath.getParent().resolve(outputPath.getFileName()); try { - Files.move(outputPath, targetPath); + Path movedPath = Files.move(outputPath, targetPath); + moveSuccessful = targetPath.equals(movedPath); } catch (Exception e) { - // Do nothing - logger.info("Moving from {} to {}: {}", outputPath, targetPath, e.getMessage()); + // Log message + logger.info("Error moving from {} to {}", outputPath, targetPath, e); } - if (targetPath.toFile().exists()) { + if (moveSuccessful) { outputPath = targetPath; - logger.info("Alignment index file was copied into the BAM folder: {}", outputPath); + logger.info("Alignment index file was moved into the BAM folder: {}", outputPath); } else { - logger.info("Couldn't copy the alignment index file into the BAM folder. The index file is in the job folder instead: {}", + logger.info("Couldn't move the alignment index file into the BAM folder. The index file is in the job folder instead: {}", outputPath); } // Link generated BAI file and update samples info, related file - File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token); + File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, + token); // Update BAM file internal in order to set the alignment index (BAI) FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY), @@ -111,22 +131,4 @@ protected void run() throws Exception { catalogManager.getFileManager().updateFileInternalAlignmentIndex(study, inputCatalogFile, fileAlignmentIndex, token); }); } - - public String getStudy() { - return study; - } - - public AlignmentIndexOperation setStudy(String study) { - this.study = study; - return this; - } - - public String getInputFile() { - return inputFile; - } - - public AlignmentIndexOperation setInputFile(String inputFile) { - this.inputFile = inputFile; - return this; - } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java index 7c3396cb48e..9dbf2d9223a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java @@ -27,7 +27,6 @@ import org.opencb.biodata.tools.alignment.BamUtils; import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.cellbase.client.rest.GeneClient; -import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.StorageManager; @@ -39,7 +38,6 @@ import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.project.Project; @@ -66,6 +64,7 @@ public class AlignmentStorageManager extends StorageManager { private AlignmentStorageEngine alignmentStorageEngine; private String jobId; + private boolean dryRun; private static final Map statsMap = new HashMap<>(); @@ -78,31 +77,29 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto initStatsMap(); } - public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFactory storageEngineFactory, String jobId) { + public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFactory storageEngineFactory, String jobId, boolean dryRun) { super(catalogManager, storageEngineFactory); // TODO: Create this alignmentStorageEngine by reflection this.alignmentStorageEngine = new LocalAlignmentStorageEngine(); this.jobId = jobId; + this.dryRun = dryRun; initStatsMap(); } - //------------------------------------------------------------------------- - // INDEX - //------------------------------------------------------------------------- - - public void index(String study, String inputFile, String outdir, String token) throws ToolException { - ObjectMap params = new ObjectMap(); - - AlignmentIndexOperation indexOperation = new AlignmentIndexOperation(); - indexOperation.setUp(null, catalogManager, storageEngineFactory, params, Paths.get(outdir), jobId, token); - - indexOperation.setStudy(study); - indexOperation.setInputFile(inputFile); - - indexOperation.start(); - } +// //------------------------------------------------------------------------- +// // INDEX +// //------------------------------------------------------------------------- +// +// public void index(String study, String inputFile, String outdir, String token) throws ToolException { +// ToolRunner toolRunner = new ToolRunner("", catalogManager, storageEngineFactory); +// +// AlignmentIndexParams params = new AlignmentIndexParams(); +// params.setFileId(inputFile); +// toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, study), Paths.get(outdir), +// jobId, token); +// } //------------------------------------------------------------------------- // QUERY diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java index 6b541da0037..67ac836c36a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java @@ -120,7 +120,8 @@ protected void run() throws ToolException { OpenCGAResult flagStatsJobResult = catalogManager.getJobManager() .submit(study, AlignmentFlagStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); flagStatsJobId = flagStatsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + flagStatsJobId + " to compute stats (" + AlignmentFlagStatsAnalysis.ID + ")"); @@ -137,7 +138,8 @@ protected void run() throws ToolException { OpenCGAResult statsJobResult = catalogManager.getJobManager() .submit(study, AlignmentStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); statsJobId = statsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + statsJobId + " to compute stats (" + AlignmentStatsAnalysis.ID + ")"); } @@ -154,7 +156,7 @@ protected void run() throws ToolException { OpenCGAResult fastQcMetricsJobResult = catalogManager.getJobManager() .submit(study, AlignmentFastQcMetricsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - token); + getJobId(), null, false, token); fastQcMetricsJobId = fastQcMetricsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + fastQcMetricsJobId + " to compute FastQC metrics (" + AlignmentFastQcMetricsAnalysis.ID + ")"); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java index d7756ea82a5..d9c2e7c5734 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java @@ -64,7 +64,7 @@ import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java index 1d2be95e82e..abdbca2e7b5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java @@ -32,7 +32,7 @@ import java.util.Collections; @Tool(id = ClinicalTsvAnnotationLoader.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class ClinicalTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "clinical-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 39de4806eb0..06e1c7f4838 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -44,7 +44,7 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; @@ -68,6 +68,7 @@ public class ExomiserInterpretationAnalysis extends InterpretationAnalysis { private String studyId; private String clinicalAnalysisId; private String sampleId; + private ClinicalAnalysis.Type clinicalAnalysisType; private ClinicalAnalysis clinicalAnalysis; @@ -116,6 +117,14 @@ protected void check() throws Exception { } sampleId = clinicalAnalysis.getProband().getSamples().get(0).getId(); + if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY) { + clinicalAnalysisType = ClinicalAnalysis.Type.FAMILY; + } else { + clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE; + } + logger.info("The clinical analysis type is {}, so the Exomiser will be run in mode {}", clinicalAnalysis.getType(), + clinicalAnalysisType); + // Update executor params with OpenCGA home and session ID setUpStorageEngineExecutor(studyId); } @@ -128,6 +137,7 @@ protected void run() throws ToolException { getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(studyId) .setSampleId(sampleId) + .setClinicalAnalysisType(clinicalAnalysisType) .execute(); saveInterpretation(studyId, clinicalAnalysis); @@ -181,7 +191,8 @@ private List getPrimaryFindings() throws IOException, StorageEn // Prepare variant query List sampleIds = new ArrayList<>(); - if (clinicalAnalysis.getFamily() != null && CollectionUtils.isNotEmpty(clinicalAnalysis.getFamily().getMembers())) { + if (clinicalAnalysis.getType() == ClinicalAnalysis.Type.FAMILY && clinicalAnalysis.getFamily() != null + && CollectionUtils.isNotEmpty(clinicalAnalysis.getFamily().getMembers())) { for (Individual member : clinicalAnalysis.getFamily().getMembers()) { Individual individual = IndividualQcUtils.getIndividualById(studyId, member.getId(), getCatalogManager(), getToken()); if (CollectionUtils.isNotEmpty(individual.getSamples())) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java index 37823aac657..d70f3f3d244 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/AuxiliarRgaAnalysis.java @@ -7,7 +7,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = AuxiliarRgaAnalysis.ID, resource = Enums.Resource.RGA, type = Tool.Type.OPERATION, - description = AuxiliarRgaAnalysis.DESCRIPTION) + description = AuxiliarRgaAnalysis.DESCRIPTION, priority = Enums.Priority.HIGH) public class AuxiliarRgaAnalysis extends OperationTool { public final static String ID = "rga-aux-index"; public final static String DESCRIPTION = ParamConstants.INDEX_AUXILIAR_COLLECTION_DESCRIPTION; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java index 0e2a56e644d..132b3d72464 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/rga/RgaAnalysis.java @@ -6,7 +6,8 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.tools.annotations.Tool; -@Tool(id = RgaAnalysis.ID, resource = Enums.Resource.RGA, type = Tool.Type.OPERATION, description = "Index RGA study.") +@Tool(id = RgaAnalysis.ID, resource = Enums.Resource.RGA, type = Tool.Type.OPERATION, description = "Index RGA study.", + priority = Enums.Priority.HIGH) public class RgaAnalysis extends OperationTool { public final static String ID = "rga-index"; public final static String DESCRIPTION = "Generate Recessive Gene Analysis secondary index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/tiering/CancerTieringInterpretationAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/tiering/CancerTieringInterpretationAnalysisExecutor.java index ce89757143a..ea8dc7c3c89 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/tiering/CancerTieringInterpretationAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/tiering/CancerTieringInterpretationAnalysisExecutor.java @@ -41,7 +41,7 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/zetta/ZettaInterpretationAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/zetta/ZettaInterpretationAnalysisExecutor.java index c45f0b484a9..cbf7e8211d2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/zetta/ZettaInterpretationAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/zetta/ZettaInterpretationAnalysisExecutor.java @@ -28,7 +28,7 @@ import org.opencb.opencga.analysis.clinical.ClinicalUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.OpenCgaToolExecutor; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -40,7 +40,6 @@ import java.util.Map; import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.PRIMARY_FINDINGS_FILENAME; -import static org.opencb.opencga.analysis.clinical.InterpretationAnalysis.SECONDARY_FINDINGS_FILENAME; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.FAMILY_SEGREGATION; @ToolExecutor(id = "opencga-local", diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java index 82b8c9e1bb4..7dcdc62c8fc 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/cohort/CohortTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = CohortTsvAnnotationLoader.ID, resource = Enums.Resource.COHORT, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class CohortTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "cohort-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java index 6d4091fdce3..68efcc010b5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/FamilyTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = FamilyTsvAnnotationLoader.ID, resource = Enums.Resource.FAMILY, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class FamilyTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "family-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java index c242582b528..3dcd8079060 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FetchAndRegisterTask.java @@ -46,7 +46,7 @@ import java.util.List; @Tool(id = FetchAndRegisterTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = "Download an external file and register it in OpenCGA.") + description = "Download an external file and register it in OpenCGA.", priority = Enums.Priority.HIGH) public class FetchAndRegisterTask extends OpenCgaToolScopeStudy { public final static String ID = "files-fetch"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java index 89edaa6e683..2750435e049 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileDeleteTask.java @@ -42,7 +42,8 @@ import java.io.IOException; import java.util.*; -@Tool(id = FileDeleteTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Delete files.") +@Tool(id = FileDeleteTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Delete files.", + priority = Enums.Priority.HIGH) public class FileDeleteTask extends OpenCgaTool { public final static String ID = "files-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java index d6506bb6d81..88cb038215a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java @@ -18,7 +18,8 @@ import java.util.List; import java.util.Map; -@Tool(id = FileLinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = FileLinkTask.DESCRIPTION) +@Tool(id = FileLinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = FileLinkTask.DESCRIPTION, + priority = Enums.Priority.HIGH) public class FileLinkTask extends OpenCgaToolScopeStudy { public static final String ID = "file-link"; @@ -69,7 +70,7 @@ protected void run() throws Exception { Job postLinkJob = catalogManager.getJobManager() .submit(getStudy(), PostLinkSampleAssociation.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - getToken()).first(); + getJobId(), null, false, getToken()).first(); logger.info("Submit post-link job : " + postLinkJob.getId()); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java index ddd77ca22af..f52871a7b6e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileTsvAnnotationLoader.java @@ -32,7 +32,7 @@ import java.util.Collections; @Tool(id = FileTsvAnnotationLoader.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class FileTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "file-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java index e333d7b3813..24a45fe40b9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileUnlinkTask.java @@ -38,7 +38,8 @@ import java.util.*; -@Tool(id = FileUnlinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Unlink files.") +@Tool(id = FileUnlinkTask.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, description = "Unlink files.", + priority = Enums.Priority.HIGH) public class FileUnlinkTask extends OpenCgaTool { public final static String ID = "files-unlink"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java index 60d67f05996..d624d2e4828 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/PostLinkSampleAssociation.java @@ -24,7 +24,7 @@ import java.util.*; @Tool(id = PostLinkSampleAssociation.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION, - description = PostLinkSampleAssociation.DESCRIPTION) + description = PostLinkSampleAssociation.DESCRIPTION, priority = Enums.Priority.HIGH) public class PostLinkSampleAssociation extends OpenCgaToolScopeStudy { public static final String ID = "postlink"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java index 3c53ac46524..b6776329c50 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/individual/IndividualTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = IndividualTsvAnnotationLoader.ID, resource = Enums.Resource.INDIVIDUAL, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class IndividualTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "individual-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java index adeb819720b..c3fa3ad84b1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/SampleTsvAnnotationLoader.java @@ -25,7 +25,7 @@ import org.opencb.opencga.core.tools.annotations.Tool; @Tool(id = SampleTsvAnnotationLoader.ID, resource = Enums.Resource.SAMPLE, type = Tool.Type.OPERATION, - description = "Load annotations from TSV file.") + description = "Load annotations from TSV file.", priority = Enums.Priority.HIGH) public class SampleTsvAnnotationLoader extends TsvAnnotationLoader { public final static String ID = "sample-tsv-load"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java index 54f22745057..b8d460824c1 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java @@ -225,7 +225,8 @@ protected void run() throws ToolException { OpenCGAResult variantStatsJobResult = catalogManager.getJobManager() .submit(study, SampleVariantStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); variantStatsJobId = variantStatsJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + variantStatsJobId + " to compute stats (" + SampleVariantStatsAnalysis.ID + ")"); @@ -267,7 +268,8 @@ protected void run() throws ToolException { OpenCGAResult signatureJobResult = catalogManager.getJobManager() .submit(getStudy(), MutationalSignatureAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " - + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); + + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null, + false, token); signatureJobId = signatureJobResult.first().getId(); logger.info("Submitted job {} to compute the mutational signature analysis {}", signatureJobId, MutationalSignatureAnalysis.ID); @@ -289,7 +291,7 @@ protected void run() throws ToolException { OpenCGAResult genomePlotJobResult = catalogManager.getJobManager() .submit(getStudy(), GenomePlotAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - token); + getJobId(), null, false, token); genomePlotJobId = genomePlotJobResult.first().getId(); addEvent(Event.Type.INFO, "Submit job " + genomePlotJobId + " to compute genome plot (" + GenomePlotAnalysis.ID + ")"); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java index 08797e1185e..cf8390d1a61 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/templates/TemplateRunner.java @@ -15,7 +15,7 @@ import java.nio.file.Paths; @Tool(id = TemplateRunner.ID, description = TemplateRunner.DESCRIPTION, type = Tool.Type.OPERATION, resource = Enums.Resource.STUDY, - scope = Tool.Scope.STUDY) + scope = Tool.Scope.STUDY, priority = Enums.Priority.HIGH) public class TemplateRunner extends OperationTool { public static final String ID = "templates"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java index 7769315137b..5a88635e45c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java @@ -67,6 +67,7 @@ public abstract class OpenCgaTool { private String jobId; private String opencgaHome; + private boolean dryRun; protected String token; protected final ObjectMap params; @@ -92,19 +93,20 @@ public OpenCgaTool() { } public final OpenCgaTool setUp(String opencgaHome, CatalogManager catalogManager, StorageEngineFactory engineFactory, - ObjectMap params, Path outDir, String jobId, String token) { + ObjectMap params, Path outDir, String jobId, boolean dryRun, String token) { VariantStorageManager manager = new VariantStorageManager(catalogManager, engineFactory); - return setUp(opencgaHome, catalogManager, manager, params, outDir, jobId, token); + return setUp(opencgaHome, catalogManager, manager, params, outDir, jobId, dryRun, token); } public final OpenCgaTool setUp(String opencgaHome, CatalogManager catalogManager, VariantStorageManager variantStorageManager, - ObjectMap params, Path outDir, String jobId, String token) { + ObjectMap params, Path outDir, String jobId, boolean dryRun, String token) { this.opencgaHome = opencgaHome; this.catalogManager = catalogManager; this.configuration = catalogManager.getConfiguration(); this.variantStorageManager = variantStorageManager; this.storageConfiguration = variantStorageManager.getStorageConfiguration(); this.jobId = jobId; + this.dryRun = dryRun; this.token = token; if (params != null) { this.params.putAll(params); @@ -188,13 +190,14 @@ public final ExecutionResult start() throws ToolException { exception = e; } if (!erm.isClosed()) { - privateLogger.error("Unexpected system shutdown!"); + String message = "Unexpected system shutdown. Job killed by the system."; + privateLogger.error(message); try { if (scratchDir != null) { deleteScratchDirectory(); } if (exception == null) { - exception = new RuntimeException("Unexpected system shutdown"); + exception = new RuntimeException(message); } logException(exception); ExecutionResult result = erm.close(exception); @@ -243,21 +246,32 @@ public final ExecutionResult start() throws ToolException { try { currentStep = "check"; privateCheck(); - check(); - currentStep = null; - erm.setSteps(getSteps()); - run(); + if (dryRun) { + logger.info("Dry run enabled. Sleep for 5 seconds and skip execution."); + Thread.sleep(5000); + } else { + currentStep = null; + erm.setSteps(getSteps()); + run(); + } } catch (ToolException e) { throw e; } catch (Exception e) { throw new ToolException(e); } + Runtime.getRuntime().removeShutdownHook(hook); } catch (Throwable e) { exception = e; + // Do not use a finally block to remove shutdownHook, as finally blocks will be executed even if the JVM is killed, + // and this would throw IllegalStateException("Shutdown in progress"); + try { + Runtime.getRuntime().removeShutdownHook(hook); + } catch (Exception e1) { + e.addSuppressed(e1); + } throw e; } finally { deleteScratchDirectory(); - Runtime.getRuntime().removeShutdownHook(hook); stopMemoryMonitor(); result = erm.close(exception); logException(exception); @@ -308,6 +322,7 @@ private void privateCheck() throws Exception { if (toolParams != null) { toolParams.updateParams(getParams()); } + check(); } /** @@ -460,7 +475,7 @@ protected final void step(String stepId, StepRunnable step) throws ToolException } catch (ToolException e) { throw e; } catch (Exception e) { - throw new ToolException("Exception from step " + stepId, e); + throw new ToolException("Exception from step '" + stepId + "'", e); } } else { privateLogger.info("------- Skip step " + stepId + " -------"); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java index 830110c7c23..4a1dd9cf02d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolRunner.java @@ -34,7 +34,6 @@ import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; public class ToolRunner { @@ -106,19 +105,22 @@ public ExecutionResult execute(Job job, String token) throws CatalogException, T * @throws ToolException if the execution fails */ public ExecutionResult execute(Job job, Path outDir, String token) throws CatalogException, ToolException { - return execute(job.getTool().getId(), new ObjectMap(job.getParams()), outDir, job.getId(), token); + return execute(job.getTool().getId(), new ObjectMap(job.getParams()), outDir, job.getId(), job.isDryRun(), token); } /** * Execute a tool + * * @param toolId Tool identifier. It can be either the tool id itself, or the class name. * @param params Params for the execution. * @param outDir Output directory. Mandatory - * @param token session id of the user that will execute the tool. + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(String toolId, ObjectMap params, Path outDir, String jobId, String token) throws ToolException { + public ExecutionResult execute(String toolId, ObjectMap params, Path outDir, String jobId, boolean dryRun, String token) + throws ToolException { OpenCgaTool tool; if (configuration != null && configuration.getAnalysis() != null && CollectionUtils.isNotEmpty(configuration.getAnalysis().getPackages())) { @@ -126,58 +128,67 @@ public ExecutionResult execute(String toolId, ObjectMap params, Path outDir, Str } else { tool = toolFactory.createTool(toolId); } - return tool.setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, token) + return tool.setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, dryRun, token) .start(); } /** * Execute a tool - * @param tool Tool class - * @param study Study id + * + * @param tool Tool class + * @param study Study id * @param toolParams Specific ToolParams for the execution. - * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param outDir Output directory. Mandatory + * @param jobId Job Id (if any) + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, String study, ToolParams toolParams, Path outDir, String jobId, String token) + public ExecutionResult execute(Class tool, String study, ToolParams toolParams, Path outDir, String jobId, + boolean dryRun, String token) throws ToolException { ObjectMap params = new ObjectMap(); params.putIfNotEmpty(ParamConstants.STUDY_PARAM, study); - return execute(tool, toolParams, params, outDir, jobId, token); + return execute(tool, toolParams, params, outDir, jobId, dryRun, token); } /** * Execute a tool - * @param tool Tool class + * + * @param tool Tool class * @param toolParams Specific ToolParams for the execution. - * @param params Params for the execution. - * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param params Params for the execution. + * @param outDir Output directory. Mandatory + * @param jobId Job Id (if any) + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, ToolParams toolParams, ObjectMap params, Path outDir, String jobId, String token) + public ExecutionResult execute(Class tool, ToolParams toolParams, ObjectMap params, Path outDir, String jobId, + boolean dryRun, String token) throws ToolException { if (toolParams != null) { params = toolParams.toObjectMap(params); } - return execute(tool, params, outDir, jobId, token); + return execute(tool, params, outDir, jobId, dryRun, token); } /** * Execute a tool - * @param tool Tool class + * + * @param tool Tool class * @param toolParams Specific ToolParams for the execution. - * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param outDir Output directory. Mandatory + * @param jobId Job Id (if any) + * @param dryRun Dry-run mode. + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, ToolParams toolParams, Path outDir, String jobId, String token) + public ExecutionResult execute(Class tool, ToolParams toolParams, Path outDir, String jobId, boolean dryRun, + String token) throws ToolException { ObjectMap params; if (toolParams != null) { @@ -185,24 +196,27 @@ public ExecutionResult execute(Class tool, ToolParams too } else { params = new ObjectMap(); } - return execute(tool, params, outDir, jobId, token); + return execute(tool, params, outDir, jobId, dryRun, token); } /** * Execute a tool - * @param tool Tool class + * + * @param tool Tool class * @param params Params for the execution. * @param outDir Output directory. Mandatory - * @param jobId Job Id (if any) - * @param token session id of the user that will execute the tool. + * @param jobId Job Id (if any) + * @param dryRun + * @param token session id of the user that will execute the tool. * @return Execution result * @throws ToolException if the execution fails */ - public ExecutionResult execute(Class tool, ObjectMap params, Path outDir, String jobId, String token) throws ToolException { + public ExecutionResult execute(Class tool, ObjectMap params, Path outDir, String jobId, boolean dryRun, + String token) throws ToolException { return toolFactory .createTool(tool) - .setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, token) + .setUp(opencgaHome, catalogManager, variantStorageManager, params, outDir, jobId, dryRun, token) .start(); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java index 2c00104e732..49ccd95298a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/julie/JulieTool.java @@ -25,7 +25,8 @@ resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - description = JulieTool.DESCRIPTION) + description = JulieTool.DESCRIPTION, + priority = Enums.Priority.HIGH) public class JulieTool extends OpenCgaTool { public static final String ID = "julie"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 847a2bb1ac6..6fb44855fb7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -71,9 +71,8 @@ import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.models.variant.VariantPruneParams; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -406,7 +405,6 @@ public DataResult familyIndex(String study, List familiesStr, bool throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { List trios = new LinkedList<>(); - List events = new LinkedList<>(); VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); VariantCatalogQueryUtils catalogUtils = new VariantCatalogQueryUtils(catalogManager); if (familiesStr.size() == 1 && familiesStr.get(0).equals(VariantQueryUtils.ALL)) { @@ -423,7 +421,7 @@ public DataResult familyIndex(String study, List familiesStr, bool } DataResult dataResult = engine.familyIndex(study, trios, params); getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream() - .flatMap(t->t.toList().stream()) + .flatMap(t -> t.toList().stream()) .collect(Collectors.toList()), token); return dataResult; }); @@ -439,11 +437,29 @@ public DataResult familyIndexBySamples(String study, Collection sa throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { Collection thisSamples = samples; + boolean allSamples; if (CollectionUtils.size(thisSamples) == 1 && thisSamples.iterator().next().equals(ParamConstants.ALL)) { thisSamples = getIndexedSamples(study, token); + allSamples = true; + } else { + allSamples = false; } List trios = catalogUtils.getTriosFromSamples(study, engine.getMetadataManager(), thisSamples, token); + if (trios.isEmpty()) { + String msg; + if (thisSamples.size() > 6) { + msg = "No trios found for " + thisSamples.size() + " samples"; + } else { + msg = "No trios found for samples " + thisSamples; + } + if (allSamples) { + logger.info(msg); + return new DataResult<>(0, Collections.singletonList(new Event(Event.Type.INFO, msg)), 0, Collections.emptyList(), 0); + } else { + throw new StorageEngineException(msg); + } + } DataResult dataResult = engine.familyIndex(study, trios, params); getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream() .flatMap(t -> t.toList().stream()) @@ -628,6 +644,7 @@ public VariantQueryResult get(Query inputQuery, QueryOptions queryOptio @SuppressWarnings("unchecked") public VariantQueryResult get(Query query, QueryOptions queryOptions, String token, Class clazz) throws CatalogException, IOException, StorageEngineException { + VariantQueryResult result = get(query, queryOptions, token); List variants; if (clazz == Variant.class) { @@ -641,16 +658,7 @@ public VariantQueryResult get(Query query, QueryOptions queryOptions, Str } else { throw new IllegalArgumentException("Unknown variant format " + clazz); } - return new VariantQueryResult<>( - result.getTime(), - result.getNumResults(), - result.getNumMatches(), - result.getEvents(), - variants, - result.getSamples(), - result.getSource(), - result.getApproximateCount(), - result.getApproximateCountSamplingSize(), null); + return new VariantQueryResult<>(result, variants); } @@ -881,7 +889,7 @@ public DataResult getSampleData(String variant, String study, QueryOpti VariantQueryResult result = new VariantQueryResult<>( ((int) stopWatch.getTime(TimeUnit.MILLISECONDS)), - 1, 1, new ArrayList<>(), Collections.singletonList(variantResult), null, null) + 1, 1, new ArrayList<>(), Collections.singletonList(variantResult), engine.getStorageEngineId()) .setNumSamples(sampleEntries.size()); if (exactNumSamples) { result.setApproximateCount(false); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java index 550c00b027a..882240d2865 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java @@ -26,6 +26,7 @@ import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import java.net.URI; import java.util.ArrayList; @@ -52,6 +53,7 @@ public void removeStudy(String study, URI outdir, String token) throws CatalogEx public void removeFile(String study, List inputFiles, URI outdir, String token) throws CatalogException, StorageEngineException { // Update study metadata BEFORE executing the operation and fetching files from Catalog + boolean force = variantStorageEngine.getOptions().getBoolean(VariantStorageOptions.FORCE.key()); StudyMetadata studyMetadata = synchronizeCatalogStudyFromStorage(study, token, true); List fileNames = new ArrayList<>(); @@ -62,7 +64,15 @@ public void removeFile(String study, List inputFiles, URI outdir, String if (!catalogIndexStatus.equals(VariantIndexStatus.READY)) { // Might be partially loaded in VariantStorage. Check FileMetadata FileMetadata fileMetadata = variantStorageEngine.getMetadataManager().getFileMetadata(studyMetadata.getId(), fileStr); - if (fileMetadata == null || fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE) { + boolean canBeRemoved; + if (force) { + // When forcing remove, just require the file to be registered in the storage + canBeRemoved = fileMetadata != null; + } else { + // Otherwise, require the file to be in status NONE + canBeRemoved = fileMetadata != null && fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE; + } + if (!canBeRemoved) { throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " + "IndexStatus = " + catalogIndexStatus); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index 042784b844a..5be8325e066 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -43,7 +43,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.variant.MutationalSignatureAnalysisExecutor; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java index c00f4943e08..57efbe0ef60 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateFamilyOperationTool.java @@ -25,7 +25,8 @@ @Tool(id = VariantAggregateFamilyOperationTool.ID, description = VariantAggregateFamilyOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAggregateFamilyOperationTool extends OperationTool { public static final String ID = "variant-aggregate-family"; public static final String DESCRIPTION = "Find variants where not all the samples are present, and fill the empty values."; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java index 49f799500bd..ab28fd9dea4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAggregateOperationTool.java @@ -16,14 +16,13 @@ package org.opencb.opencga.analysis.variant.operations; -import org.opencb.opencga.core.tools.annotations.Tool; -import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; +import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; -import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @Tool(id = VariantAggregateOperationTool.ID, description = VariantAggregateOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantAggregateOperationTool extends OperationTool { public static final String ID = "variant-aggregate"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java index 07f5e396ef7..47d36b85333 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationDeleteOperationTool.java @@ -24,7 +24,7 @@ @Tool(id = VariantAnnotationDeleteOperationTool.ID, description = VariantAnnotationDeleteOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantAnnotationDeleteOperationTool extends OperationTool { public static final String ID = "variant-annotation-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java index d3405dcaecd..aa08de950c7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationIndexOperationTool.java @@ -32,7 +32,8 @@ @Tool(id = VariantAnnotationIndexOperationTool.ID, description = VariantAnnotationIndexOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAnnotationIndexOperationTool extends OperationTool { public static final String ID = "variant-annotation-index"; public static final String DESCRIPTION = "Create and load variant annotations into the database"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java index 891838e34b3..b684d612519 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationRebuilderOperationTool.java @@ -22,7 +22,8 @@ @Tool(id = VariantAnnotationRebuilderOperationTool.ID, description = VariantAnnotationRebuilderOperationTool.ID, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAnnotationRebuilderOperationTool extends OperationTool { public static final String ID = "variant-annotation-rebuilder"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java index 33e400bcb12..9f7994935b0 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantAnnotationSaveOperationTool.java @@ -23,7 +23,8 @@ @Tool(id = VariantAnnotationSaveOperationTool.ID, description = VariantAnnotationSaveOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantAnnotationSaveOperationTool extends OperationTool { public static final String ID = "variant-annotation-save"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java index c7258c79f74..65dc97676cb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java @@ -31,7 +31,7 @@ @Deprecated @Tool(id = VariantFamilyIndexOperationTool.ID, description = VariantFamilyIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantFamilyIndexOperationTool extends OperationTool { public static final String ID = "variant-family-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java index bb3cb4d3f3a..db2da6c58e4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileDeleteOperationTool.java @@ -20,7 +20,7 @@ import org.apache.solr.common.StringUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.VariantFileDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileDeleteParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -33,7 +33,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Tool(id = VariantFileDeleteOperationTool.ID, description = VariantFileDeleteOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantFileDeleteOperationTool extends OperationTool { public static final String ID = "variant-file-delete"; @@ -55,6 +55,7 @@ protected void check() throws Exception { throw new ToolException("Missing file/s"); } params.put(VariantStorageOptions.RESUME.key(), variantFileDeleteParams.isResume()); + params.put(VariantStorageOptions.FORCE.key(), variantFileDeleteParams.isForce()); } @Override diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java index 1178c9a8081..f45b7011650 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFileIndexJobLauncherTool.java @@ -16,8 +16,8 @@ import org.opencb.opencga.core.models.file.FileInternal; import org.opencb.opencga.core.models.file.VariantIndexStatus; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileIndexJobLauncherParams; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @@ -31,7 +31,7 @@ import static org.opencb.opencga.catalog.db.api.FileDBAdaptor.QueryParams.*; @Tool(id = VariantFileIndexJobLauncherTool.ID, description = VariantFileIndexJobLauncherTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantFileIndexJobLauncherTool extends OpenCgaToolScopeStudy { public static final String ID = "variant-index-job-launcher"; @@ -142,7 +142,7 @@ protected void run() throws Exception { String jobId = buildJobId(file); Job job = catalogManager.getJobManager().submit(getStudy(), VariantIndexOperationTool.ID, Enums.Priority.MEDIUM, indexParams.toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study)), jobId, "Job generated by " + getId(), - Collections.emptyList(), jobTags, getToken()).first(); + Collections.emptyList(), jobTags, getJobId(), null, false, getToken()).first(); submittedJobs++; logger.info("[{}] Create variant-index job '{}' for file '{}'{}", submittedJobs, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java index cce9fc91e8a..d4e7270d2e6 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java @@ -17,13 +17,9 @@ package org.opencb.opencga.analysis.variant.operations; import io.jsonwebtoken.lang.Collections; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.VariantIndexParams; -import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.storage.core.StoragePipelineResult; @@ -34,14 +30,12 @@ import java.net.URI; import java.nio.file.Files; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.opencb.opencga.analysis.variant.manager.operations.VariantFileIndexerOperationManager.*; @Tool(id = VariantIndexOperationTool.ID, description = VariantIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantIndexOperationTool extends OperationTool { public static final String ID = "variant-index"; public static final String DESCRIPTION = "Index variant files into the variant storage"; @@ -112,9 +106,6 @@ protected void check() throws Exception { protected List getSteps() { List steps = new ArrayList<>(); steps.add(getId()); - if (indexParams.isFamily()) { - steps.add("family-index"); - } return steps; } @@ -155,26 +146,5 @@ protected void run() throws Exception { } } }); - - if (indexParams.isFamily()) { - step("family-index", () -> { - if (inputFiles.isEmpty()) { - // Nothing to do! - return; - } - OpenCGAResult fileResult = getCatalogManager().getFileManager() - .search(study, - new Query(FileDBAdaptor.QueryParams.URI.key(), inputFiles), - new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.SAMPLE_IDS.key()), getToken()); - - Set samples = new HashSet<>(); - for (org.opencb.opencga.core.models.file.File file : fileResult.getResults()) { - samples.addAll(file.getSampleIds()); - } - if (!samples.isEmpty()) { - variantStorageManager.familyIndexBySamples(study, samples, params, getToken()); - } - }); - } } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java index faaee5e681a..f2426fa5f04 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantPruneOperationTool.java @@ -1,12 +1,12 @@ package org.opencb.opencga.analysis.variant.operations; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.VariantPruneParams; +import org.opencb.opencga.core.models.operations.variant.VariantPruneParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @Tool(id = VariantPruneOperationTool.ID, description = VariantPruneOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantPruneOperationTool extends OperationTool { public static final String DESCRIPTION = "Prune orphan variants from studies in a project."; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java index 14c456cc592..8c5620291be 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSampleDeleteOperationTool.java @@ -20,7 +20,7 @@ import org.apache.solr.common.StringUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantSampleDeleteParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -33,7 +33,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Tool(id = VariantSampleDeleteOperationTool.ID, description = VariantSampleDeleteOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantSampleDeleteOperationTool extends OperationTool { public static final String ID = "variant-sample-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java index b2b6734cf87..a26afb81955 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreDeleteOperationTool.java @@ -21,7 +21,7 @@ import org.opencb.opencga.core.models.common.Enums; @Tool(id= VariantScoreDeleteOperationTool.ID, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantScoreDeleteOperationTool extends OperationTool { public static final String ID = "variant-score-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java index d7049df006e..e863a630167 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantScoreIndexOperationTool.java @@ -16,18 +16,16 @@ package org.opencb.opencga.analysis.variant.operations; -import org.opencb.opencga.core.tools.annotations.Tool; -import org.opencb.opencga.core.models.operations.variant.VariantScoreIndexParams; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.operations.variant.VariantScoreIndexParams; +import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor; import java.net.URI; -import static org.opencb.opencga.core.api.ParamConstants.STUDY_PARAM; - @Tool(id= VariantScoreIndexOperationTool.ID, description = VariantScoreIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantScoreIndexOperationTool extends OperationTool { public static final String ID = "variant-score-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java index 24f54414f78..a812c8165c5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryAnnotationIndexOperationTool.java @@ -23,7 +23,8 @@ @Tool(id = VariantSecondaryAnnotationIndexOperationTool.ID, description = VariantSecondaryAnnotationIndexOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantSecondaryAnnotationIndexOperationTool extends OperationTool { public static final String ID = "variant-secondary-annotation-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java index b452fffb820..92e5643dedc 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesDeleteOperationTool.java @@ -22,7 +22,8 @@ @Tool(id = VariantSecondaryIndexSamplesDeleteOperationTool.ID, description = VariantSecondaryIndexSamplesDeleteOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) @Deprecated public class VariantSecondaryIndexSamplesDeleteOperationTool extends OperationTool { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java index e31f32a62a1..9671c0fbda2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondaryIndexSamplesOperationTool.java @@ -21,7 +21,7 @@ import org.opencb.opencga.core.models.common.Enums; @Tool(id = VariantSecondaryIndexSamplesOperationTool.ID, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) @Deprecated public class VariantSecondaryIndexSamplesOperationTool extends OperationTool { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java index d4dd7ab8a89..259433a3b9e 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java @@ -30,7 +30,7 @@ import java.util.List; @Tool(id = VariantSecondarySampleIndexOperationTool.ID, description = VariantSecondarySampleIndexOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantSecondarySampleIndexOperationTool extends OperationTool { public static final String ID = "variant-secondary-sample-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java index 43124238717..0853cfa5d5c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsDeleteOperationTool.java @@ -25,7 +25,8 @@ @Tool(id = VariantStatsDeleteOperationTool.ID, description = VariantStatsDeleteOperationTool.DESCRIPTION, type = Tool.Type.OPERATION, scope = Tool.Scope.STUDY, - resource = Enums.Resource.VARIANT) + resource = Enums.Resource.VARIANT, + priority = Enums.Priority.HIGH) public class VariantStatsDeleteOperationTool extends OperationTool { public static final String ID = "variant-stats-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java index adae4198666..4ab8f6c58c9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStatsIndexOperationTool.java @@ -39,7 +39,7 @@ * Created by jacobo on 06/03/15. */ @Tool(id = VariantStatsIndexOperationTool.ID, resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, - description = VariantStatsIndexOperationTool.DESCRIPTION) + description = VariantStatsIndexOperationTool.DESCRIPTION, priority = Enums.Priority.HIGH) public class VariantStatsIndexOperationTool extends OperationTool { public final static String ID = "variant-stats-index"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java index 47c33107bcf..04606492bff 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataRepairTool.java @@ -26,7 +26,7 @@ import static org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams.What.*; @Tool(id = VariantStorageMetadataRepairTool.ID, description = VariantStorageMetadataRepairTool.DESCRIPTION, - type = Tool.Type.OPERATION, scope = Tool.Scope.GLOBAL, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, scope = Tool.Scope.GLOBAL, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantStorageMetadataRepairTool extends OperationTool { public static final String ID = "variant-storage-metadata-repair"; public static final String DESCRIPTION = "Execute some repairs on Variant Storage Metadata. Advanced users only."; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java index 6e49349369c..8f7cac4f6a2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStorageMetadataSynchronizeOperationTool.java @@ -1,12 +1,12 @@ package org.opencb.opencga.analysis.variant.operations; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; +import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @Tool(id = VariantStorageMetadataSynchronizeOperationTool.ID, resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, - scope = Tool.Scope.STUDY, description = VariantStorageMetadataSynchronizeOperationTool.DESCRIPTION) + scope = Tool.Scope.STUDY, description = VariantStorageMetadataSynchronizeOperationTool.DESCRIPTION, priority = Enums.Priority.HIGH) public class VariantStorageMetadataSynchronizeOperationTool extends OperationTool { public static final String ID = "variant-storage-metadata-synchronize"; public static final String DESCRIPTION = "Synchronize catalog with variant storage metadata"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java index d24a21668e2..baa52c6f2c2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantStudyDeleteOperationTool.java @@ -19,7 +19,7 @@ import org.apache.solr.common.StringUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -32,7 +32,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ @Tool(id = VariantStudyDeleteOperationTool.ID, description = VariantStudyDeleteOperationTool.DESCRIPTION, - type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT) + type = Tool.Type.OPERATION, resource = Enums.Resource.VARIANT, priority = Enums.Priority.HIGH) public class VariantStudyDeleteOperationTool extends OperationTool { public static final String ID = "variant-study-delete"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index ae2b5072473..d68f363b36d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -31,6 +31,8 @@ public abstract class DockerWrapperAnalysisExecutor extends OpenCgaToolExecutor public static final String STDOUT_FILENAME = "stdout.txt"; public static final String STDERR_FILENAME = "stderr.txt"; + public static final String DOCKER_CLI_MSG = "Docker CLI: "; + public String getDockerImageName() { return "opencb/opencga-ext-tools"; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java index 3d8253861a4..c4a9d87d4f7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysis.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.tools.annotations.Tool; @@ -52,6 +53,7 @@ protected void run() throws Exception { getToolExecutor(ExomiserWrapperAnalysisExecutor.class) .setStudyId(study) .setSampleId(analysisParams.getSample()) + .setClinicalAnalysisType(ClinicalAnalysis.Type.valueOf(analysisParams.getClinicalAnalysisType())) .execute(); }); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java index 86bb3760b3c..a6325932fb9 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/exomiser/ExomiserWrapperAnalysisExecutor.java @@ -18,6 +18,7 @@ import org.opencb.opencga.catalog.managers.FamilyManager; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.tools.annotations.ToolExecutor; @@ -50,6 +51,7 @@ public class ExomiserWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut private String studyId; private String sampleId; + private ClinicalAnalysis.Type clinicalAnalysisType; private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -100,22 +102,24 @@ public void run() throws ToolException { // Check multi-sample (family) analysis File pedigreeFile = null; Pedigree pedigree = null; - if (individual.getMother() != null && individual.getMother().getId() != null - && individual.getFather() != null && individual.getFather().getId() != null) { - Family family = IndividualQcUtils.getFamilyByIndividualId(getStudyId(), individual.getId(), - getVariantStorageManager().getCatalogManager(), getToken()); - if (family != null) { - pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId()); - } - - if (pedigree != null) { - if (individual.getFather() != null) { - samples.add(individual.getFather().getSamples().get(0).getId()); + if (clinicalAnalysisType == ClinicalAnalysis.Type.FAMILY) { + if (individual.getMother() != null && individual.getMother().getId() != null + && individual.getFather() != null && individual.getFather().getId() != null) { + Family family = IndividualQcUtils.getFamilyByIndividualId(getStudyId(), individual.getId(), + getVariantStorageManager().getCatalogManager(), getToken()); + if (family != null) { + pedigree = FamilyManager.getPedigreeFromFamily(family, individual.getId()); } - if (individual.getMother() != null) { - samples.add(individual.getMother().getSamples().get(0).getId()); + + if (pedigree != null) { + if (individual.getFather() != null) { + samples.add(individual.getFather().getSamples().get(0).getId()); + } + if (individual.getMother() != null) { + samples.add(individual.getMother().getSamples().get(0).getId()); + } + pedigreeFile = createPedigreeFile(family, pedigree); } - pedigreeFile = createPedigreeFile(family, pedigree); } } File sampleFile = createSampleFile(individual, hpos, pedigree); @@ -192,7 +196,9 @@ public void run() throws ToolException { .append(" --spring.config.location=/jobdir/").append(EXOMISER_PROPERTIES_TEMPLATE_FILENAME); // Execute command and redirect stdout and stderr to the files - logger.info("{}: Docker command line: {}", ID, sb); + String msg = DOCKER_CLI_MSG + sb; + logger.info(msg); + addWarning(msg); runCommandLine(sb.toString()); } @@ -483,4 +489,13 @@ public ExomiserWrapperAnalysisExecutor setSampleId(String sampleId) { this.sampleId = sampleId; return this; } + + public ClinicalAnalysis.Type getClinicalAnalysisType() { + return clinicalAnalysisType; + } + + public ExomiserWrapperAnalysisExecutor setClinicalAnalysisType(ClinicalAnalysis.Type clinicalAnalysisType) { + this.clinicalAnalysisType = clinicalAnalysisType; + return this; + } } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java index bdc744fc49c..9339ddf40e2 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java @@ -16,10 +16,7 @@ package org.opencb.opencga.analysis.alignment; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.*; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -37,8 +34,11 @@ import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.alignment.AlignmentGeneCoverageStatsParams; +import org.opencb.opencga.core.models.alignment.AlignmentIndexParams; +import org.opencb.opencga.core.models.alignment.CoverageIndexParams; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileLinkParams; +import org.opencb.opencga.core.models.file.FileRelatedFile; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.user.User; @@ -49,9 +49,11 @@ import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collections; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -259,7 +261,6 @@ public void setUpCatalogManager() throws CatalogException { // assertEquals(2, individual.getSamples().size()); } - @Test public void geneCoverageStatsTest() throws IOException, ToolException, CatalogException { Path outdir = Paths.get(opencga.createTmpOutdir("_genecoveragestats")); @@ -277,7 +278,7 @@ public void geneCoverageStatsTest() throws IOException, ToolException, CatalogEx String geneName = "BRCA2"; params.setGenes(Arrays.asList(geneName)); - toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, new ObjectMap(), outdir, null, token); + toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, new ObjectMap(), outdir, "coverage-job-id", false, token); bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(bamFilename, "", "", "", null, null, null, null, null), false, token).first(); @@ -285,4 +286,150 @@ public void geneCoverageStatsTest() throws IOException, ToolException, CatalogEx assertEquals(geneName, bamFile.getQualityControl().getCoverage().getGeneCoverageStats().get(0).getGeneName()); assertEquals(10, bamFile.getQualityControl().getCoverage().getGeneCoverageStats().get(0).getStats().size()); } + + @Test + public void testNonReadOnlyAlignmentIndex() throws Exception { + Path nonReadOnlyDir = Paths.get(opencga.createTmpOutdir("_non_readonly_alignment_index")); + Path bamPath = Paths.get(opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam").getPath()); + String bamFilename = "NonReadOnlyAligmentIndex_" + bamPath.getFileName(); + Files.copy(bamPath, nonReadOnlyDir.resolve(bamFilename)); + + File bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(nonReadOnlyDir.resolve(bamFilename).toAbsolutePath().toString(), "non_readonly_alignment_index", "", "", null, null, null, + null, null), true, token).first(); + + // Run alignment index + AlignmentIndexParams params = new AlignmentIndexParams(); + params.setFileId(bamFile.getId()); + Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); + toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-non-readonly-coverage-index", false, token); + + // Checking BAI file + Path baiPath = nonReadOnlyDir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); + Assert.assertTrue(Files.exists(baiPath)); + + // Checking BAI file is registered in the BAM file internals + File baiFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFilename + AlignmentConstants.BAI_EXTENSION), QueryOptions.empty(), true, token).first(); + bamFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFile.getId()), QueryOptions.empty(), true, token).first(); + Assert.assertEquals(baiFile.getId(), bamFile.getInternal().getAlignment().getIndex().getFileId()); + } + + @Test + public void testReadOnlyAlignmentIndex() throws Exception { + Path readOnlyDir = Paths.get(opencga.createTmpOutdir("_readonly_for_alignment_index")); + Path bamPath = Paths.get(opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam").getPath()); + String bamFilename = "ReadOnlyAligmentIndex_" + bamPath.getFileName(); + Files.copy(bamPath, readOnlyDir.resolve(bamFilename)); + + // Make read-only + Runtime.getRuntime().exec("chmod 555 " + readOnlyDir.toAbsolutePath()); + + File bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(readOnlyDir.resolve(bamFilename).toAbsolutePath().toString(), "readonly_alignment_index", "", "", null, null, null, + null, null), true, token).first(); + + // Run alignment index + AlignmentIndexParams params = new AlignmentIndexParams(); + params.setFileId(bamFile.getId()); + Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); + toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-readonly-coverage-index", false, token); + + // Checking BAI file + Path baiPath = alignmentIndexOutdir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); + Assert.assertTrue(Files.exists(baiPath)); + + // Checking BAI file is registered in the BAM file internals + File baiFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFilename + AlignmentConstants.BAI_EXTENSION), QueryOptions.empty(), true, token).first(); + bamFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFile.getId()), QueryOptions.empty(), true, token).first(); + Assert.assertEquals(baiFile.getId(), bamFile.getInternal().getAlignment().getIndex().getFileId()); + + Runtime.getRuntime().exec("chmod 777 " + readOnlyDir.toAbsolutePath()); + } + + @Test + public void testNonReadOnlyCoverageIndex() throws Exception { + Path nonReadOnlyDir = Paths.get(opencga.createTmpOutdir("_non_readonly_for_coverage_index")); + Path bamPath = Paths.get(opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam").getPath()); + String bamFilename = "NonReadOnlyCoverageIndex_" + bamPath.getFileName(); + Files.copy(bamPath, nonReadOnlyDir.resolve(bamFilename)); + + File bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(nonReadOnlyDir.resolve(bamFilename).toAbsolutePath().toString(), "non_readonly_alignment_coverage_index", "", "", null, null, null, + null, null), true, token).first(); + + // Run alignment index + AlignmentIndexParams indexParams = new AlignmentIndexParams(); + indexParams.setFileId(bamFile.getId()); + Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); + toolRunner.execute(AlignmentIndexOperation.class, indexParams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-non-readonly-alignment-coverage-index", false, token); + + // Checking BAI file + Path baiPath = nonReadOnlyDir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); + Assert.assertTrue(Files.exists(baiPath)); + + // Checking BAI file is registered in the BAM file internals + File baiFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFilename + AlignmentConstants.BAI_EXTENSION), QueryOptions.empty(), true, token).first(); + bamFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFile.getId()), QueryOptions.empty(), true, token).first(); + Assert.assertEquals(baiFile.getId(), bamFile.getInternal().getAlignment().getIndex().getFileId()); + + // Run coverage index + CoverageIndexParams coverageOarams = new CoverageIndexParams(); + coverageOarams.setBamFileId(bamFile.getId()); + coverageOarams.setBaiFileId(baiFile.getId()); + Path coverageIndexOutdir = Paths.get(opencga.createTmpOutdir("_coverage_index")); + toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", false, token); + + // Checking BW file + Path bwPath = nonReadOnlyDir.resolve(bamFilename + AlignmentConstants.BIGWIG_EXTENSION); + Assert.assertTrue(Files.exists(bwPath)); + + // Checking BAM file is registered in the related files of BW file + File bwFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bwPath.getFileName().toString()), QueryOptions.empty(), true, token).first(); + Assert.assertEquals(bamFile.getId(), bwFile.getRelatedFiles().get(0).getFile().getId()); + Assert.assertEquals(FileRelatedFile.Relation.ALIGNMENT, bwFile.getRelatedFiles().get(0).getRelation()); + } + + @Test + public void testReadOnlyCoverageIndex() throws Exception { + Path readOnlyDir = Paths.get(opencga.createTmpOutdir("_readonly_for_coverage_index")); + Path bamPath = Paths.get(opencga.getResourceUri("biofiles/HG00096.chrom20.small.bam").getPath()); + String bamFilename = "ReadOnlyCoverageIndex_" + bamPath.getFileName(); + Files.copy(bamPath, readOnlyDir.resolve(bamFilename)); + + File bamFile = catalogManager.getFileManager().link(STUDY, new FileLinkParams(readOnlyDir.resolve(bamFilename).toAbsolutePath().toString(), "readonly_alignment_coverage_index", "", "", null, null, null, + null, null), true, token).first(); + + // Run alignment index + AlignmentIndexParams indexParams = new AlignmentIndexParams(); + indexParams.setFileId(bamFile.getId()); + Path alignmentIndexOutdir = Paths.get(opencga.createTmpOutdir("_alignment_index")); + toolRunner.execute(AlignmentIndexOperation.class, indexParams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), alignmentIndexOutdir, "jobId-readonly-coverage-index", false, token); + + // Checking BAI file + Path baiPath = readOnlyDir.resolve(bamFilename + AlignmentConstants.BAI_EXTENSION); + Assert.assertTrue(Files.exists(baiPath)); + + // Checking BAI file is registered in the BAM file internals + File baiFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFilename + AlignmentConstants.BAI_EXTENSION), QueryOptions.empty(), true, token).first(); + bamFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bamFile.getId()), QueryOptions.empty(), true, token).first(); + Assert.assertEquals(baiFile.getId(), bamFile.getInternal().getAlignment().getIndex().getFileId()); + + // Make read-only + Runtime.getRuntime().exec("chmod 555 " + readOnlyDir.toAbsolutePath()); + + // Run coverage index + CoverageIndexParams coverageOarams = new CoverageIndexParams(); + coverageOarams.setBamFileId(bamFile.getId()); + coverageOarams.setBaiFileId(baiFile.getId()); + Path coverageIndexOutdir = Paths.get(opencga.createTmpOutdir("_coverage_index")); + toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", false, token); + + // Checking BW file + Path bwPath = coverageIndexOutdir.resolve(bamFilename + AlignmentConstants.BIGWIG_EXTENSION); + Assert.assertTrue(Files.exists(bwPath)); + + // Checking BAM file is registered in the related files of BW file + File bwFile = catalogManager.getFileManager().get(STUDY, Collections.singletonList(bwPath.getFileName().toString()), QueryOptions.empty(), true, token).first(); + Assert.assertEquals(bamFile.getId(), bwFile.getRelatedFiles().get(0).getFile().getId()); + Assert.assertEquals(FileRelatedFile.Relation.ALIGNMENT, bwFile.getRelatedFiles().get(0).getRelation()); + + Runtime.getRuntime().exec("chmod 777 " + readOnlyDir.toAbsolutePath()); + } } \ No newline at end of file diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java index 26bf31c9dfc..f77a1025377 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalAnalysisUtilsTest.java @@ -76,6 +76,10 @@ public static AbstractClinicalManagerTest getClinicalTest(OpenCGATestExternalRes variantStorageManager.index(clinicalTest.studyFqn, "HG005.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "HG006.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); variantStorageManager.index(clinicalTest.studyFqn, "HG007.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); + variantStorageManager.index(clinicalTest.studyFqn, "HG104.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); + variantStorageManager.index(clinicalTest.studyFqn, "HG105.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); + variantStorageManager.index(clinicalTest.studyFqn, "HG106.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); + variantStorageManager.index(clinicalTest.studyFqn, "HG107.1k.vcf.gz", outDir.toString(), storageOptions, clinicalTest.token); return clinicalTest; } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java index dae59da96c3..73235db4d6d 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java @@ -48,6 +48,7 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.core.tools.result.ExecutionResult; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import java.io.File; @@ -138,8 +139,11 @@ public void customAnalysisFromClinicalAnalysisTest() throws Exception { .setConfig(config); ExecutionResult result = customAnalysis.start(); - - checkInterpretation(238, result); + int expected = opencga.getVariantStorageManager().get(new VariantQuery(query), + new QueryOptions(QueryOptions.LIMIT, 500), clinicalTest.token) + .getResults().size(); + assertNotEquals(0, expected); + checkInterpretation(expected, result); } @Test diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java index 387b439571b..441f03d06a3 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysisTest.java @@ -1,17 +1,18 @@ package org.opencb.opencga.analysis.clinical.exomiser; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; +import org.apache.commons.lang3.StringUtils; +import org.junit.*; +import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.tools.variant.VariantNormalizer; import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.clinical.ClinicalAnalysisUtilsTest; import org.opencb.opencga.analysis.variant.OpenCGATestExternalResource; +import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.AbstractClinicalManagerTest; import org.opencb.opencga.core.exceptions.ToolException; @@ -24,8 +25,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Collections; +import java.util.List; +import static com.mongodb.assertions.Assertions.assertFalse; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -47,8 +51,8 @@ public static void setUp() throws Exception { clinicalTest = ClinicalAnalysisUtilsTest.getClinicalTest(opencga); } - @After - public void tearDown() throws Exception { + @AfterClass + public static void tearDown() throws Exception { opencga.clear(); } @@ -68,7 +72,7 @@ public void testNormalization() throws NonStandardCompliantSampleField { } @Test - public void singleExomiserAnalysis() throws IOException, CatalogException, ToolException { + public void singleSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); prepareExomiserData(); @@ -92,14 +96,30 @@ public void singleExomiserAnalysis() throws IOException, CatalogException, ToolE .get(clinicalTest.studyFqn, clinicalTest.CA_ID2, QueryOptions.empty(), clinicalTest.token).first(); assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size()); assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0); + + // Check Exomiser docker CLI + boolean pedFound = false; + for (Event event : result.getEvents()) { + if (event.getType() == Event.Type.WARNING && StringUtils.isNotEmpty(event.getMessage()) + && event.getMessage().startsWith(DockerWrapperAnalysisExecutor.DOCKER_CLI_MSG)) { + List splits = Arrays.asList(event.getMessage().split(" ")); + pedFound = splits.contains("--ped") && splits.contains("/jobdir/" + clinicalTest.PROBAND_ID2 + ".ped"); + } + } + assertFalse(pedFound); + + // Only proband sample is returned in primary findings + for (ClinicalVariant cv : clinicalAnalysis.getInterpretation().getPrimaryFindings()) { + assertEquals(1, cv.getStudies().get(0).getSamples().size()); + } } @Test - public void familyExomiserAnalysis() throws IOException, CatalogException, ToolException { + public void trioFamilyExomiserAnalysis() throws IOException, CatalogException, ToolException { assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); prepareExomiserData(); - outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_family")); + outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_family")); ClinicalAnalysis clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager() .get(clinicalTest.studyFqn, clinicalTest.CA_ID3, QueryOptions.empty(), clinicalTest.token).first(); @@ -118,6 +138,67 @@ public void familyExomiserAnalysis() throws IOException, CatalogException, ToolE .get(clinicalTest.studyFqn, clinicalTest.CA_ID3, QueryOptions.empty(), clinicalTest.token).first(); assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size()); assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0); + + // Check Exomiser docker CLI + boolean pedFound = false; + for (Event event : result.getEvents()) { + if (event.getType() == Event.Type.WARNING && StringUtils.isNotEmpty(event.getMessage()) + && event.getMessage().startsWith(DockerWrapperAnalysisExecutor.DOCKER_CLI_MSG)) { + List splits = Arrays.asList(event.getMessage().split(" ")); + pedFound = splits.contains("--ped") && splits.contains("/jobdir/" + clinicalTest.PROBAND_ID3 + ".ped"); + } + } + assertTrue(pedFound); + + // All family samples are returned in primary findings + for (ClinicalVariant cv : clinicalAnalysis.getInterpretation().getPrimaryFindings()) { + assertEquals(4, cv.getStudies().get(0).getSamples().size()); + } + + System.out.println("results at out dir = " + outDir.toAbsolutePath()); + } + + @Test + public void trioSingleExomiserAnalysis() throws IOException, CatalogException, ToolException { + assumeThat(Paths.get("/opt/opencga/analysis/resources/exomiser").toFile().exists(), is(true)); + + prepareExomiserData(); + outDir = Paths.get(opencga.createTmpOutdir("_interpretation_analysis_trio_single")); + + ClinicalAnalysis clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager() + .get(clinicalTest.studyFqn, clinicalTest.CA_ID4, QueryOptions.empty(), clinicalTest.token).first(); + assertEquals(0, clinicalAnalysis.getSecondaryInterpretations().size()); + + ExomiserInterpretationAnalysis exomiser = new ExomiserInterpretationAnalysis(); + + exomiser.setUp(opencga.getOpencgaHome().toAbsolutePath().toString(), new ObjectMap(), outDir, clinicalTest.token); + exomiser.setStudyId(clinicalTest.studyFqn) + .setClinicalAnalysisId(clinicalTest.CA_ID4); + + ExecutionResult result = exomiser.start(); + + // Refresh clinical analysis + clinicalAnalysis = clinicalTest.catalogManager.getClinicalAnalysisManager() + .get(clinicalTest.studyFqn, clinicalTest.CA_ID4, QueryOptions.empty(), clinicalTest.token).first(); + assertEquals(1, clinicalAnalysis.getSecondaryInterpretations().size()); + assertTrue(clinicalAnalysis.getSecondaryInterpretations().get(0).getPrimaryFindings().size() > 0); + + // Check Exomiser docker CLI + boolean pedFound = false; + for (Event event : result.getEvents()) { + if (event.getType() == Event.Type.WARNING && StringUtils.isNotEmpty(event.getMessage()) + && event.getMessage().startsWith(DockerWrapperAnalysisExecutor.DOCKER_CLI_MSG)) { + List splits = Arrays.asList(event.getMessage().split(" ")); + pedFound = splits.contains("--ped") && splits.contains("/jobdir/" + clinicalTest.PROBAND_ID4 + ".ped"); + } + } + assertFalse(pedFound); + + // Only proband sample is returned in primary findings + for (ClinicalVariant cv : clinicalAnalysis.getInterpretation().getPrimaryFindings()) { + assertEquals(1, cv.getStudies().get(0).getSamples().size()); + } + System.out.println("results at out dir = " + outDir.toAbsolutePath()); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java index af5c8d9be8e..91ac3624471 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java @@ -202,7 +202,7 @@ public void testPedigreeGraphAnalysis() throws ToolException, IOException { params.setFamilyId(family.getId()); toolRunner.execute(PedigreeGraphAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, studyId), outDir, null, - sessionIdUser); + false, sessionIdUser); String b64Image = PedigreeGraphUtils.getB64Image(outDir); MatcherAssert.assertThat(b64Image, CoreMatchers.startsWith("iVBORw0KGgoAAAANSUhEUgAAAeAAAAHg")); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java index 4fa44f7cd98..74cc8339db8 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaManagerTest.java @@ -165,7 +165,7 @@ public void setUp() throws Throwable { KnockoutAnalysisParams params = new KnockoutAnalysisParams(); params.setSample(file.getSampleIds()); - toolRunner.execute(KnockoutAnalysis.class, params.toObjectMap(), outDir, null, ownerToken); + toolRunner.execute(KnockoutAnalysis.class, params.toObjectMap(), outDir, null, false, ownerToken); File file = catalogManager.getFileManager().link(STUDY, new FileLinkParams() diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index 1f05dfc5dd3..5c5956d7cf7 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -214,8 +214,6 @@ public Path isolateOpenCGA() throws IOException { Files.createDirectories(conf); Files.createDirectories(userHome); - catalogManagerExternalResource.getConfiguration().getAdmin().setSecretKey(null); - catalogManagerExternalResource.getConfiguration().getAdmin().setAlgorithm(null); catalogManagerExternalResource.getConfiguration().serialize( new FileOutputStream(conf.resolve("configuration.yml").toFile())); InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index 59d2ba43d5d..c91909d3abb 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -74,6 +74,7 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.Location; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.ProjectCreateParams; @@ -313,7 +314,7 @@ public void testMalformedVcfFileIndex() throws Exception { try { toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams().setFile(file.getId()).setAnnotate(true), - Paths.get(opencga.createTmpOutdir()), null, token); + Paths.get(opencga.createTmpOutdir()), null, false, token); } catch (ToolException e) { System.out.println(ExceptionUtils.prettyExceptionMessage(e, true, true)); } @@ -329,7 +330,7 @@ public void testVariantStats() throws Exception { VariantStatsAnalysis variantStatsAnalysis = new VariantStatsAnalysis() .setStudy(STUDY) .setSamples(samples.subList(1, 3)); - variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); ExecutionResult ar = variantStatsAnalysis.start(); checkExecutionResult(ar); @@ -354,7 +355,7 @@ public void testVariantStatsTwoCohorts() throws Exception { VariantStatsAnalysis variantStatsAnalysis = new VariantStatsAnalysis() .setStudy(STUDY) .setCohort(Arrays.asList("c1", "c2")); - variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + variantStatsAnalysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); ExecutionResult ar = variantStatsAnalysis.start(); checkExecutionResult(ar); @@ -381,7 +382,7 @@ public void testVariantStatsWithFilter() throws Exception { VariantStatsAnalysisParams params = new VariantStatsAnalysisParams() .setSamples(samples.subList(1, 3)) .setRegion(region); - ExecutionResult ar = toolRunner.execute(VariantStatsAnalysis.class, STUDY, params, outDir, "", token); + ExecutionResult ar = toolRunner.execute(VariantStatsAnalysis.class, STUDY, params, outDir, "", false, token); checkExecutionResult(ar); MutableInt count = new MutableInt(); @@ -479,7 +480,7 @@ private ExecutionResult sampleVariantStats(String region, String indexId, boolea params.getVariantQuery() .appendQuery(query) .setRegion(region); - ExecutionResult result = toolRunner.execute(SampleVariantStatsAnalysis.class, STUDY, params, outDir, null, token); + ExecutionResult result = toolRunner.execute(SampleVariantStatsAnalysis.class, STUDY, params, outDir, null, false, token); if (nothingToDo) { assertEquals("All samples stats indexed. Nothing to do!", result.getEvents().get(0).getMessage()); @@ -515,7 +516,7 @@ public void testCohortStats() throws Exception { CohortVariantStatsAnalysis analysis = new CohortVariantStatsAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_cohort_stats")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); List samples = file.getSampleIds(); analysis.setStudy(STUDY) .setSamplesQuery(new Query(SampleDBAdaptor.QueryParams.ID.key(), samples.subList(0, 3))); @@ -532,7 +533,7 @@ public void testCohortStatsIndex() throws Exception { .setIndex(true); ExecutionResult result = toolRunner.execute(CohortVariantStatsAnalysis.class, STUDY, toolParams, - outDir, null, token); + outDir, null, false, token); checkExecutionResult(result, storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)); Cohort cohort = catalogManager.getCohortManager().get(STUDY, StudyEntry.DEFAULT_COHORT, new QueryOptions(), token).first(); @@ -556,7 +557,7 @@ public void testCohortStatsIndex() throws Exception { outDir = Paths.get(opencga.createTmpOutdir("_cohort_stats_index_2")); System.out.println("output = " + outDir.toAbsolutePath()); - result = toolRunner.execute(CohortVariantStatsAnalysis.class, STUDY, toolParams, outDir, null, token); + result = toolRunner.execute(CohortVariantStatsAnalysis.class, STUDY, toolParams, outDir, null, false, token); checkExecutionResult(result, storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)); @@ -579,7 +580,7 @@ public void testExport() throws Exception { variantExportParams.setOutputFileName("chr22.vcf"); toolRunner.execute(VariantExportTool.class, - variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); assertTrue(outDir.resolve(variantExportParams.getOutputFileName() + ".gz").toFile().exists()); } @@ -594,7 +595,7 @@ public void testExportVep() throws Exception { variantExportParams.setOutputFileName("chr1-5-22"); variantExportParams.setOutputFileFormat(VariantWriterFactory.VariantOutputFormat.ENSEMBL_VEP.name()); toolRunner.execute(VariantExportTool.class, - variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); } @Test @@ -614,7 +615,7 @@ public void testExportTped() throws Exception { variantExportParams.setInclude("id,studies.samples"); toolRunner.execute(VariantExportTool.class, - variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + variantExportParams.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); System.out.println(outDir); Path tped = outDir.resolve(variantExportParams.getOutputFileName() + ".tped"); @@ -635,7 +636,7 @@ public void testGwas() throws Exception { GwasAnalysis analysis = new GwasAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_gwas")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); List samples = file.getSampleIds(); analysis.setStudy(STUDY) .setCaseCohortSamplesQuery(new Query(SampleDBAdaptor.QueryParams.ID.key(), samples.subList(0, 2))) @@ -649,7 +650,7 @@ public void testGwasByPhenotype() throws Exception { GwasAnalysis analysis = new GwasAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_gwas_phenotype")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); analysis.setStudy(STUDY) .setPhenotype(PHENOTYPE_NAME); @@ -664,7 +665,7 @@ public void testKnockoutGenes() throws Exception { params.setSample(file.getSampleIds()); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); } @@ -679,7 +680,7 @@ public void testKnockoutGenesSpecificGenes() throws Exception { ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, STUDY) - .append("executionMethod", "byGene"), outDir, null, token); + .append("executionMethod", "byGene"), outDir, null, false, token); checkExecutionResult(er, false); assertEquals(4, er.getAttributes().get("otherGenesCount")); assertEquals(3, er.getAttributes().get("proteinCodingGenesCount")); @@ -695,7 +696,7 @@ public void testKnockoutGenesSpecificGenesAndBiotypeProteinCoding() throws Excep params.setBiotype(VariantAnnotationConstants.PROTEIN_CODING); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); assertEquals(0, er.getAttributes().get("otherGenesCount")); assertEquals(3, er.getAttributes().get("proteinCodingGenesCount")); @@ -711,7 +712,7 @@ public void testKnockoutGenesSpecificGenesAndBiotypeNMD() throws Exception { params.setBiotype("nonsense_mediated_decay"); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); assertEquals(3, er.getAttributes().get("otherGenesCount")); // MIR1909 only has miRNA biotype assertEquals(0, er.getAttributes().get("proteinCodingGenesCount")); @@ -748,7 +749,7 @@ public void testKnockoutGenesByBiotype() throws Exception { // + "," + "TR_V_gene"); ExecutionResult er = toolRunner.execute(KnockoutAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); checkExecutionResult(er, false); } @@ -760,7 +761,7 @@ public void testSampleMultiVariantFilterAnalysis() throws Exception { params.setQuery("(biotype=protein_coding AND ct=missense_variant AND gene=BRCA2) OR (gene=BTN3A2)"); ExecutionResult er = toolRunner.execute(SampleEligibilityAnalysis.class, - params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, token); + params.toObjectMap().append(ParamConstants.STUDY_PARAM, STUDY), outDir, null, false, token); // checkExecutionResult(er, false); } @@ -791,7 +792,7 @@ public void testMutationalSignatureFittingSNV() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - outDir, null, token); + outDir, null, false, token); java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.SIGNATURE_COEFFS_FILENAME).toFile(); byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); @@ -851,7 +852,7 @@ public void testMutationalSignatureCatalogueSV() throws Exception { params.setSkip("fitting"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - outDir, null, token); + outDir, null, false, token); java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT).toFile(); byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); @@ -911,7 +912,7 @@ public void testMutationalSignatureFittingSV() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - outDir, null, token); + outDir, null, false, token); java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.SIGNATURE_COEFFS_FILENAME).toFile(); byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); @@ -961,7 +962,7 @@ public void testHRDetect() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - snvFittingOutDir, null, token); + snvFittingOutDir, null, false, token); java.io.File snvSignatureFittingFile = snvFittingOutDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); assertTrue(snvSignatureFittingFile.exists()); @@ -993,7 +994,7 @@ public void testHRDetect() throws Exception { params.setSkip("catalogue"); toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), - svFittingOutDir, null, token); + svFittingOutDir, null, false, token); java.io.File svSignatureFittingFile = svFittingOutDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); assertTrue(svSignatureFittingFile.exists()); @@ -1010,7 +1011,7 @@ public void testHRDetect() throws Exception { hrdParams.setIndelQuery("{\"sample\": \"" + cancer_sample + "\", \"type\": \"" + VariantType.INDEL + "\"}"); hrdParams.setBootstrap(true); - toolRunner.execute(HRDetectAnalysis.class, hrdParams, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), hrdetectOutDir, null, token); + toolRunner.execute(HRDetectAnalysis.class, hrdParams, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), hrdetectOutDir, null, false, token); java.io.File hrDetectFile = hrdetectOutDir.resolve(HRDetectAnalysis.HRDETECT_SCORES_FILENAME_DEFAULT).toFile(); assertTrue("File missing : " + hrDetectFile, hrDetectFile.exists()); @@ -1091,7 +1092,7 @@ public void testClinicalAnalysisLoading() throws IOException, ToolException, Cat params.setFile(file.getId()); toolRunner.execute(ClinicalAnalysisLoadTask.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, - CANCER_STUDY), loadingOutDir, null, token); + CANCER_STUDY), loadingOutDir, null, false, token); String ca1Id = "SAP-45016-1"; String ca2Id = "OPA-6607-1"; diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index d92fe79abfc..8ad7f3c3479 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -47,16 +47,12 @@ import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.operations.variant.VariantAnnotationIndexParams; -import org.opencb.opencga.core.models.operations.variant.VariantSecondaryAnnotationIndexParams; -import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; +import org.opencb.opencga.core.models.operations.variant.*; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; -import org.opencb.opencga.core.models.variant.VariantIndexParams; -import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.core.tools.result.ExecutionResult; @@ -235,10 +231,10 @@ private void loadDataset() throws Throwable { .setFile(file.getId()) .setAnnotate(false) .setLoadHomRef(YesNoAuto.YES.name()), - Paths.get(opencga.createTmpOutdir("_index")), "index", token); + Paths.get(opencga.createTmpOutdir("_index")), "index", false, token); toolRunner.execute(VariantAnnotationIndexOperationTool.class, STUDY, new VariantAnnotationIndexParams(), - Paths.get(opencga.createTmpOutdir("_annotation-index")), "index", token); + Paths.get(opencga.createTmpOutdir("_annotation-index")), "index", false, token); for (int i = 0; i < file.getSampleIds().size(); i++) { if (i % 2 == 0) { @@ -320,7 +316,7 @@ public void testVariantFileReload() throws Exception { new VariantIndexParams() .setForceReload(false) .setFile(file.getId()), - Paths.get(opencga.createTmpOutdir()), "index_reload", token); + Paths.get(opencga.createTmpOutdir()), "index_reload", false, token); fail("Should have thrown an exception"); } catch (ToolException e) { assertEquals(StorageEngineException.class, e.getCause().getClass()); @@ -331,7 +327,7 @@ public void testVariantFileReload() throws Exception { new VariantIndexParams() .setForceReload(true) .setFile(file.getId()), - Paths.get(opencga.createTmpOutdir()), "index_reload", token); + Paths.get(opencga.createTmpOutdir()), "index_reload", false, token); } @@ -346,7 +342,7 @@ public void testVariantSecondaryAnnotationIndex() throws Exception { toolRunner.execute(VariantSecondaryAnnotationIndexOperationTool.class, STUDY, new VariantSecondaryAnnotationIndexParams(), - Paths.get(opencga.createTmpOutdir()), "annotation_index", token); + Paths.get(opencga.createTmpOutdir()), "annotation_index", false, token); for (String sample : samples) { SampleInternalVariantSecondaryAnnotationIndex index = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex(); @@ -369,12 +365,25 @@ public void testVariantSecondarySampleIndex() throws Exception { assertEquals(sample, 1, sampleIndex.getVersion().intValue()); } + try { + toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, STUDY, + new VariantSecondarySampleIndexParams() + .setFamilyIndex(true) + .setSample(Arrays.asList(mother)), + Paths.get(opencga.createTmpOutdir()), "index", false, token); + fail("Expected to fail"); + } catch (ToolException e) { + assertEquals("Exception from step 'familyIndex'", e.getMessage()); + assertEquals("No trios found for samples [" + mother + "]", e.getCause().getMessage()); + } + // Run family index. The family index status should be READY on offspring - toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, STUDY, + ExecutionResult result = toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, STUDY, new VariantSecondarySampleIndexParams() .setFamilyIndex(true) .setSample(Arrays.asList(ParamConstants.ALL)), - Paths.get(opencga.createTmpOutdir()), "index", token); + Paths.get(opencga.createTmpOutdir()), "index", false, token); + assertEquals(0, result.getEvents().size()); for (String sample : samples) { SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); @@ -392,7 +401,7 @@ public void testVariantSecondarySampleIndex() throws Exception { // Initially nothing should change, even after running a manual synchronization toolRunner.execute(VariantStorageMetadataSynchronizeOperationTool.class, new VariantStorageMetadataSynchronizeParams().setStudy(STUDY_FQN), - Paths.get(opencga.createTmpOutdir()), "", catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken()); + Paths.get(opencga.createTmpOutdir()), "", false, catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).getToken()); for (String sample : samples) { SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token) @@ -464,7 +473,7 @@ public void testVariantSecondarySampleIndexPartialFamily() throws Exception { new VariantSecondarySampleIndexParams() .setFamilyIndex(true) .setSample(Arrays.asList(daughter)), - Paths.get(opencga.createTmpOutdir()), "index", token); + Paths.get(opencga.createTmpOutdir()), "index", false, token); for (String sample : samples) { SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); @@ -487,7 +496,7 @@ public void testGwasIndex() throws Exception { GwasAnalysis analysis = new GwasAnalysis(); Path outDir = Paths.get(opencga.createTmpOutdir("_gwas_index")); System.out.println("output = " + outDir.toAbsolutePath()); - analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", token); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, executorParams, outDir, "", false, token); List samples = catalogManager.getSampleManager().get(STUDY, file.getSampleIds().subList(0, 2), QueryOptions.empty(), token).getResults(); catalogManager.getCohortManager().create(STUDY, new Cohort().setId("CASE").setSamples(samples), new QueryOptions(), token); @@ -523,13 +532,20 @@ public void testCellbaseConfigure() throws Exception { assertEquals("GRCh38", cellBaseUtils.getAssembly()); String newCellbase = "https://uk.ws.zettagenomics.com/cellbase/"; - String newCellbaseVersion = "v5.8"; + String newCellbaseVersion = "v5.2"; + String newCellbaseDataRelease = "1"; assertNotEquals(newCellbase, cellBaseUtils.getURL()); assertNotEquals(newCellbaseVersion, cellBaseUtils.getVersion()); + assertNotEquals(newCellbaseDataRelease, cellBaseUtils.getDataRelease()); - variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, "1", ""), false, null, token); + variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, newCellbaseDataRelease, ""), false, null, token); CellBaseConfiguration cellbaseConfiguration = catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getCellbase(); + + assertEquals(newCellbase, cellbaseConfiguration.getUrl()); + assertEquals(newCellbaseVersion, cellbaseConfiguration.getVersion()); + assertEquals(newCellbaseDataRelease, cellbaseConfiguration.getDataRelease()); + // assertTrue(family.getPedigreeGraph() != null); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java index d1dcd2e9353..6063e8f1070 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java @@ -87,8 +87,7 @@ */ public abstract class AbstractVariantOperationManagerTest extends GenericTest { - private String JOB_STATUS_FILE = "status.json"; - private String OUT_LOG_EXTENSION = ".out"; + private String OUT_LOG_EXTENSION = ".log"; private String ERR_LOG_EXTENSION = ".err"; protected CatalogManager catalogManager; @@ -403,8 +402,7 @@ protected List copyResults(Path tmpOutdirPath, String study, String catalo try { logger.info("Scanning files from {} to move to {}", tmpOutdirPath, outDir.getUri()); // Avoid copy the job.status file! - Predicate fileStatusFilter = uri -> !uri.getPath().endsWith(JOB_STATUS_FILE) - && !ExecutionResultManager.isExecutionResultFile(uri.getPath()) + Predicate fileStatusFilter = uri -> !ExecutionResultManager.isExecutionResultFile(uri.getPath()) && !uri.getPath().endsWith(OUT_LOG_EXTENSION) && !uri.getPath().endsWith(ERR_LOG_EXTENSION); files = fileScanner.scan(ORGANIZATION, outDir, tmpOutdirPath.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, fileStatusFilter, diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java index 4396a3d6e99..c343ceff241 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/PlatinumFileIndexerTest.java @@ -33,8 +33,8 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileIndexJobLauncherParams; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.storage.core.StorageEngineFactory; @@ -105,11 +105,10 @@ public void testIndexFamily() throws Exception { params.setFamily(true); ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() - .append(ParamConstants.STUDY_PARAM, studyId), outDir, null, sessionId); + .append(ParamConstants.STUDY_PARAM, studyId), outDir, null, false, sessionId); - assertEquals(2, er.getSteps().size()); + assertEquals(1, er.getSteps().size()); assertEquals("variant-index", er.getSteps().get(0).getId()); - assertEquals("family-index", er.getSteps().get(1).getId()); variantManager.iterator(new Query(VariantQueryParam.STUDY.key(), studyId), new QueryOptions(), sessionId).forEachRemaining(variant -> { System.out.println("variant = " + variant); @@ -213,7 +212,7 @@ public void testLauncher() throws CatalogException, IOException, ToolException { VariantFileIndexJobLauncherParams params = new VariantFileIndexJobLauncherParams().setDirectory("data/vcfs"); List tags = Arrays.asList("tag1", "tag2"); Job job = catalogManager.getJobManager().submit(studyFqn, VariantFileIndexJobLauncherTool.ID, Enums.Priority.HIGH, - params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, sessionId).first(); + params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, null, null, false, sessionId).first(); ExecutionResult result = toolRunner.execute(job, Paths.get(opencga.createTmpOutdir(studyId, "_LOAD_", sessionId)), sessionId); List tagsFromResult = result.getAttributes().getAsStringList(VariantFileIndexJobLauncherTool.JOB_TAGS_ATTRIBUTE); @@ -232,7 +231,7 @@ public void testLauncher() throws CatalogException, IOException, ToolException { //// Execute again, no new jobs should be submitted tags = Arrays.asList("tag10", "tag20"); job = catalogManager.getJobManager().submit(studyFqn, VariantFileIndexJobLauncherTool.ID, Enums.Priority.HIGH, - params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, sessionId).first(); + params.toParams(STUDY_PARAM, studyFqn), null, null, null, tags, null, null, false, sessionId).first(); result = toolRunner.execute(job, Paths.get(opencga.createTmpOutdir(studyId, "_LOAD_", sessionId)), sessionId); tagsFromResult = result.getAttributes().getAsStringList(VariantFileIndexJobLauncherTool.JOB_TAGS_ATTRIBUTE); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java index 09bcacc2ff7..cfe9deeac5f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/StatsVariantStorageTest.java @@ -212,7 +212,7 @@ public void calculateStats(VariantStatsIndexParams params, QueryOptions options) options.put(ParamConstants.STUDY_PARAM, studyId); ToolRunner toolRunner = new ToolRunner(null, catalogManager, opencga.getStorageEngineFactory()); - toolRunner.execute(VariantStatsIndexOperationTool.class, params, options, Paths.get(tmpOutdir), null, sessionId); + toolRunner.execute(VariantStatsIndexOperationTool.class, params, options, Paths.get(tmpOutdir), null, false, sessionId); } @Test diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java index 124e03ff1b9..ef373cf1cd7 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java @@ -26,7 +26,6 @@ import org.mockito.Mockito; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.biodata.models.variant.metadata.VariantSetStats; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -43,8 +42,8 @@ import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileInternalVariantIndex; import org.opencb.opencga.core.models.file.VariantIndexStatus; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.storage.core.StorageEngineFactory; @@ -186,7 +185,7 @@ public void testDeleteIndexedFile() throws Exception { Study study = catalogManager.getFileManager().getStudy(ORGANIZATION, inputFile, sessionId); thrown.expect(CatalogException.class); - thrown.expectMessage("The status is READY"); + thrown.expectMessage("Could not unlink file '" + inputFile.getId() + "'"); catalogManager.getFileManager().unlink(study.getFqn(), inputFile.getId(), sessionId); } @@ -200,7 +199,7 @@ public void testDeleteSampleFromIndexedFile() throws Exception { Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), inputFile.getSampleIds().get(100)); thrown.expect(CatalogException.class); thrown.expectMessage("Sample associated to the files"); - DataResult delete = catalogManager.getSampleManager().delete(studyFqn, query, null, sessionId); + catalogManager.getSampleManager().delete(studyFqn, query, null, sessionId); } @Test @@ -444,7 +443,7 @@ public void testIndexMalformed() throws Exception { ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, studyId) .append(VariantStorageOptions.TRANSFORM_FAIL_ON_MALFORMED_VARIANT.key(), false) - , outDir, null, sessionId); + , outDir, null, false, sessionId); assertEquals(Event.Type.WARNING, er.getEvents().get(0).getType()); assertThat(er.getEvents().get(0).getMessage(), CoreMatchers.containsString("Found malformed variants")); @@ -462,7 +461,7 @@ public void testIndexDuplicated() throws Exception { ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, studyId) .append(VariantStorageOptions.TRANSFORM_FAIL_ON_MALFORMED_VARIANT.key(), false) - , outDir, null, sessionId); + , outDir, null, false, sessionId); assertEquals(Event.Type.WARNING, er.getEvents().get(0).getType()); assertThat(er.getEvents().get(0).getMessage(), CoreMatchers.containsString("Found duplicated variants")); @@ -483,7 +482,7 @@ public void testIndexWeirdFileName() throws Exception { ExecutionResult er = toolRunner.execute(VariantIndexOperationTool.class, params.toObjectMap() .append(ParamConstants.STUDY_PARAM, studyId) - , outDir, null, sessionId); + , outDir, null, false, sessionId); } @Override diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java index 594e86af3dd..fcc6abed280 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantVirtualFileIndexTest.java @@ -26,7 +26,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileLinkParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,15 +72,15 @@ public void testIndexVirtual() throws Exception { toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams() .setFile("chr20.variant-test-file.vcf.gz") .setLoadSplitData("REGION") - , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, sessionId); + , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, false, sessionId); toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams() .setFile("chr21.variant-test-file.vcf.gz") .setLoadSplitData("REGION") - , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, sessionId); + , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, false, sessionId); toolRunner.execute(VariantIndexOperationTool.class, new VariantIndexParams() .setFile("chr22.variant-test-file.vcf.gz") .setLoadSplitData("REGION") - , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, sessionId); + , new ObjectMap(ParamConstants.STUDY_PARAM, studyId), Paths.get(opencga.createTmpOutdir()), null, false, sessionId); File file = catalogManager.getFileManager().get(studyId, "chr20.variant-test-file.vcf.gz", null, sessionId).first(); assertTrue(FileUtils.isPartial(file)); diff --git a/opencga-app/app/cloud/docker/opencga-init/override_yaml.py b/opencga-app/app/cloud/docker/opencga-init/override_yaml.py index 1993f85232b..04bd8139e82 100644 --- a/opencga-app/app/cloud/docker/opencga-init/override_yaml.py +++ b/opencga-app/app/cloud/docker/opencga-init/override_yaml.py @@ -19,9 +19,6 @@ parser.add_argument("--catalog-database-authentication-database", required=False, default="admin") parser.add_argument("--catalog-database-authentication-mechanism", required=False) parser.add_argument("--catalog-database-replica-set", required=False) -parser.add_argument("--catalog-search-hosts", required=True) -parser.add_argument("--catalog-search-user", required=False) -parser.add_argument("--catalog-search-password", required=False) parser.add_argument("--rest-host", required=True) parser.add_argument("--grpc-host", required=True) parser.add_argument("--analysis-execution-mode", required=False) @@ -138,13 +135,6 @@ def hostOverride(conf,hosts_var): if args.catalog_database_authentication_mechanism is not None: config["catalog"]["database"]["options"]["authenticationMechanism"] = args.catalog_database_authentication_mechanism -# Inject search database -hostOverride(config["catalog"]["searchEngine"], args.catalog_search_hosts) - -if args.catalog_search_user is not None: - config["catalog"]["searchEngine"]["user"] = args.catalog_search_user - config["catalog"]["searchEngine"]["password"] = args.catalog_search_password - # Inject execution settings config["analysis"]["scratchDir"] = "/tmp/opencga_scratch" if args.max_concurrent_jobs is not None: diff --git a/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py b/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py index 9d28765d399..ee6a547ea2b 100644 --- a/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py +++ b/opencga-app/app/cloud/docker/opencga-init/test/test_override_yaml.py @@ -1,11 +1,10 @@ +import os import subprocess -from shutil import copyfile +import sys import unittest import yaml from io import StringIO -import sys -import os - +from shutil import copyfile os.chdir(sys.path[0]) @@ -40,9 +39,6 @@ def test_end_2_end(self): "--catalog-database-hosts", "test-catalog-database-host1,test-catalog-database-host2,test-catalog-database-host3", "--catalog-database-user", "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", "test-catalog-search-user", - "--catalog-search-password", "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", "test-grpc-host", "--max-concurrent-jobs", "25", @@ -170,18 +166,6 @@ def test_end_2_end(self): self.assertEqual(config["catalog"]["database"]["options"]["sslEnabled"], True) self.assertEqual(config["catalog"]["database"]["options"]["sslInvalidCertificatesAllowed"], True) self.assertEqual(config["catalog"]["database"]["options"]["authenticationDatabase"], "admin") - self.assertEqual( - config["catalog"]["searchEngine"]["hosts"][0], "test-catalog-search-host1" - ) - self.assertEqual( - config["catalog"]["searchEngine"]["hosts"][1], "test-catalog-search-host2" - ) - self.assertEqual( - config["catalog"]["searchEngine"]["user"], "test-catalog-search-user" - ) - self.assertEqual( - config["catalog"]["searchEngine"]["password"], "test-catalog-search-password" - ) self.assertEqual(config["analysis"]["execution"]["id"], "test-analysis-execution-mode") self.assertEqual(config["analysis"]["execution"]["maxConcurrentJobs"]["variant-index"], 25) self.assertEqual(client_config["rest"]["hosts"][0]["url"], "test-rest-host") @@ -200,9 +184,6 @@ def test_azure_batch_execution(self): "--catalog-database-hosts", "test-catalog-database-host1,test-catalog-database-host2,test-catalog-database-host3", "--catalog-database-user", "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", "test-catalog-search-user", - "--catalog-search-password", "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", "test-grpc-host", "--analysis-execution-mode", "AZURE", @@ -273,9 +254,6 @@ def test_kubernetes_execution(self): "--catalog-database-hosts", "test-catalog-database-host1,test-catalog-database-host2,test-catalog-database-host3", "--catalog-database-user", "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", "test-catalog-search-user", - "--catalog-search-password", "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", "test-grpc-host", "--analysis-execution-mode", "k8s", @@ -357,12 +335,6 @@ def test_cellbasedb_with_empty_hosts(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -437,12 +409,6 @@ def test_cellbasedb_with_no_db_hosts(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -519,12 +485,6 @@ def test_cellbase_rest_set(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -604,12 +564,6 @@ def test_cellbase_rest_empty_set(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", @@ -688,12 +642,6 @@ def test_cellbase_rest_not_set(self): "test-catalog-database-user", "--catalog-database-password", "test-catalog-database-password", - "--catalog-search-hosts", - "test-catalog-search-host1,test-catalog-search-host2", - "--catalog-search-user", - "test-catalog-search-user", - "--catalog-search-password", - "test-catalog-search-password", "--rest-host", "test-rest-host", "--grpc-host", diff --git a/opencga-app/app/misc/scripts/hadoop-ssh.sh b/opencga-app/app/misc/scripts/hadoop-ssh.sh index 30141a04074..dc07d1b96f9 100755 --- a/opencga-app/app/misc/scripts/hadoop-ssh.sh +++ b/opencga-app/app/misc/scripts/hadoop-ssh.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash if [ -z "${HADOOP_SSH_USER}" ] ; then @@ -25,6 +25,8 @@ if [ -n "${HADOOP_SSH_KEY}" ] && [ -f "${HADOOP_SSH_KEY}" ] ; then SSH_OPTS="${SSH_OPTS} -i ${HADOOP_SSH_KEY}" fi +trap 'echo "SSH Process interrupted! Run time : ${SECONDS}s" && exit 1 1>&2 ' INT TERM + echo "Connect to Hadoop edge node ${HADOOP_SSH_USER}@${HADOOP_SSH_HOST}" 1>&2 echo "${SSHPASS_CMD}ssh ${SSH_OPTS} ${HADOOP_SSH_USER}@${HADOOP_SSH_HOST}" 1>&2 @@ -37,10 +39,12 @@ for arg in "$@" ; do arg=$(echo "$arg" | sed "s/'/'\\\\\\''/g") # aaa'aaa --> 'aaa'\''aaa' CMD="${CMD}'${arg}' " done -echo ${CMD} +echo ${CMD} 1>&2 ${SSHPASS_CMD} ssh ${SSH_OPTS} "${HADOOP_SSH_USER}@${HADOOP_SSH_HOST}" /bin/bash << EOF +echo "PID=\$\$" >&2 + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH} export HADOOP_USER_CLASSPATH_FIRST=${HADOOP_USER_CLASSPATH_FIRST} @@ -59,3 +63,10 @@ exec ${CMD} EOF +EXIT_CODE=$? + +echo "SSH Process completed!" 1>&2 +echo " - Run time : ${SECONDS}s" 1>&2 +echo " - Exit code: ${EXIT_CODE}" 1>&2 + +exit ${EXIT_CODE} \ No newline at end of file diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index 1c77750c7f7..cac11d63828 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java index 9dd3c922c14..f4106569f24 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/AdminCommandExecutor.java @@ -22,7 +22,6 @@ import org.opencb.opencga.app.cli.CommandExecutor; import org.opencb.opencga.app.cli.admin.AdminCliOptionsParser; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.config.Admin; import java.util.Collections; @@ -82,9 +81,6 @@ protected void setCatalogDatabaseCredentials(String host, String prefix, String configuration.getCatalog().getDatabase().setUser(user); } - if (configuration.getAdmin() == null) { - configuration.setAdmin(new Admin()); - } if (StringUtils.isNotEmpty(password)) { configuration.getCatalog().getDatabase().setPassword(password); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java index 5520de81126..090802f3741 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java @@ -23,13 +23,11 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDataStore; import org.opencb.opencga.app.cli.admin.AdminCliOptionsParser; -import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.db.mongodb.MongoDBUtils; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.master.monitor.MonitorService; import javax.ws.rs.client.Client; @@ -173,19 +171,12 @@ private void install() throws CatalogException { validateConfiguration(commandOptions); - this.configuration.getAdmin().setAlgorithm("HS256"); - - this.configuration.getAdmin().setSecretKey(commandOptions.secretKey); - if (StringUtils.isEmpty(configuration.getAdmin().getSecretKey())) { - configuration.getAdmin().setSecretKey(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)); - } - if (StringUtils.isEmpty(commandOptions.commonOptions.adminPassword)) { throw new CatalogException("No admin password found. Please, insert your password."); } try (CatalogManager catalogManager = new CatalogManager(configuration)) { - catalogManager.installCatalogDB("HS256", configuration.getAdmin().getSecretKey(), commandOptions.commonOptions.adminPassword, + catalogManager.installCatalogDB("HS256", commandOptions.secretKey, commandOptions.commonOptions.adminPassword, commandOptions.email, commandOptions.force); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java index eb6f7ae747c..3f5f7cee80f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java @@ -6,7 +6,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.app.cli.admin.options.MigrationCommandOptions; import org.opencb.opencga.app.cli.main.io.Table; -import org.opencb.opencga.app.migrations.v3_0_0.OrganizationMigration; +import org.opencb.opencga.app.migrations.v3.v3_0_0.OrganizationMigration; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.migration.Migration; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java index ff180f5d284..7ef509bbafc 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/migration/v2_0_0/VariantStorage200MigrationTool.java @@ -11,7 +11,8 @@ resource = Enums.Resource.VARIANT, type = Tool.Type.OPERATION, scope = Tool.Scope.PROJECT, - description = VariantStorage200MigrationTool.DESCRIPTION) + description = VariantStorage200MigrationTool.DESCRIPTION, + priority = Enums.Priority.HIGH) public class VariantStorage200MigrationTool extends OpenCgaTool { public static final String ID = "variant-storage-migration-2.0.0"; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java index 243a3e6918f..96522720e12 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java @@ -297,6 +297,9 @@ public static class JobOptions { @Parameter(names = {"--job"}, description = "Job id executing the command line", arity = 1) public String jobId; + @Parameter(names = {"--dry-run"}, description = "Dry run mode execution", arity = 0) + public boolean dryRun; + @Parameter(names = {"--job-id"}, hidden = true, description = "Deprecated, use --job", arity = 1) @Deprecated public void setJobId(String job) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java index 07ed1b36c1f..699b415da04 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java @@ -18,7 +18,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.analysis.alignment.AlignmentCoverageAnalysis; -import org.opencb.opencga.analysis.alignment.AlignmentStorageManager; +import org.opencb.opencga.analysis.alignment.AlignmentIndexOperation; import org.opencb.opencga.analysis.alignment.qc.AlignmentGeneCoverageStatsAnalysis; import org.opencb.opencga.analysis.alignment.qc.AlignmentQcAnalysis; import org.opencb.opencga.analysis.wrappers.bwa.BwaWrapperAnalysis; @@ -50,6 +50,7 @@ public class AlignmentCommandExecutor extends InternalCommandExecutor { private AlignmentCommandOptions alignmentCommandOptions; private String jobId; + private boolean dryRun; // private AlignmentStorageEngine alignmentStorageManager; public AlignmentCommandExecutor(AlignmentCommandOptions options) { @@ -65,6 +66,7 @@ public void execute() throws Exception { configure(); jobId = alignmentCommandOptions.internalJobOptions.jobId; + dryRun = alignmentCommandOptions.internalJobOptions.dryRun; switch (subCommandString) { case "index-run": @@ -119,9 +121,12 @@ public void execute() throws Exception { private void indexRun() throws Exception { AlignmentCommandOptions.IndexAlignmentCommandOptions cliOptions = alignmentCommandOptions.indexAlignmentCommandOptions; - AlignmentStorageManager alignmentManager = new AlignmentStorageManager(catalogManager, storageEngineFactory, alignmentCommandOptions.internalJobOptions.jobId); + ObjectMap params = new AlignmentIndexParams( + cliOptions.fileId, + cliOptions.overwrite + ).toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - alignmentManager.index(cliOptions.study, cliOptions.file, cliOptions.outdir, cliOptions.commonOptions.token); + toolRunner.execute(AlignmentIndexOperation.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } @@ -162,7 +167,7 @@ private void qcRun() throws ToolException { ).toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void geneCoverageStatsRun() throws ToolException { @@ -176,7 +181,7 @@ private void geneCoverageStatsRun() throws ToolException { ).toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentGeneCoverageStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // private void statsRun() throws ToolException { @@ -239,7 +244,7 @@ private void coverageRun() throws ToolException { cliOptions.windowSize ).toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(AlignmentCoverageAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(AlignmentCoverageAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void delete() { @@ -264,7 +269,7 @@ private void bwa() throws Exception { cliOptions.bwaParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(BwaWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(BwaWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Samtools @@ -279,7 +284,7 @@ private void samtools() throws Exception { cliOptions.samtoolsParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(SamtoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SamtoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Deeptools @@ -293,7 +298,7 @@ private void deeptools() throws Exception { cliOptions.deeptoolsParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(DeeptoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(DeeptoolsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // FastQC @@ -307,7 +312,7 @@ private void fastqc() throws Exception { cliOptions.fastqcParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(FastqcWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(FastqcWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Picard @@ -321,7 +326,7 @@ private void picard() throws Exception { cliOptions.picardParams) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(PicardWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(PicardWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } //------------------------------------------------------------------------- diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java index 0a32ddece0a..ac18f769392 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java @@ -17,7 +17,6 @@ package org.opencb.opencga.app.cli.internal.executors; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -39,33 +38,14 @@ import org.opencb.opencga.analysis.clinical.zetta.ZettaInterpretationConfiguration; import org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils; import org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.managers.ClinicalAnalysisManager; -import org.opencb.opencga.catalog.managers.SampleManager; -import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.clinical.*; -import org.opencb.opencga.core.models.family.FamilyCreateParams; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualUpdateParams; -import org.opencb.opencga.core.models.panel.Panel; -import org.opencb.opencga.core.models.panel.PanelCreateParams; -import org.opencb.opencga.core.models.sample.Sample; -import org.opencb.opencga.core.models.sample.SampleCreateParams; -import org.opencb.opencga.core.models.sample.SampleReferenceParam; -import org.opencb.opencga.core.response.RestResponse; - -import java.io.File; + import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.CancerTieringCommandOptions.CANCER_TIERING_INTERPRETATION_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.ExomiserInterpretationCommandOptions.EXOMISER_INTERPRETATION_RUN_COMMAND; @@ -74,8 +54,6 @@ import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.TeamCommandOptions.TEAM_INTERPRETATION_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.TieringCommandOptions.TIERING_INTERPRETATION_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.ClinicalCommandOptions.ZettaCommandOptions.ZETTA_INTERPRETATION_RUN_COMMAND; -import static org.opencb.opencga.catalog.utils.ParamUtils.SaveInterpretationAs.PRIMARY; -import static org.opencb.opencga.catalog.utils.ParamUtils.SaveInterpretationAs.SECONDARY; /** * Created on 01/04/20 @@ -137,7 +115,7 @@ private void rgaIndex() throws ToolException { ObjectMap params = new RgaAnalysisParams(options.file) .toObjectMap(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.study); - toolRunner.execute(RgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.commonOptions.token); + toolRunner.execute(RgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.jobOptions.dryRun, options.commonOptions.token); } private void auxRgaIndex() throws ToolException { @@ -147,7 +125,7 @@ private void auxRgaIndex() throws ToolException { ObjectMap params = new ObjectMap() .appendAll(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.study); - toolRunner.execute(AuxiliarRgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.commonOptions.token); + toolRunner.execute(AuxiliarRgaAnalysis.class, params, outDir, options.jobOptions.jobId, options.jobOptions.dryRun, options.commonOptions.token); } private void tiering() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java index 2529baaee1e..aa39f3d0919 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/DiseasePanelInternalCommandExecutor.java @@ -45,7 +45,8 @@ private void importPanels() throws ToolException { .toObjectMap(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.studyId); - toolRunner.execute(PanelImportTask.class, params, Paths.get(options.outDir), jobId, token); + toolRunner.execute(PanelImportTask.class, params, Paths.get(options.outDir), jobId, + diseasePanelCommandOptions.internalJobOptions.dryRun, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java index e79fbae1a4a..04bafdfaada 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/FileCommandExecutor.java @@ -90,7 +90,8 @@ private void postlink() throws ToolException { .toObjectMap(options.commonOptions.params) .append(ParamConstants.STUDY_PARAM, options.studyId); - toolRunner.execute(PostLinkSampleAssociation.class, params, outDir, fileCommandOptions.internalJobOptions.jobId, token); + toolRunner.execute(PostLinkSampleAssociation.class, params, outDir, fileCommandOptions.internalJobOptions.jobId, + fileCommandOptions.internalJobOptions.dryRun, token); } private void fetch() throws ToolException { @@ -98,7 +99,7 @@ private void fetch() throws ToolException { Path outDir = Paths.get(options.outDir); - toolRunner.execute(FetchAndRegisterTask.class, new FileFetch(options.url, options.path), outDir, null, options.commonOptions.token); + toolRunner.execute(FetchAndRegisterTask.class, new FileFetch(options.url, options.path), outDir, null, false, options.commonOptions.token); } private void tsvLoad() throws ToolException { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java index 0da33d83bfe..aa4eb7471d8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/StudyCommandExecutor.java @@ -45,6 +45,6 @@ private void templateRun() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.studyId); - toolRunner.execute(TemplateRunner.class, params, Paths.get(cliOptions.outdir), cliOptions.jobOptions.jobId, token); + toolRunner.execute(TemplateRunner.class, params, Paths.get(cliOptions.outdir), cliOptions.jobOptions.jobId, cliOptions.jobOptions.dryRun, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java index b075f5de782..ebcf46144db 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ToolsCommandExecutor.java @@ -53,7 +53,7 @@ public void execute() throws Exception { private void executeTool() throws ToolException { ToolsCommandOptions.ExecuteToolCommandOptions cliOptions = this.toolCommandOptions.executeToolCommandOptions; toolRunner.execute(cliOptions.toolId, new ObjectMap(cliOptions.params), Paths.get(cliOptions.outDir), - toolCommandOptions.internalJobOptions.jobId, token); + toolCommandOptions.internalJobOptions.jobId, toolCommandOptions.internalJobOptions.dryRun, token); } private void executeJob() throws CatalogException, ToolException { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index e502850d9ee..11d18532218 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -62,6 +62,7 @@ import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.exceptions.AnalysisExecutionException; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.core.models.operations.variant.*; @@ -128,6 +129,7 @@ public class VariantInternalCommandExecutor extends InternalCommandExecutor { // private AnalysisCliOptionsParser.VariantCommandOptions variantCommandOptions; private VariantCommandOptions variantCommandOptions; private String jobId; + private boolean dryRun; public VariantInternalCommandExecutor(VariantCommandOptions variantCommandOptions) { super(variantCommandOptions.commonCommandOptions); @@ -143,6 +145,7 @@ public void execute() throws Exception { configure(); jobId = variantCommandOptions.internalJobOptions.jobId; + dryRun = variantCommandOptions.internalJobOptions.dryRun; switch (subCommandString) { case VARIANT_DELETE_COMMAND: @@ -343,7 +346,7 @@ private void query(VariantCommandOptions.AbstractVariantQueryCommandOptions cliO cliOptions.outputFileName, cliOptions.outputFileFormat, cliOptions.variantsFile); - toolRunner.execute(VariantExportTool.class, toolParams, queryOptions, Paths.get(outdir), jobId, token); + toolRunner.execute(VariantExportTool.class, toolParams, queryOptions, Paths.get(outdir), jobId, dryRun, token); } } @@ -362,7 +365,7 @@ private void fileDelete() throws ToolException { cliOptions.genericVariantDeleteOptions.resume); toolRunner.execute(VariantFileDeleteOperationTool.class, params.toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study), - Paths.get(cliOptions.outdir), jobId, token); + Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void index() throws ToolException { @@ -404,7 +407,7 @@ private void index() throws ToolException { .append(VariantStorageOptions.STDIN.key(), cliOptions.stdin) .append(VariantStorageOptions.STDOUT.key(), cliOptions.stdout); - toolRunner.execute(VariantIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void secondaryIndex() throws ToolException { @@ -419,9 +422,9 @@ private void secondaryIndex() throws ToolException { .append(ParamConstants.PROJECT_PARAM, cliOptions.project); if (CollectionUtils.isEmpty(cliOptions.sample)) { - toolRunner.execute(VariantSecondaryAnnotationIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantSecondaryAnnotationIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } else { - toolRunner.execute(VariantSecondaryIndexSamplesOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantSecondaryIndexSamplesOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } } @@ -454,7 +457,7 @@ private void statsRun() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantStatsAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // private void statsIndex() throws ToolException { @@ -485,7 +488,7 @@ private void scoreLoad() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantScoreIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantScoreIndexOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void scoreRemove() throws ToolException { @@ -498,7 +501,7 @@ private void scoreRemove() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantScoreDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantScoreDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleIndex() @@ -516,7 +519,7 @@ private void sampleIndex() toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, params.toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study), Paths.get(cliOptions.outdir), - jobId, token); + jobId, dryRun, token); } private void familyIndex() @@ -532,7 +535,7 @@ private void familyIndex() toolRunner.execute(VariantFamilyIndexOperationTool.class, params.toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study), Paths.get(cliOptions.outdir), - jobId, token); + jobId, dryRun, token); } private void annotate() throws ToolException { @@ -557,7 +560,7 @@ private void annotate() throws ToolException { .append(ParamConstants.PROJECT_PARAM, cliOptions.project) .append(ParamConstants.STUDY_PARAM, cliOptions.study), Paths.get(cliOptions.outdir), - jobId, token); + jobId, dryRun, token); } private void annotationSave() throws ToolException { @@ -567,7 +570,7 @@ private void annotationSave() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.PROJECT_PARAM, cliOptions.project); - toolRunner.execute(VariantAnnotationSaveOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAnnotationSaveOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void annotationDelete() throws ToolException { @@ -577,7 +580,7 @@ private void annotationDelete() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.PROJECT_PARAM, cliOptions.project); - toolRunner.execute(VariantAnnotationDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAnnotationDeleteOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void annotationQuery() throws CatalogException, IOException, StorageEngineException { @@ -645,7 +648,7 @@ private void aggregateFamily() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantAggregateFamilyOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAggregateFamilyOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void aggregate() throws ToolException { @@ -657,7 +660,7 @@ private void aggregate() throws ToolException { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(VariantAggregateOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(VariantAggregateOperationTool.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleRun() throws Exception { @@ -667,7 +670,7 @@ private void sampleRun() throws Exception { toolRunner.execute(SampleVariantFilterAnalysis.class, cliOptions.toolParams.toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study), - Paths.get(cliOptions.outdir), jobId, token); + Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void gwas() throws Exception { @@ -689,7 +692,7 @@ private void gwas() throws Exception { } GwasAnalysis gwasAnalysis = new GwasAnalysis(); gwasAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); gwasAnalysis.setStudy(cliOptions.study) .setPhenotype(cliOptions.phenotype) .setIndex(cliOptions.index) @@ -723,7 +726,7 @@ private void knockout() throws Exception { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(KnockoutAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(KnockoutAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleEligibility() throws Exception { @@ -736,7 +739,7 @@ private void sampleEligibility() throws Exception { .toObjectMap(cliOptions.commonOptions.params) .append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(SampleEligibilityAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SampleEligibilityAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void sampleStats() throws Exception { @@ -755,7 +758,7 @@ private void sampleStats() throws Exception { cliOptions.batchSize, variantQuery ); - toolRunner.execute(SampleVariantStatsAnalysis.class, toolParams, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SampleVariantStatsAnalysis.class, toolParams, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void cohortStats() throws Exception { @@ -774,7 +777,7 @@ private void cohortStats() throws Exception { CohortVariantStatsAnalysis cohortVariantStatsAnalysis = new CohortVariantStatsAnalysis(); cohortVariantStatsAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); cohortVariantStatsAnalysis.setStudy(cliOptions.study) .setCohortName(cliOptions.cohort) .setIndex(cliOptions.index) @@ -795,7 +798,7 @@ private void julie() throws Exception { Path outdir = Paths.get(cliOptions.outdir); - toolRunner.execute(JulieTool.class, toolParams, params, outdir, jobId, token); + toolRunner.execute(JulieTool.class, toolParams, params, outdir, jobId, dryRun, token); } private void mutationalSignature() throws Exception { @@ -823,7 +826,7 @@ private void mutationalSignature() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(MutationalSignatureAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(MutationalSignatureAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void hrDetect() throws Exception { @@ -845,7 +848,7 @@ private void hrDetect() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(HRDetectAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(HRDetectAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void genomePlot() throws Exception { @@ -859,7 +862,7 @@ private void genomePlot() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(GenomePlotAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(GenomePlotAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void mendelianError() throws Exception { @@ -869,7 +872,7 @@ private void mendelianError() throws Exception { MendelianErrorAnalysis mendelianErrorAnalysis = new MendelianErrorAnalysis(); mendelianErrorAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); mendelianErrorAnalysis.setStudy(cliOptions.study) .setFamilyId(cliOptions.family) .setIndividualId(cliOptions.individual) @@ -884,7 +887,7 @@ private void inferredSex() throws Exception { InferredSexAnalysis inferredSexAnalysis = new InferredSexAnalysis(); inferredSexAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); inferredSexAnalysis.setStudyId(cliOptions.study) .setIndividualId(cliOptions.individual) .setSampleId(cliOptions.sample) @@ -898,7 +901,7 @@ private void relatedness() throws Exception { RelatednessAnalysis relatednessAnalysis = new RelatednessAnalysis(); relatednessAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); relatednessAnalysis.setStudyId(cliOptions.study) .setIndividualIds(cliOptions.individuals) .setSampleIds(cliOptions.samples) @@ -914,7 +917,7 @@ private void familyQc() throws Exception { FamilyQcAnalysis familyQcAnalysis = new FamilyQcAnalysis(); familyQcAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); familyQcAnalysis.setStudyId(cliOptions.study) .setFamilyId(cliOptions.family) .setRelatednessMethod(cliOptions.relatednessMethod) @@ -929,7 +932,7 @@ private void individualQc() throws Exception { IndividualQcAnalysis individualQcAnalysis = new IndividualQcAnalysis(); individualQcAnalysis.setUp(appHome, catalogManager, storageEngineFactory, params, Paths.get(cliOptions.outdir), - variantCommandOptions.internalJobOptions.jobId, token); + variantCommandOptions.internalJobOptions.jobId, dryRun, token); individualQcAnalysis.setStudyId(cliOptions.study) .setIndividualId(cliOptions.individual) .setSampleId(cliOptions.sample) @@ -972,7 +975,7 @@ private void sampleQc() throws Exception { cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(SampleQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(SampleQcAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } // Wrappers @@ -985,7 +988,7 @@ private void plink() throws Exception { cliOptions.plinkParams) .toObjectMap(cliOptions.basicOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(PlinkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(PlinkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void rvtests() throws Exception { @@ -997,7 +1000,7 @@ private void rvtests() throws Exception { cliOptions.rvtestsParams) .toObjectMap(cliOptions.basicOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(RvtestsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(RvtestsWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void gatk() throws Exception { @@ -1009,7 +1012,7 @@ private void gatk() throws Exception { cliOptions.gatkParams) .toObjectMap(cliOptions.basicOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(GatkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(GatkWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void exomiser() throws Exception { @@ -1017,10 +1020,11 @@ private void exomiser() throws Exception { ObjectMap params = new ExomiserWrapperParams( cliOptions.sample, + cliOptions.clinicalAnalysisType, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); - toolRunner.execute(ExomiserWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + toolRunner.execute(ExomiserWrapperAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, dryRun, token); } private void checkSignatureVersion(String sigVersion) throws ClientException { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java index 99f09597f4b..c9d92e51d4f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/AlignmentCommandOptions.java @@ -104,8 +104,11 @@ public class IndexAlignmentCommandOptions extends GeneralCliOptions.StudyOption @ParametersDelegate public Object internalJobOptions = internalJobOptionsObject; - @Parameter(names = {"--file"}, description = FILE_ID_DESCRIPTION, required = true, arity = 1) - public String file; + @Parameter(names = {"--file-id"}, description = FILE_ID_DESCRIPTION, required = true, arity = 1) + public String fileId; + + @Parameter(names = {"--overwrite"}, description = "Force to overwrite the alignment index file", arity = 0) + public boolean overwrite; @Parameter(names = {"-o", "--outdir"}, description = OUTPUT_DIRECTORY_DESCRIPTION) public String outdir; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index bad1beff087..aa25fe630e1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -48,6 +48,7 @@ import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser; import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.variant.AnnotationVariantQueryParams; import org.opencb.opencga.core.models.variant.SampleVariantFilterParams; import org.opencb.opencga.core.tools.variant.IndividualQcAnalysisExecutor; @@ -1841,10 +1842,13 @@ public class ExomiserAnalysisCommandOptions { @Parameter(names = {"--study"}, description = "Study where all the samples belong to.") public String study; - @Parameter(names = {"--sample"}, description = "Sample ID.", required = true) + @Parameter(names = {"--sample"}, description = FieldConstants.SAMPLE_ID_DESCRIPTION, required = true) public String sample; - @Parameter(names = {"-o", "--outdir"}, description = "Output directory.") + @Parameter(names = {"--clinical-analysis-type"}, description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION) + public String clinicalAnalysisType = ClinicalAnalysis.Type.SINGLE.name(); + + @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) public String outdir; } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index d6bd902fff1..4d1ed716a48 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -49,7 +49,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List jobsList = asList( "acl-update","create","distinct","retry","search","top","acl","delete","info","update","log-head","log-tail") + private List jobsList = asList( "acl-update","create","distinct","retry","search","top","acl","delete","info","update","kill","log-head","log-tail") .stream() .map(Candidate::new) .collect(toList()); @@ -89,7 +89,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List organizationsList = asList( "create","notes-create","notes-search","notes-delete","notes-update","info","update") + private List organizationsList = asList( "create","notes-create","notes-search","notes-delete","notes-update","update-status-user","user-update","configuration-update","info","update") .stream() .map(Candidate::new) .collect(toList()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index 33d53de3bf8..d99dbba3488 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -160,6 +160,7 @@ public OpencgaCliOptionsParser() { jobsSubCommands.addCommand("delete", jobsCommandOptions.deleteCommandOptions); jobsSubCommands.addCommand("info", jobsCommandOptions.infoCommandOptions); jobsSubCommands.addCommand("update", jobsCommandOptions.updateCommandOptions); + jobsSubCommands.addCommand("kill", jobsCommandOptions.killCommandOptions); jobsSubCommands.addCommand("log-head", jobsCommandOptions.headLogCommandOptions); jobsSubCommands.addCommand("log-tail", jobsCommandOptions.tailLogCommandOptions); jobsSubCommands.addCommand("log", jobsCommandOptions.logCommandOptions); @@ -272,6 +273,9 @@ public OpencgaCliOptionsParser() { organizationsSubCommands.addCommand("notes-search", organizationsCommandOptions.searchNotesCommandOptions); organizationsSubCommands.addCommand("notes-delete", organizationsCommandOptions.deleteNotesCommandOptions); organizationsSubCommands.addCommand("notes-update", organizationsCommandOptions.updateNotesCommandOptions); + organizationsSubCommands.addCommand("update-status-user", organizationsCommandOptions.userUpdateStatusCommandOptions); + organizationsSubCommands.addCommand("user-update", organizationsCommandOptions.updateUserCommandOptions); + organizationsSubCommands.addCommand("configuration-update", organizationsCommandOptions.updateConfigurationCommandOptions); organizationsSubCommands.addCommand("info", organizationsCommandOptions.infoCommandOptions); organizationsSubCommands.addCommand("update", organizationsCommandOptions.updateCommandOptions); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandOptions.java index 0a317ace8a8..74ff7b9ce30 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandOptions.java @@ -54,16 +54,25 @@ public class RunTemplatesCommandOptions { @Parameter(names = {"--resume"}, description = "Resume study metadata ingestion.", arity = 0) public boolean resume; - @Parameter(names = {"--jobId"}, description = "Job id.", arity = 1) + @Parameter(names = {"--job-id", "--jobId"}, description = "Job id.", arity = 1) public String jobId; - @Parameter(names = {"--jobDependsOn"}, description = "Job depends on.", arity = 1) + @Parameter(names = {"--job-depends-on", "--jobDependsOn"}, description = "Job depends on.", arity = 1) public String jobDependsOn; - @Parameter(names = {"--jobDescription"}, description = "Job description.", arity = 1) + @Parameter(names = {"--job-description", "--jobDescription"}, description = "Job description.", arity = 1) public String jobDescription; - @Parameter(names = {"--jobTags"}, description = "Job tags.", arity = 1) + @Parameter(names = {"--job-tags", "--jobTags"}, description = "Job tags.", arity = 1) public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Job scheduled start time.", arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Job priority.", arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Job dry-run mode.", arity = 1) + public Boolean jobDryRun; } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index 217b7421f32..ddc77f0009b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -121,6 +121,9 @@ private RestResponse runBwa() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -162,6 +165,9 @@ private RestResponse runCoverageIndex() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -200,6 +206,9 @@ private RestResponse coverageQcGeneCoverageStatsRun() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -295,6 +304,9 @@ private RestResponse runDeeptools() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -333,6 +345,9 @@ private RestResponse runFastqc() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -371,6 +386,9 @@ private RestResponse runIndex() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -387,7 +405,7 @@ private RestResponse runIndex() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), AlignmentIndexParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "file",commandOptions.file, true); + putNestedIfNotEmpty(beanParams, "fileId",commandOptions.fileId, true); putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); alignmentIndexParams = JacksonUtils.getDefaultObjectMapper().copy() @@ -408,6 +426,9 @@ private RestResponse runPicard() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -446,6 +467,9 @@ private RestResponse runQc() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -517,6 +541,9 @@ private RestResponse runSamtools() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index e96ddce01c2..84eeffe1be6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -518,6 +518,9 @@ private RestResponse runInterpreterCancerTiering() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -556,6 +559,9 @@ private RestResponse runInterpreterExomiser() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -592,6 +598,9 @@ private RestResponse runInterpreterTeam() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -631,6 +640,9 @@ private RestResponse runInterpreterTiering() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -670,6 +682,9 @@ private RestResponse runInterpreterZetta() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -756,6 +771,9 @@ private RestResponse load() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -907,6 +925,9 @@ private RestResponse runRgaIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotNull("auxiliarIndex", commandOptions.auxiliarIndex); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 33dfbc54a41..3427a69163c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -28,6 +28,7 @@ import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsExportParams; import org.opencb.opencga.core.models.variant.AnnotationVariantQueryParams; import org.opencb.opencga.core.models.variant.CircosAnalysisParams; @@ -50,7 +51,6 @@ import org.opencb.opencga.core.models.variant.SampleVariantFilterParams; import org.opencb.opencga.core.models.variant.SampleVariantStatsAnalysisParams; import org.opencb.opencga.core.models.variant.VariantExportParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.models.variant.VariantStatsAnalysisParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -370,6 +370,9 @@ private RestResponse runCohortStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -410,6 +413,9 @@ private RestResponse runExomiser() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -427,6 +433,7 @@ private RestResponse runExomiser() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); + putNestedIfNotEmpty(beanParams, "clinicalAnalysisType",commandOptions.clinicalAnalysisType, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); exomiserWrapperParams = JacksonUtils.getDefaultObjectMapper().copy() @@ -450,6 +457,9 @@ private RestResponse runExport() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -590,6 +600,9 @@ private RestResponse runFamilyQc() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -628,6 +641,9 @@ private RestResponse deleteFile() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("file", commandOptions.file); queryParams.putIfNotNull("resume", commandOptions.resume); @@ -649,6 +665,9 @@ private RestResponse runGatk() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -687,6 +706,9 @@ private RestResponse runGenomePlot() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -727,6 +749,9 @@ private RestResponse runGwas() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -774,6 +799,9 @@ private RestResponse runHrDetect() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -822,6 +850,9 @@ private RestResponse runIndex() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -887,6 +918,9 @@ private RestResponse runIndividualQc() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -926,6 +960,9 @@ private RestResponse runInferredSex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -998,6 +1035,9 @@ private RestResponse runKnockout() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1043,6 +1083,9 @@ private RestResponse runMendelianError() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1135,6 +1178,9 @@ private RestResponse runMutationalSignature() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1186,6 +1232,9 @@ private RestResponse runPlink() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1313,6 +1362,9 @@ private RestResponse runRelatedness() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1353,6 +1405,9 @@ private RestResponse runRvtests() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1426,6 +1481,9 @@ private RestResponse runSampleEligibility() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1464,6 +1522,9 @@ private RestResponse runSampleQc() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1564,6 +1625,9 @@ private RestResponse runSample() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1657,6 +1721,9 @@ private RestResponse runSampleStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1729,6 +1796,9 @@ private RestResponse runStatsExport() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1769,6 +1839,9 @@ private RestResponse runStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java index 033aff423b5..00dbbbc130f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java @@ -215,6 +215,9 @@ private RestResponse importPanels() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index b6644a30d88..f9f6f5fedb8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -351,6 +351,9 @@ private RestResponse fetch() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -438,6 +441,9 @@ private RestResponse runLink() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -479,6 +485,9 @@ private RestResponse runPostlink() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index 8f6e674164a..cf28e7983bf 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -97,6 +97,9 @@ public void execute() throws Exception { case "update": queryResponse = update(); break; + case "kill": + queryResponse = kill(); + break; case "log-head": queryResponse = headLog(); break; @@ -236,6 +239,7 @@ private RestResponse retry() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -400,6 +404,20 @@ private RestResponse update() throws Exception { return openCGAClient.getJobClient().update(commandOptions.jobs, jobUpdateParams, queryParams); } + private RestResponse kill() throws Exception { + logger.debug("Executing kill in Jobs command line"); + + JobsCommandOptions.KillCommandOptions commandOptions = jobsCommandOptions.killCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getJobClient().kill(commandOptions.job, queryParams); + } + private RestResponse headLog() throws Exception { logger.debug("Executing headLog in Jobs command line"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 068b4a3792a..6135886595d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -22,21 +22,21 @@ import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; import org.opencb.opencga.core.models.operations.variant.VariantAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantAnnotationSaveParams; +import org.opencb.opencga.core.models.operations.variant.VariantConfigureParams; import org.opencb.opencga.core.models.operations.variant.VariantFamilyIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileIndexJobLauncherParams; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantPruneParams; +import org.opencb.opencga.core.models.operations.variant.VariantSampleDeleteParams; import org.opencb.opencga.core.models.operations.variant.VariantScoreIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondaryAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsDeleteParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; -import org.opencb.opencga.core.models.variant.VariantConfigureParams; -import org.opencb.opencga.core.models.variant.VariantFileDeleteParams; -import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; -import org.opencb.opencga.core.models.variant.VariantPruneParams; -import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; -import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; -import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; +import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -211,6 +211,9 @@ private RestResponse aggregateVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -248,6 +251,9 @@ private RestResponse deleteVariantAnnotation() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("annotationId", commandOptions.annotationId); @@ -264,6 +270,9 @@ private RestResponse indexVariantAnnotation() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -309,6 +318,9 @@ private RestResponse saveVariantAnnotation() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); @@ -375,6 +387,9 @@ private RestResponse deleteVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -394,6 +409,7 @@ private RestResponse deleteVariant() throws Exception { ObjectMap beanParams = new ObjectMap(); putNestedIfNotNull(beanParams, "file",commandOptions.file, true); putNestedIfNotNull(beanParams, "resume",commandOptions.resume, true); + putNestedIfNotNull(beanParams, "force",commandOptions.force, true); variantFileDeleteParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -412,6 +428,9 @@ private RestResponse aggregateVariantFamily() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -450,6 +469,9 @@ private RestResponse indexVariantFamily() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -489,6 +511,9 @@ private RestResponse indexVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -554,6 +579,9 @@ private RestResponse launcherVariantIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -624,6 +652,9 @@ private RestResponse runVariantJulie() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); @@ -659,6 +690,9 @@ private RestResponse repairVariantMetadata() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); VariantStorageMetadataRepairToolParams variantStorageMetadataRepairToolParams = null; @@ -692,6 +726,9 @@ private RestResponse synchronizeVariantMetadata() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -729,6 +766,9 @@ private RestResponse pruneVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); VariantPruneParams variantPruneParams = null; @@ -763,6 +803,9 @@ private RestResponse deleteVariantSample() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -801,6 +844,9 @@ private RestResponse indexVariantSample() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -867,6 +913,9 @@ private RestResponse deleteVariantScore() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotNull("resume", commandOptions.resume); @@ -888,6 +937,9 @@ private RestResponse indexVariantScore() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -929,6 +981,9 @@ private RestResponse variantSecondaryAnnotationIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -968,6 +1023,9 @@ private RestResponse variantSecondarySampleIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); @@ -1034,6 +1092,9 @@ private RestResponse secondaryIndexVariant() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("project", commandOptions.project); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -1073,6 +1134,9 @@ private RestResponse deleteVariantSecondaryIndex() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("samples", commandOptions.samples); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { @@ -1093,6 +1157,9 @@ private RestResponse deleteVariantStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1130,6 +1197,9 @@ private RestResponse indexVariantStats() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1170,6 +1240,9 @@ private RestResponse deleteVariantStudy() throws Exception { queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("study", commandOptions.study); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java index 991f44fd9d8..c0ce6156d34 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java @@ -12,6 +12,7 @@ import org.opencb.opencga.app.cli.main.options.OrganizationsCommandOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.catalog.utils.ParamUtils.UpdateAction; import org.opencb.opencga.client.exceptions.ClientException; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.config.Optimizations; @@ -23,6 +24,10 @@ import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.organizations.TokenConfiguration; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserQuota; +import org.opencb.opencga.core.models.user.UserStatusUpdateParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -75,6 +80,15 @@ public void execute() throws Exception { case "notes-update": queryResponse = updateNotes(); break; + case "update-status-user": + queryResponse = userUpdateStatus(); + break; + case "user-update": + queryResponse = updateUser(); + break; + case "configuration-update": + queryResponse = updateConfiguration(); + break; case "info": queryResponse = info(); break; @@ -116,6 +130,7 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); putNestedIfNotEmpty(beanParams, "creationDate",commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate",commandOptions.modificationDate, true); + putNestedIfNotEmpty(beanParams, "configuration.defaultUserExpirationDate",commandOptions.configurationDefaultUserExpirationDate, true); putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); organizationCreateParams = JacksonUtils.getDefaultObjectMapper().copy() @@ -223,6 +238,113 @@ private RestResponse updateNotes() throws Exception { return openCGAClient.getOrganizationClient().updateNotes(commandOptions.id, noteUpdateParams, queryParams); } + private RestResponse userUpdateStatus() throws Exception { + logger.debug("Executing userUpdateStatus in Organizations command line"); + + OrganizationsCommandOptions.UserUpdateStatusCommandOptions commandOptions = organizationsCommandOptions.userUpdateStatusCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + UserStatusUpdateParams userStatusUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/user/{user}/status/update")); + return res; + } else if (commandOptions.jsonFile != null) { + userStatusUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), UserStatusUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "status",commandOptions.status, true); + + userStatusUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), UserStatusUpdateParams.class); + } + return openCGAClient.getOrganizationClient().userUpdateStatus(commandOptions.user, userStatusUpdateParams, queryParams); + } + + private RestResponse updateUser() throws Exception { + logger.debug("Executing updateUser in Organizations command line"); + + OrganizationsCommandOptions.UpdateUserCommandOptions commandOptions = organizationsCommandOptions.updateUserCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("organization", commandOptions.organization); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + + + OrganizationUserUpdateParams organizationUserUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/user/{user}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), OrganizationUserUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); + putNestedIfNotNull(beanParams, "quota.diskUsage",commandOptions.quotaDiskUsage, true); + putNestedIfNotNull(beanParams, "quota.cpuUsage",commandOptions.quotaCpuUsage, true); + putNestedIfNotNull(beanParams, "quota.maxDisk",commandOptions.quotaMaxDisk, true); + putNestedIfNotNull(beanParams, "quota.maxCpu",commandOptions.quotaMaxCpu, true); + putNestedIfNotEmpty(beanParams, "account.expirationDate",commandOptions.accountExpirationDate, true); + putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); + + organizationUserUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), OrganizationUserUpdateParams.class); + } + return openCGAClient.getOrganizationClient().updateUser(commandOptions.user, organizationUserUpdateParams, queryParams); + } + + private RestResponse updateConfiguration() throws Exception { + logger.debug("Executing updateConfiguration in Organizations command line"); + + OrganizationsCommandOptions.UpdateConfigurationCommandOptions commandOptions = organizationsCommandOptions.updateConfigurationCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + queryParams.putIfNotNull("authenticationOriginsAction", commandOptions.authenticationOriginsAction); + + + OrganizationConfiguration organizationConfiguration = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/organizations/{organization}/configuration/update")); + return res; + } else if (commandOptions.jsonFile != null) { + organizationConfiguration = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), OrganizationConfiguration.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "defaultUserExpirationDate",commandOptions.defaultUserExpirationDate, true); + putNestedIfNotNull(beanParams, "optimizations.simplifyPermissions",commandOptions.optimizationsSimplifyPermissions, true); + putNestedIfNotEmpty(beanParams, "token.algorithm",commandOptions.tokenAlgorithm, true); + putNestedIfNotEmpty(beanParams, "token.secretKey",commandOptions.tokenSecretKey, true); + putNestedIfNotNull(beanParams, "token.expiration",commandOptions.tokenExpiration, true); + + organizationConfiguration = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), OrganizationConfiguration.class); + } + return openCGAClient.getOrganizationClient().updateConfiguration(commandOptions.organization, organizationConfiguration, queryParams); + } + private RestResponse info() throws Exception { logger.debug("Executing info in Organizations command line"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 645c65e2974..2e8bd7b0040 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -521,6 +521,9 @@ private RestResponse runTemplates() throws Exception { queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotNull("overwrite", commandOptions.overwrite); queryParams.putIfNotNull("resume", commandOptions.resume); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index 6eeeae17408..4883cedef5c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -305,7 +305,6 @@ private RestResponse update() throws Exception { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true); putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true); - putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); userUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/VcfOutputWriter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/VcfOutputWriter.java index 6093c975a32..0c398dd2ef6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/VcfOutputWriter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/VcfOutputWriter.java @@ -9,7 +9,7 @@ import org.opencb.biodata.models.variant.metadata.VariantStudyMetadata; import org.opencb.biodata.models.variant.protobuf.VariantProto; import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.io.VcfDataWriter; import java.io.PrintStream; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java index 46abae24683..61cc3634fb1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java @@ -95,6 +95,15 @@ public class RunBwaCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -142,14 +151,23 @@ public class RunCoverageIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--bam-file-id"}, description = "The body web service bamFileId parameter", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--bam-file-id"}, description = "BAM file ID.", required = false, arity = 1) public String bamFileId; - @Parameter(names = {"--bai-file-id"}, description = "The body web service baiFileId parameter", required = false, arity = 1) + @Parameter(names = {"--bai-file-id"}, description = "BAI file ID.", required = false, arity = 1) public String baiFileId; - @Parameter(names = {"--window-size"}, description = "The body web service windowSize parameter", required = false, arity = 1) - public Integer windowSize; + @Parameter(names = {"--window-size"}, description = "Window size (i.e., the size of the bins, in bases, for the output of the BIGWIG file).", required = false, arity = 1) + public Integer windowSize = 50; } @@ -180,6 +198,15 @@ public class CoverageQcGeneCoverageStatsRunCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--bam-file"}, description = "The body web service bamFile parameter", required = false, arity = 1) public String bamFile; @@ -311,6 +338,15 @@ public class RunDeeptoolsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -349,6 +385,15 @@ public class RunFastqcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--input-file"}, description = "The body web service inputFile parameter", required = false, arity = 1) public String inputFile; @@ -387,10 +432,19 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) - public String file; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--file-id"}, description = "File ID, (i.e., BAM/CRAM file ID).", required = false, arity = 1) + public String fileId; - @Parameter(names = {"--overwrite"}, description = "The body web service overwrite parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite"}, description = "Flag to force indexing.", required = false, help = true, arity = 0) public boolean overwrite = false; } @@ -422,6 +476,15 @@ public class RunPicardCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -460,6 +523,15 @@ public class RunQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--bam-file"}, description = "ID for the BAM file to process.", required = false, arity = 1) public String bamFile; @@ -569,6 +641,15 @@ public class RunSamtoolsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java index fcfe1679c19..7a80b5b62dc 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisClinicalCommandOptions.java @@ -611,6 +611,15 @@ public class RunInterpreterCancerTieringCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -649,6 +658,15 @@ public class RunInterpreterExomiserCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -681,6 +699,15 @@ public class RunInterpreterTeamCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -722,6 +749,15 @@ public class RunInterpreterTieringCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -763,6 +799,15 @@ public class RunInterpreterZettaCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--clinical-analysis"}, description = "The body web service clinicalAnalysis parameter", required = false, arity = 1) public String clinicalAnalysis; @@ -945,6 +990,15 @@ public class LoadCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) public String file; @@ -1232,6 +1286,15 @@ public class RunRgaIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--auxiliar-index"}, description = "Index auxiliar collection to improve performance assuming RGA is completely indexed.", required = false, help = true, arity = 0) public boolean auxiliarIndex = false; @@ -1721,7 +1784,7 @@ public class QueryVariantCommandOptions { @Parameter(names = {"--include-interpretation"}, description = "Interpretation ID to include the fields related to this interpretation", required = false, arity = 1) public String includeInterpretation; - @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) + @Parameter(names = {"--id"}, description = "List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T", required = false, arity = 1) public String id; @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) @@ -1802,7 +1865,7 @@ public class QueryVariantCommandOptions { @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) public String ct; - @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ...", required = false, arity = 1) public String xref; @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 82b9edc17a4..1c83e822d82 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -177,7 +177,7 @@ public class AggregationStatsCommandOptions { @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) public String ct; - @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ...", required = false, arity = 1) public String xref; @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) @@ -259,7 +259,7 @@ public class QueryAnnotationCommandOptions { @ParametersDelegate public CommonCommandOptions commonOptions = commonCommandOptions; - @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) + @Parameter(names = {"--id"}, description = "List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T", required = false, arity = 1) public String id; @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) @@ -366,6 +366,15 @@ public class RunCohortStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; @@ -410,10 +419,22 @@ public class RunExomiserCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--clinical-analysis-type"}, description = "Clinical analysis type: SINGLE or FAMILY.", required = false, arity = 1) + public String clinicalAnalysisType = "SINGLE"; + + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; } @@ -454,6 +475,15 @@ public class RunExportCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; @@ -770,6 +800,15 @@ public class RunFamilyQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; @@ -802,6 +841,15 @@ public class DeleteFileCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -840,6 +888,15 @@ public class RunGatkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -878,6 +935,15 @@ public class RunGenomePlotCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; @@ -922,6 +988,15 @@ public class RunGwasCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--phenotype"}, description = "The body web service phenotype parameter", required = false, arity = 1) public String phenotype; @@ -987,13 +1062,22 @@ public class RunHrDetectCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "ID to identify the HRDetect results.", required = false, arity = 1) public String id; @Parameter(names = {"--description"}, description = "Decription for these particular HRDetect results.", required = false, arity = 1) public String description; - @Parameter(names = {"--sample-id"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) + @Parameter(names = {"--sample-id"}, description = "Sample ID.", required = false, arity = 1) public String sampleId; @Parameter(names = {"--snv-fitting-id"}, description = "Mutational signature fitting ID for SNV.", required = false, arity = 1) @@ -1055,6 +1139,15 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; @@ -1079,7 +1172,7 @@ public class RunIndexCommandOptions { @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO", required = false, help = true, arity = 0) public boolean family = false; @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) @@ -1174,6 +1267,15 @@ public class RunIndividualQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; @@ -1215,6 +1317,15 @@ public class RunInferredSexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--individual"}, description = "Individual ID", required = false, arity = 1) public String individual; @@ -1293,6 +1404,15 @@ public class RunKnockoutCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; @@ -1352,6 +1472,15 @@ public class RunMendelianErrorCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, arity = 1) public String family; @@ -1490,6 +1619,15 @@ public class RunMutationalSignatureCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "Signature ID.", required = false, arity = 1) public String id; @@ -1567,6 +1705,15 @@ public class RunPlinkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) public String outdir; @@ -1611,7 +1758,7 @@ public class QueryCommandOptions { @Parameter(names = {"--saved-filter"}, description = "Use a saved filter at User level", required = false, arity = 1) public String savedFilter; - @Parameter(names = {"--id"}, description = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T", required = false, arity = 1) + @Parameter(names = {"--id"}, description = "List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T", required = false, arity = 1) public String id; @Parameter(names = {"--region"}, description = "List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000", required = false, arity = 1) @@ -1737,7 +1884,7 @@ public class QueryCommandOptions { @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) public String ct; - @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ...", required = false, arity = 1) + @Parameter(names = {"--xref"}, description = "List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ...", required = false, arity = 1) public String xref; @Parameter(names = {"--biotype"}, description = "List of biotypes, e.g. protein_coding", required = false, arity = 1) @@ -1841,6 +1988,15 @@ public class RunRelatednessCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--individuals"}, description = "The body web service individuals parameter", required = false, arity = 1) public String individuals; @@ -1885,6 +2041,15 @@ public class RunRvtestsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--command"}, description = "The body web service command parameter", required = false, arity = 1) public String command; @@ -1997,6 +2162,15 @@ public class RunSampleEligibilityCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--query"}, description = "The body web service query parameter", required = false, arity = 1) public String query; @@ -2035,7 +2209,16 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--sample"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--sample"}, description = "Sample ID.", required = false, arity = 1) public String sample; @Parameter(names = {"--vs-id"}, description = "Variant stats ID.", required = false, arity = 1) @@ -2228,6 +2411,15 @@ public class RunSampleCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) public String id; @@ -2400,6 +2592,15 @@ public class RunSampleStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) public String sample; @@ -2540,6 +2741,15 @@ public class RunStatsExportCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohorts"}, description = "The body web service cohorts parameter", required = false, arity = 1) public String cohorts; @@ -2584,6 +2794,15 @@ public class RunStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java index a752ded42f5..d69121d7c21 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/DiseasePanelsCommandOptions.java @@ -243,6 +243,15 @@ public class ImportCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--source"}, description = "Comma separated list of sources to import panels from. Current supported sources are 'panelapp' and 'cancer-gene-census'", required = false, arity = 1) public String source; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java index 8001c0d6e63..dfbd2cb0e5d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java @@ -363,6 +363,15 @@ public class FetchCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -459,6 +468,15 @@ public class RunLinkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--uri", "--input", "-i"}, description = "The body web service uri parameter", required = false, arity = 1) public String uri; @@ -506,6 +524,15 @@ public class RunPostlinkCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--files"}, description = "The body web service files parameter", required = false, arity = 1) public String files; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java index 3902f2a61a8..327d9487dfb 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java @@ -43,6 +43,7 @@ public class JobsCommandOptions extends CustomJobsCommandOptions { public DeleteCommandOptions deleteCommandOptions; public InfoCommandOptions infoCommandOptions; public UpdateCommandOptions updateCommandOptions; + public KillCommandOptions killCommandOptions; public HeadLogCommandOptions headLogCommandOptions; public TailLogCommandOptions tailLogCommandOptions; @@ -60,6 +61,7 @@ public JobsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander this.deleteCommandOptions = new DeleteCommandOptions(); this.infoCommandOptions = new InfoCommandOptions(); this.updateCommandOptions = new UpdateCommandOptions(); + this.killCommandOptions = new KillCommandOptions(); this.headLogCommandOptions = new HeadLogCommandOptions(); this.tailLogCommandOptions = new TailLogCommandOptions(); @@ -257,6 +259,9 @@ public class RetryCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -446,6 +451,26 @@ public class UpdateCommandOptions { } + @Parameters(commandNames = {"kill"}, commandDescription ="Send a signal to kill a pending or running job") + public class KillCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--job"}, description = "Job ID or UUID", required = true, arity = 1) + public String job; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + } + @Parameters(commandNames = {"log-head"}, commandDescription ="Show the first lines of a log file (up to a limit)") public class HeadLogCommandOptions { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index babd0e312f2..4d398070124 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -157,13 +157,22 @@ public class AggregateVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--overwrite"}, description = "The body web service overwrite parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite"}, description = "Overwrite aggregation for all files and variants. Repeat operation for already processed variants.", required = false, help = true, arity = 0) public boolean overwrite = false; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; } @@ -186,6 +195,15 @@ public class DeleteVariantAnnotationCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @@ -218,6 +236,15 @@ public class IndexVariantAnnotationCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @@ -277,10 +304,19 @@ public class SaveVariantAnnotationCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; - @Parameter(names = {"--annotation-id"}, description = "The body web service annotationId parameter", required = false, arity = 1) + @Parameter(names = {"--annotation-id"}, description = "New Variant Annotation identifier", required = false, arity = 1) public String annotationId; } @@ -332,6 +368,15 @@ public class DeleteVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -341,6 +386,9 @@ public class DeleteVariantCommandOptions { @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) public boolean resume = false; + @Parameter(names = {"--force"}, description = "The body web service force parameter", required = false, help = true, arity = 0) + public boolean force = false; + } @Parameters(commandNames = {"variant-family-aggregate"}, commandDescription ="Find variants where not all the samples are present, and fill the empty values.") @@ -367,16 +415,25 @@ public class AggregateVariantFamilyCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--samples"}, description = "The body web service samples parameter", required = false, arity = 1) + @Parameter(names = {"--samples"}, description = "Samples within the same study to aggregate", required = false, arity = 1) public String samples; - @Parameter(names = {"--gaps-genotype"}, description = "The body web service gapsGenotype parameter", required = false, arity = 1) + @Parameter(names = {"--gaps-genotype"}, description = "Genotype to be used in gaps. Either 0/0, ./. or ?/?", required = false, arity = 1) public String gapsGenotype; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; } @@ -405,6 +462,15 @@ public class IndexVariantFamilyCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -446,6 +512,15 @@ public class IndexVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -473,7 +548,7 @@ public class IndexVariantCommandOptions { @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO", required = false, help = true, arity = 0) public boolean family = false; @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) @@ -565,6 +640,15 @@ public class LauncherVariantIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -607,7 +691,7 @@ public class LauncherVariantIndexCommandOptions { @Parameter(names = {"--index-params-fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsFailOnMalformedLines; - @Parameter(names = {"--index-params-family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO", required = false, help = true, arity = 0) public boolean indexParamsFamily = false; @Parameter(names = {"--index-params-somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) @@ -699,6 +783,15 @@ public class RunVariantJulieCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "project", required = false, arity = 1) public String project; @@ -737,6 +830,15 @@ public class RepairVariantMetadataCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--studies"}, description = "The body web service studies parameter", required = false, arity = 1) public String studies; @@ -769,6 +871,15 @@ public class SynchronizeVariantMetadataCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -804,6 +915,15 @@ public class PruneVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "The body web service project parameter", required = false, arity = 1) public String project; @@ -839,6 +959,15 @@ public class DeleteVariantSampleCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -877,6 +1006,15 @@ public class IndexVariantSampleCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -935,6 +1073,15 @@ public class DeleteVariantScoreCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -973,6 +1120,15 @@ public class IndexVariantScoreCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -1020,6 +1176,15 @@ public class VariantSecondaryAnnotationIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @@ -1061,6 +1226,15 @@ public class VariantSecondarySampleIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -1125,6 +1299,15 @@ public class SecondaryIndexVariantCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--project", "-p"}, description = "Project [organization@]project where project can be either the ID or the alias", required = false, arity = 1) public String project; @@ -1160,6 +1343,15 @@ public class DeleteVariantSecondaryIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; @@ -1195,6 +1387,15 @@ public class DeleteVariantStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; @@ -1230,6 +1431,15 @@ public class IndexVariantStatsCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--cohort"}, description = "The body web service cohort parameter", required = false, arity = 1) public String cohort; @@ -1274,6 +1484,15 @@ public class DeleteVariantStudyCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java index c2ab1e89a90..68be35b6cac 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OrganizationsCommandOptions.java @@ -38,6 +38,9 @@ public class OrganizationsCommandOptions { public SearchNotesCommandOptions searchNotesCommandOptions; public DeleteNotesCommandOptions deleteNotesCommandOptions; public UpdateNotesCommandOptions updateNotesCommandOptions; + public UserUpdateStatusCommandOptions userUpdateStatusCommandOptions; + public UpdateUserCommandOptions updateUserCommandOptions; + public UpdateConfigurationCommandOptions updateConfigurationCommandOptions; public InfoCommandOptions infoCommandOptions; public UpdateCommandOptions updateCommandOptions; @@ -51,6 +54,9 @@ public OrganizationsCommandOptions(CommonCommandOptions commonCommandOptions, JC this.searchNotesCommandOptions = new SearchNotesCommandOptions(); this.deleteNotesCommandOptions = new DeleteNotesCommandOptions(); this.updateNotesCommandOptions = new UpdateNotesCommandOptions(); + this.userUpdateStatusCommandOptions = new UserUpdateStatusCommandOptions(); + this.updateUserCommandOptions = new UpdateUserCommandOptions(); + this.updateConfigurationCommandOptions = new UpdateConfigurationCommandOptions(); this.infoCommandOptions = new InfoCommandOptions(); this.updateCommandOptions = new UpdateCommandOptions(); @@ -89,6 +95,9 @@ public class CreateCommandOptions { @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) public String modificationDate; + @Parameter(names = {"--configuration-default-user-expiration-date"}, description = "The body web service defaultUserExpirationDate parameter", required = false, arity = 1) + public String configurationDefaultUserExpirationDate; + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; @@ -216,6 +225,135 @@ public class UpdateNotesCommandOptions { } + @Parameters(commandNames = {"update-status-user"}, commandDescription ="Update the user status") + public class UserUpdateStatusCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--user", "-u"}, description = "User ID", required = true, arity = 1) + public String user; + + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--status"}, description = "The body web service status parameter", required = false, arity = 1) + public String status; + + } + + @Parameters(commandNames = {"user-update"}, commandDescription ="Update the user information") + public class UpdateUserCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--user", "-u"}, description = "User ID", required = true, arity = 1) + public String user; + + @Parameter(names = {"--organization"}, description = "Organization id", required = false, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + + @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) + public String email; + + @Parameter(names = {"--quota-disk-usage"}, description = "The body web service diskUsage parameter", required = false, arity = 1) + public Long quotaDiskUsage; + + @Parameter(names = {"--quota-cpu-usage"}, description = "The body web service cpuUsage parameter", required = false, arity = 1) + public Integer quotaCpuUsage; + + @Parameter(names = {"--quota-max-disk"}, description = "The body web service maxDisk parameter", required = false, arity = 1) + public Long quotaMaxDisk; + + @Parameter(names = {"--quota-max-cpu"}, description = "The body web service maxCpu parameter", required = false, arity = 1) + public Integer quotaMaxCpu; + + @Parameter(names = {"--account-expiration-date"}, description = "The body web service expirationDate parameter", required = false, arity = 1) + public String accountExpirationDate; + + @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"configuration-update"}, commandDescription ="Update the Organization configuration attributes") + public class UpdateConfigurationCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--organization"}, description = "Organization id", required = true, arity = 1) + public String organization; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--authentication-origins-action"}, description = "Action to be performed if the array of authenticationOrigins is being updated.", required = false, arity = 1) + public String authenticationOriginsAction = "ADD"; + + @Parameter(names = {"--default-user-expiration-date"}, description = "The body web service defaultUserExpirationDate parameter", required = false, arity = 1) + public String defaultUserExpirationDate; + + @Parameter(names = {"--optimizations-simplify-permissions"}, description = "The body web service simplifyPermissions parameter", required = false, help = true, arity = 0) + public boolean optimizationsSimplifyPermissions = false; + + @Parameter(names = {"--token-algorithm"}, description = "The body web service algorithm parameter", required = false, arity = 1) + public String tokenAlgorithm; + + @Parameter(names = {"--token-secret-key"}, description = "The body web service secretKey parameter", required = false, arity = 1) + public String tokenSecretKey; + + @Parameter(names = {"--token-expiration"}, description = "The body web service expiration parameter", required = false, arity = 1) + public Long tokenExpiration; + + } + @Parameters(commandNames = {"info"}, commandDescription ="Return the organization information") public class InfoCommandOptions { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java index 885a2ffaa70..66d7cee55a8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java @@ -285,9 +285,6 @@ public class UpdateCommandOptions { @Parameter(names = {"--email"}, description = "The body web service email parameter", required = false, arity = 1) public String email; - @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) - public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; - } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_3/catalog/java/Migration1.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_3/catalog/java/Migration1.java new file mode 100644 index 00000000000..425fd72353f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_3/catalog/java/Migration1.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_0_3.catalog.java; + +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "recalculate_roles", description = "Recalculate roles from Family #1763", version = "2.0.3", date = 20210528, + deprecatedSince = "3.0.0") +public class Migration1 extends MigrationTool { + + @Override + protected void run() throws CatalogException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_5/catalog/java/initialiseGroups.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_5/catalog/java/initialiseGroups.java new file mode 100644 index 00000000000..0c0273e7820 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_5/catalog/java/initialiseGroups.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_0_5.catalog.java; + + +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "initialise_groups", description = "Initialise userIds list from groups #1791", version = "2.0.5", date = 20210621, + deprecatedSince = "3.0.0") +public class initialiseGroups extends MigrationTool { + + @Override + protected void run() throws CatalogException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java new file mode 100644 index 00000000000..b99e0a9f1d5 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_0_6/catalog/removeDeletedFileReferencesFromSample.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_0_6.catalog; + + +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_file_references_from_sample", description = "Remove deleted file references from samples #1815", version = "2.0.6", + date = 20210901, deprecatedSince = "3.0.0") +public class removeDeletedFileReferencesFromSample extends MigrationTool { + + @Override + protected void run() throws CatalogException { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java new file mode 100644 index 00000000000..06a78900ab0 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_12_0.catalog; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_annotation_sets_to_clinical_analysis" , + description = "Add private annotation fields to ClinicalAnalysis documents #TASK-5198", + version = "2.12.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20231116, + deprecatedSince = "3.0.0" +) +public class AddAnnotationSetsInClinicalAnalysisMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java new file mode 100644 index 00000000000..96f702d96f4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_12_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_clinical_report_data_model" , + description = "Complete Clinical Report data model #TASK-5198", + version = "2.12.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20231128, + deprecatedSince = "3.0.0" +) +public class CompleteClinicalReportDataModelMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java new file mode 100644 index 00000000000..6f6f5cbf18d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_12_6/SyncCohortsAndSamplesMigration.java @@ -0,0 +1,73 @@ +package org.opencb.opencga.app.migrations.v2.v2_12_6; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Updates; +import org.apache.commons.collections4.CollectionUtils; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.api.CohortDBAdaptor; +import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.List; +import java.util.stream.Collectors; + +@Migration(id = "syncCohortsAndSamplesMigration" , + description = "Sync array of samples from cohort with array of cohortIds from Sample", + version = "2.12.6", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20240621 +) +public class SyncCohortsAndSamplesMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + MongoCollection sampleCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION); + MongoCollection sampleArchiveCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION); + + queryMongo(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, new Document(), + Projections.include(CohortDBAdaptor.QueryParams.ID.key(), CohortDBAdaptor.QueryParams.SAMPLES.key()), + cohortDoc -> { + String cohortId = cohortDoc.getString(CohortDBAdaptor.QueryParams.ID.key()); + List samples = cohortDoc.getList(CohortDBAdaptor.QueryParams.SAMPLES.key(), Document.class); + if (CollectionUtils.isNotEmpty(samples)) { + List sampleUids = samples + .stream() + .map(s -> s.get(SampleDBAdaptor.QueryParams.UID.key(), Number.class).longValue()) + .collect(Collectors.toList()); + // Ensure all those samples have a reference to the cohortId + Bson query = Filters.and( + Filters.in(SampleDBAdaptor.QueryParams.UID.key(), sampleUids), + Filters.eq(MongoDBAdaptor.LAST_OF_VERSION, true) + ); + Bson update = Updates.addToSet(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId); + long addedMissingCohort = sampleCollection.updateMany(query, update).getModifiedCount(); + sampleArchiveCollection.updateMany(query, update); + + // Ensure there aren't any samples pointing to this cohort that are not in the samples array + query = Filters.and( + Filters.nin(SampleDBAdaptor.QueryParams.UID.key(), sampleUids), + Filters.eq(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId), + Filters.eq(MongoDBAdaptor.LAST_OF_VERSION, true) + ); + update = Updates.pull(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), cohortId); + long removedNonAssociatedCohort = sampleCollection.updateMany(query, update).getModifiedCount(); + sampleArchiveCollection.updateMany(query, update); + + if (addedMissingCohort > 0 || removedNonAssociatedCohort > 0) { + logger.info("Fixed cohort '{}' references. " + + "Added missing reference to {} samples. " + + "Removed non-associated reference from {} samples.", + cohortId, addedMissingCohort, removedNonAssociatedCohort); + } + } + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddMissingIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddMissingIndexes.java new file mode 100644 index 00000000000..7efdea6a56a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddMissingIndexes.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_indexes", + description = "Add missing indexes", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210928, + deprecatedSince = "3.0.0") +public class AddMissingIndexes extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddPanelsToInterpretations.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddPanelsToInterpretations.java new file mode 100644 index 00000000000..a35f794d130 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/AddPanelsToInterpretations.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panels_to_interpretations", + description = "Add panels to Interpretations #1802", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20210713, + deprecatedSince = "3.0.0") +public class AddPanelsToInterpretations extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/CreateAuditIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/CreateAuditIndexes.java new file mode 100644 index 00000000000..48cd8ce2f3b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/CreateAuditIndexes.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "create_audit_indexes", description = "Create Audit indexes", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210622, + deprecatedSince = "3.0.0") +public class CreateAuditIndexes extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/PanelLock.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/PanelLock.java new file mode 100644 index 00000000000..747feb3bbcb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/PanelLock.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panel_lock", + description = "Add new panelLock to ClinicalAnalysis #1802", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210713, + deprecatedSince = "3.0.0") +public class PanelLock extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java new file mode 100644 index 00000000000..58625266d9b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RemoveRCVersionsFromMigrationRuns.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_rc_version_from_migration_runs", + description = "Remove RC versions from migration runs stored in catalog", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210723, + deprecatedSince = "3.0.0") +public class RemoveRCVersionsFromMigrationRuns extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java new file mode 100644 index 00000000000..452e99d627f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/RenameDatastoreConfigurationToOptions.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_datastore_configuration_to_options", description = "Rename project.internal.datastores.variant.configuration to options", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210617, + deprecatedSince = "3.0.0") +public class RenameDatastoreConfigurationToOptions extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java new file mode 100644 index 00000000000..18e6272d7a6 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/StudyClinicalConfigurationRelocation.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "move_study_clinical_config_to_internal", + description = "Move Study ClinicalConfiguration to internal.configuration", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210708, + deprecatedSince = "3.0.0") +public class StudyClinicalConfigurationRelocation extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/VariantFileStatsRelocation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/VariantFileStatsRelocation.java new file mode 100644 index 00000000000..f37d46cb9f2 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/java/VariantFileStatsRelocation.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.java; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "move_variant_file_stats_to_qc", description = "Move opencga_file_variant_stats annotation set from variable sets to " + + "FileQualityControl", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 1, + date = 20210614, + deprecatedSince = "3.0.0") +public class VariantFileStatsRelocation extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration1.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration1.java new file mode 100644 index 00000000000..0530e369b04 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration1.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "build_rga_indexes", description = "Create index for sample RGA status #1693", version = "2.1.0", + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528, deprecatedSince = "3.0.0") +public class Migration1 extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration2.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration2.java new file mode 100644 index 00000000000..a0c0938de73 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration2.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "init_userId_group_arrays", description = "Initialise all userIds arrays from groups #1735", version = "2.1.0", + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528, deprecatedSince = "3.0.0") +public class Migration2 extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration3.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration3.java new file mode 100644 index 00000000000..9b21dcdbc9c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration3.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "init_ca_panel_arrays", description = "Initialise panels array in Clinical Analysis #1759", version = "2.1.0", + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210528, deprecatedSince = "3.0.0") +public class Migration3 extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration4.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration4.java new file mode 100644 index 00000000000..ef72a501f44 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/catalog/javascript/Migration4.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.catalog.javascript; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "update_qc_file_sample_fields", description = "Update QC fields from Sample and File #1730", version = "2.1.0", + language = Migration.MigrationLanguage.JAVASCRIPT, date = 20210531, patch = 4, deprecatedSince = "3.0.0") +public class Migration4 extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/AddCellbaseConfigurationToProject.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/AddCellbaseConfigurationToProject.java new file mode 100644 index 00000000000..5dac53d4688 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/AddCellbaseConfigurationToProject.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.storage; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_cellbase_configuration_to_project", description = "Add cellbase configuration from storage-configuration.yml to project.internal.cellbase", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 3, + date = 20210616, + deprecatedSince = "3.0.0") +public class AddCellbaseConfigurationToProject extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/DefaultSampleIndexConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/DefaultSampleIndexConfiguration.java new file mode 100644 index 00000000000..7b33bd7d709 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/DefaultSampleIndexConfiguration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.storage; + + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "default_sample_index_configuration", description = "Add a default backward compatible sample index configuration", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 7, + date = 20210721, + deprecatedSince = "3.0.0") // Needs to run after StudyClinicalConfigurationRelocation +public class DefaultSampleIndexConfiguration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/NewClinicalSignificanceFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/NewClinicalSignificanceFields.java new file mode 100644 index 00000000000..b4d243ee279 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_1_0/storage/NewClinicalSignificanceFields.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_1_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + + +@Migration(id = "new_clinical_significance_fields", description = "Add new clinical significance fields and combinations for variant storage and solr", version = "2.1.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + patch = 1, + date = 20210708, + deprecatedSince = "3.0.0") +public class NewClinicalSignificanceFields extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddMissingClinicalAudit.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddMissingClinicalAudit.java new file mode 100644 index 00000000000..7008388f42e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddMissingClinicalAudit.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_create_interpretation_in_clinical_audit", + description = "Add missing CREATE_INTERPRETATION audits in ClinicalAnalysis", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211227, deprecatedSince = "3.0.0") +public class AddMissingClinicalAudit extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNameFieldInCohort.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNameFieldInCohort.java new file mode 100644 index 00000000000..1f23c1b331c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNameFieldInCohort.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_name_field_in_cohort_1902", + description = "Add new name field to Cohort #1902", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220228, deprecatedSince = "3.0.0") +public class AddNameFieldInCohort extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewAllowedBiotype.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewAllowedBiotype.java new file mode 100644 index 00000000000..1aa51b53014 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewAllowedBiotype.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_new_allowed_biotype", + description = "Add new allowed biotype 'guide_RNA', #1856", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20211209, deprecatedSince = "3.0.0") +public class AddNewAllowedBiotype extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewFileInternalIndex_1850.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewFileInternalIndex_1850.java new file mode 100644 index 00000000000..099dd960790 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddNewFileInternalIndex_1850.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "new_file_internal_index_1850", + description = "Add new FileInternalVariant and FileInternalAlignment index #1850", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211127, deprecatedSince = "3.0.0") +public class AddNewFileInternalIndex_1850 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddSampleInternalVariant_1851.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddSampleInternalVariant_1851.java new file mode 100644 index 00000000000..21218e3222d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/AddSampleInternalVariant_1851.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_sample_internal_variant_1851", + description = "Add new SampleInternalVariant #1851", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211129, deprecatedSince = "3.0.0") +public class AddSampleInternalVariant_1851 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ChangeInterpretationMethods.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ChangeInterpretationMethods.java new file mode 100644 index 00000000000..7731a4510fe --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ChangeInterpretationMethods.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "change_interpretation_method", + description = "Remove list of methods from Interpretations #1841", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211112, deprecatedSince = "3.0.0") +public class ChangeInterpretationMethods extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java new file mode 100644 index 00000000000..6de2592fbaa --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ClinicalVariantEvidenceMigration.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_clinical_variant_evidence_review", + description = "Add new ClinicalVariantEvidenceReview object, #1874", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220112, deprecatedSince = "3.0.0") +public class ClinicalVariantEvidenceMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/DeleteUnusedVariableSets.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/DeleteUnusedVariableSets.java new file mode 100644 index 00000000000..7ddea73508c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/DeleteUnusedVariableSets.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "delete_unused_variablesets", + description = "Delete unused VariableSets, #1859", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20211210, deprecatedSince = "3.0.0") +public class DeleteUnusedVariableSets extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixFamilyReferences.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixFamilyReferences.java new file mode 100644 index 00000000000..fd160750475 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixFamilyReferences.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "fix_family_references_in_individual", + description = "Fix Family references, #TASK-489", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220324, deprecatedSince = "3.0.0") +public class FixFamilyReferences extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixNonExistingMoIFromPanels.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixNonExistingMoIFromPanels.java new file mode 100644 index 00000000000..d642d8499af --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/FixNonExistingMoIFromPanels.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "fix_non_existing_mois_from_panels", + description = "Remove non-existing MOIs from Panels", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20220111, deprecatedSince = "3.0.0") +public class FixNonExistingMoIFromPanels extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java new file mode 100644 index 00000000000..b7ff196615b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveClinicalAnalysisQualityControl.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improve_ca_quality_control", + description = "Quality control normalize comments and fields #1826", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211001, deprecatedSince = "3.0.0") +public class ImproveClinicalAnalysisQualityControl extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveFileQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveFileQualityControl.java new file mode 100644 index 00000000000..16043d38b26 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveFileQualityControl.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improve_file_quality_control", + description = "Quality control normalize comments and fields #1826", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211001, deprecatedSince = "3.0.0") +public class ImproveFileQualityControl extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveIndividualQualityControl.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveIndividualQualityControl.java new file mode 100644 index 00000000000..b3350e099f3 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveIndividualQualityControl.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improve_individual_quality_control", + description = "Quality control normalize comments and fields #1826", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211001, deprecatedSince = "3.0.0") +public class ImproveIndividualQualityControl extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java new file mode 100644 index 00000000000..39b2df0d2a2 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/ImproveVariableSetNamesAndDescriptions.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "improveVariableSetNamesAndDescriptions", + description = "Improve VariableSet names and descriptions", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211210) +public class ImproveVariableSetNamesAndDescriptions extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveFileDocsFromCA.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveFileDocsFromCA.java new file mode 100644 index 00000000000..af85465d69e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveFileDocsFromCA.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_file_docs_from_clinical_analyses", + description = "Store references of File in Clinical Analysis and not full File documents #1837", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211102, deprecatedSince = "3.0.0") +public class RemoveFileDocsFromCA extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveParallelIndexes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveParallelIndexes.java new file mode 100644 index 00000000000..7b00543f9a5 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RemoveParallelIndexes.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_parallel_indexes", + description = "Remove parallel array indexes #CU-20jc4tx", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220310, deprecatedSince = "3.0.0") +public class RemoveParallelIndexes extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFamilyQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFamilyQualityControlFields.java new file mode 100644 index 00000000000..821fb020a28 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFamilyQualityControlFields.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_family_quality_control_fields", + description = "Rename FamilyQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameFamilyQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFileQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFileQualityControlFields.java new file mode 100644 index 00000000000..c3c5d4ff7f8 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameFileQualityControlFields.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_file_quality_control_fields", + description = "Rename FileQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameFileQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameIndividualQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameIndividualQualityControlFields.java new file mode 100644 index 00000000000..6df5fc82131 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameIndividualQualityControlFields.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_individual_quality_control_fields", + description = "Rename IndividualQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameIndividualQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameInterpretationFindingStats.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameInterpretationFindingStats.java new file mode 100644 index 00000000000..bd4ad966774 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameInterpretationFindingStats.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_interpretation_stats_field", + description = "Rename interpretation stats field #1819", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211115, deprecatedSince = "3.0.0") +public class RenameInterpretationFindingStats extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameSampleQualityControlFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameSampleQualityControlFields.java new file mode 100644 index 00000000000..0c5b36cdf93 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/RenameSampleQualityControlFields.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_sample_quality_control_fields", + description = "Rename SampleQualityControl fields #1844", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20211119, deprecatedSince = "3.0.0") +public class RenameSampleQualityControlFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java new file mode 100644 index 00000000000..9a43d99ac53 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addDefaultVariantCallerInterpretationStudyConfiguration.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_variant_caller_interpretation_configuration", + description = "Add default variant caller Interpretation Study configuration #1822", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20210916, deprecatedSince = "3.0.0") +public class addDefaultVariantCallerInterpretationStudyConfiguration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java new file mode 100644 index 00000000000..28e32235bb0 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddInternalLastModified.java @@ -0,0 +1,10 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.MigrationTool; + +public abstract class AddInternalLastModified extends MigrationTool { + + protected void addInternalModificationDate(String collection) { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java new file mode 100644 index 00000000000..c127e7b2695 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToClinicalInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_clinical.internal", description = "Add internal modificationDate to Clinical #1810", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToClinicalInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java new file mode 100644 index 00000000000..0aaccf98b74 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToCohortInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_cohort.internal", description = "Add internal modificationDate to Cohort #1810", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToCohortInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java new file mode 100644 index 00000000000..914b6351a0c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFamilyInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_family.internal", description = "Add internal modificationDate to Family #1810", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToFamilyInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java new file mode 100644 index 00000000000..7ffc3cf7f37 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToFileInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_file.internal", description = "Add internal modificationDate to File #1810", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToFileInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java new file mode 100644 index 00000000000..05ca7bc522d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToIndividualInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_individual.internal", description = "Add internal modificationDate to Individual #1810", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToIndividualInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java new file mode 100644 index 00000000000..9340231dfde --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToInterpretationInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_interpretation.internal", description = "Add internal modificationDate to Interpretation #1810", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToInterpretationInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java new file mode 100644 index 00000000000..543e02f328e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToJobInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_job.internal", description = "Add internal modificationDate to Job #1810", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToJobInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java new file mode 100644 index 00000000000..82a3142a222 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToProjectInternal.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_modificationDate_to_project.internal", description = "Add internal modificationDate to Project #1810", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToProjectInternal extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java new file mode 100644 index 00000000000..875bbaa26a7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToSampleInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_sample.internal", description = "Add internal modificationDate to Sample #1810", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToSampleInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java new file mode 100644 index 00000000000..987a742cb5d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInternalLastModified/AddModificationDateToStudyInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addInternalLastModified; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_modificationDate_to_study.internal", description = "Add internal modificationDate to Study #1810", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210812, deprecatedSince = "3.0.0") +public class AddModificationDateToStudyInternal extends AddInternalLastModified { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInterpretationStats.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInterpretationStats.java new file mode 100644 index 00000000000..ac1c112d0a1 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addInterpretationStats.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_interpretation_stats", + description = "Add interpretation stats #1819", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 2, + date = 20210908, deprecatedSince = "3.0.0") +public class addInterpretationStats extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java new file mode 100644 index 00000000000..efd1107ba7b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDate.java @@ -0,0 +1,10 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.MigrationTool; + +public abstract class AddRegistrationDate extends MigrationTool { + + protected void addRegistrationDate(String collection) { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java new file mode 100644 index 00000000000..14a48ed90be --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToClinicalInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_clinical.internal", description = "Add registrationDate to Clinical #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToClinicalInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java new file mode 100644 index 00000000000..a7dcf145b33 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToCohortInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_cohort.internal", description = "Add registrationDate to Cohort #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToCohortInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java new file mode 100644 index 00000000000..66f375ce955 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFamilyInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_family.internal", description = "Add registrationDate to Family #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToFamilyInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java new file mode 100644 index 00000000000..cfa29de842a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToFileInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_file.internal", description = "Add registrationDate to File #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToFileInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java new file mode 100644 index 00000000000..58d233f6d7f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToIndividualInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_individual.internal", description = "Add registrationDate to Individual #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToIndividualInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java new file mode 100644 index 00000000000..115b4e4fc6c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToInterpretationInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_interpretation.internal", description = "Add registrationDate to Interpretation #1804", + version = "2.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToInterpretationInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java new file mode 100644 index 00000000000..c56124cdda6 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToJobInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_job.internal", description = "Add registrationDate to Job #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToJobInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java new file mode 100644 index 00000000000..ae3f65b70d9 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToProjectInternal.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_registrationDate_to_project.internal", description = "Add registrationDate to Project #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToProjectInternal extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java new file mode 100644 index 00000000000..e317c667a98 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToSampleInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_sample.internal", description = "Add registrationDate to Sample #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToSampleInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java new file mode 100644 index 00000000000..7d036f65bc4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRegistrationDate/AddRegistrationDateToStudyInternal.java @@ -0,0 +1,13 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.addRegistrationDate; + +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_registrationDate_to_study.internal", description = "Add registrationDate to Study #1804", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, + date = 20210720, deprecatedSince = "3.0.0") +public class AddRegistrationDateToStudyInternal extends AddRegistrationDate { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRgaIndexToStudyInternal.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRgaIndexToStudyInternal.java new file mode 100644 index 00000000000..d995f4ba97f --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/addRgaIndexToStudyInternal.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_rga_index_summary_to_study_internal", + description = "Add RGA Index information to Study Internal #", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + patch = 1, + date = 20210719, deprecatedSince = "3.0.0") +public class addRgaIndexToStudyInternal extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java new file mode 100644 index 00000000000..25e93347812 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1795/AddFamilyIdsInIndividual.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1795; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_familyIds_in_individual", description = "Add new list of familyIds in Individual #1795", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, date = 20210630, deprecatedSince = "3.0.0") +public class AddFamilyIdsInIndividual extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java new file mode 100644 index 00000000000..b6d1d3ff298 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1796/AddCohortIdsInSample.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1796; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationException; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_cohortIds_in_sample", description = "Add new list of cohortIds in Sample #1796", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, date = 20210706, deprecatedSince = "3.0.0") +public class AddCohortIdsInSample extends MigrationTool { + + @Override + protected void run() throws MigrationException { + } +} + diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java new file mode 100644 index 00000000000..fcb2313cb39 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteClinicalStatusDataModel.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_clinical_status_models", + description = "Complete Clinical Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, + deprecatedSince = "3.0.0") +public class CompleteClinicalStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java new file mode 100644 index 00000000000..692e58cc397 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteCohortStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_cohort_status_models", + description = "Complete Cohort Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteCohortStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java new file mode 100644 index 00000000000..5c1bfbfc2aa --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFamilyStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_family_status_models", + description = "Complete Family Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteFamilyStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java new file mode 100644 index 00000000000..c1c84058b5c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteFileStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_file_status_models", + description = "Complete File Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteFileStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java new file mode 100644 index 00000000000..cb7fda578ca --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteIndividualStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_individual_status_models", + description = "Complete Individual Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteIndividualStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java new file mode 100644 index 00000000000..dad8a984810 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteInterpretationStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_interpretation_status_models", + description = "Complete Interpretation Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteInterpretationStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java new file mode 100644 index 00000000000..4a08d4695e7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteJobStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_job_status_models", + description = "Complete Job Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteJobStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java new file mode 100644 index 00000000000..8311dcc2153 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompletePanelStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_panel_status_models", + description = "Complete Panel Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompletePanelStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java new file mode 100644 index 00000000000..09ce8a4fc88 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteProjectStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_project_status_models", + description = "Complete Project Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteProjectStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java new file mode 100644 index 00000000000..bb9b3e54086 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteSampleStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_sample_status_models", + description = "Complete Sample Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteSampleStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java new file mode 100644 index 00000000000..8b119d92cc4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteStudyStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_study_status_models", + description = "Complete Study Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteStudyStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java new file mode 100644 index 00000000000..937e6d3f9fb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issue_1849/CompleteUserStatusDataModel.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issue_1849; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "complete_user_status_models", + description = "Complete User Status data models #1849", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211126, deprecatedSince = "3.0.0") +public class CompleteUserStatusDataModel extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java new file mode 100644 index 00000000000..9a5eb6cec84 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/IndividualOntologyTermMigration.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issues_1853_1855; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "individual_ontology_term_migration_#1853", + description = "Change sex and ethnicity types for OntologyTermAnnotation #1855", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211203, deprecatedSince = "3.0.0") +public class IndividualOntologyTermMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java new file mode 100644 index 00000000000..e6659fb4ad7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/NewStudyDataModelFields.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issues_1853_1855; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "new_study_data_model_fields_#1853", + description = "New Study 'sources', 'type' and 'additionalInfo' fields #1853", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211202, deprecatedSince = "3.0.0") +public class NewStudyDataModelFields extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java new file mode 100644 index 00000000000..a054a6a0566 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/issues_1853_1855/SampleProcessingAndCollectionChanges.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog.issues_1853_1855; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "sample_source_treatments_#1854", + description = "Sample source, treatments, processing and collection changes #1854", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20211201, patch = 2, deprecatedSince = "3.0.0") +public class SampleProcessingAndCollectionChanges extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/renameVariableSetFieldFromVariable.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/renameVariableSetFieldFromVariable.java new file mode 100644 index 00000000000..bd3f00937e9 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/catalog/renameVariableSetFieldFromVariable.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.catalog; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_variableset_field", + description = "Rename Variable variableSet field to variables #1823", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20210920, deprecatedSince = "3.0.0") +public class renameVariableSetFieldFromVariable extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/SynchronizeCatalogStorage.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/SynchronizeCatalogStorage.java new file mode 100644 index 00000000000..609d50c915a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/SynchronizeCatalogStorage.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "synchronize_catalog_storage_1850_1851", + description = "Synchronize catalog storage #1850 #1851", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220118, deprecatedSince = "3.0.0") +public class SynchronizeCatalogStorage extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/UpdateSampleIndexStatus.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/UpdateSampleIndexStatus.java new file mode 100644 index 00000000000..2965c326760 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_0/storage/UpdateSampleIndexStatus.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "update_sample_index_status_1901", + description = " Improve sample-index multi-schema management. #1901", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220302, deprecatedSince = "3.0.0") +public class UpdateSampleIndexStatus extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddMissingBiotypes.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddMissingBiotypes.java new file mode 100644 index 00000000000..7173a402e96 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddMissingBiotypes.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_biotypes", + description = "Add missing biotypes, #TASK-625", version = "2.2.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220412, deprecatedSince = "3.0.0") +public class AddMissingBiotypes extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddPanelGeneNameIndex.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddPanelGeneNameIndex.java new file mode 100644 index 00000000000..0cd9ba5ff1b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/AddPanelGeneNameIndex.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panel_gene_name_index", + description = "Add Panel gene.name index #TASK-602", version = "2.2.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220406, deprecatedSince = "3.0.0") +public class AddPanelGeneNameIndex extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/InitSharedProjectField.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/InitSharedProjectField.java new file mode 100644 index 00000000000..49b4b252f60 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/catalog/InitSharedProjectField.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "init_shared_project", + description = "Initialise sharedProjects #TASK-702", version = "2.2.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220502, deprecatedSince = "3.0.0") +public class InitSharedProjectField extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java new file mode 100644 index 00000000000..cd11e50f8aa --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_2_1/storage/SampleIndexConfigurationAddMissingStatus.java @@ -0,0 +1,15 @@ +package org.opencb.opencga.app.migrations.v2.v2_2_1.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "sample_index_configuration_add_missing_status", + description = "Sample index configuration add missing status #TASK-512", version = "2.2.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220328, deprecatedSince = "3.0.0") +public class SampleIndexConfigurationAddMissingStatus extends StorageMigrationTool { + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddLockedToInterpretation.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddLockedToInterpretation.java new file mode 100644 index 00000000000..52a95ccf66e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddLockedToInterpretation.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_locked_to_interpretation_TASK-552", + description = "Add new 'locked' field to Interpretation #TASK-552", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220401, + deprecatedSince = "3.0.0") +public class AddLockedToInterpretation extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddPanelInternalField.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddPanelInternalField.java new file mode 100644 index 00000000000..e2edbf68060 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AddPanelInternalField.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_panel_internal_field-TASK_734", + description = "Add 'internal' to Panel data model #TASK-734", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220509, + deprecatedSince = "3.0.0") +public class AddPanelInternalField extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AutoIncrementVersion.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AutoIncrementVersion.java new file mode 100644 index 00000000000..8cbd5a59eef --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/AutoIncrementVersion.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "autoincrement_version_TASK-323", + description = "Reorganise data in new collections (autoincrement version) #TASK-323", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + offline = true, + date = 20220512, + deprecatedSince = "3.0.0") +public class AutoIncrementVersion extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/InitSampleIndexConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/InitSampleIndexConfiguration.java new file mode 100644 index 00000000000..81d3f1e8628 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/InitSampleIndexConfiguration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "init_sampleIndexConfiguration-TASK550", + description = "Initialise SampleIndexConfiguration in Study internal #TASK-550", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220502, + deprecatedSince = "3.0.0") +public class InitSampleIndexConfiguration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java new file mode 100644 index 00000000000..9f4f30213fb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/MoveCellbaseFromProjectInternalToProject.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "move_cellbase_from_project_internal_to_project_TASK-354", + description = "Move cellbase from project.internal to project #TASK-354", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220422, + deprecatedSince = "3.0.0") + +public class MoveCellbaseFromProjectInternalToProject extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeletedCollections.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeletedCollections.java new file mode 100644 index 00000000000..2a1532c6da4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeletedCollections.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_deleted_collections_TASK-359", + description = "Rename deleted collections #TASK-359", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220502, + deprecatedSince = "3.0.0") +public class RenameDeletedCollections extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java new file mode 100644 index 00000000000..b39a56cdbec --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/RenameDeprecatedVariantStorageToolId.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_variant_storage_tool_ids_TASK-705", + description = "Rename variant storage tool ids #TASK-705", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220506, + deprecatedSince = "3.0.0") +public class RenameDeprecatedVariantStorageToolId extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/UpdateJWTSecretKey.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/UpdateJWTSecretKey.java new file mode 100644 index 00000000000..be328bbeb6a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/catalog/UpdateJWTSecretKey.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "update_jwt_secret_key_TASK-807", + description = "Update JWT secret key #TASK-807", version = "2.3.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220513, + deprecatedSince = "3.0.0") +public class UpdateJWTSecretKey extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/storage/AddMissingColumnsToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/storage/AddMissingColumnsToPhoenix.java new file mode 100644 index 00000000000..df2e308a727 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_0/storage/AddMissingColumnsToPhoenix.java @@ -0,0 +1,14 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id="add_missing_columns_to_phoenix_TASK-789", description = "Add missing 1000G columns to phoenix #TASK-789", + version = "2.3.0", domain = Migration.MigrationDomain.STORAGE, date = 20220614, deprecatedSince = "3.0.0" +) +public class AddMissingColumnsToPhoenix extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java new file mode 100644 index 00000000000..2c645980391 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_3_1/catalog/MoveCellbaseFromProjectInternalToProjectBugFix.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_3_1.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "bugfix_move_cellbase_from_project_internal_to_project_TASK-1100", + description = "Bugfix: Move cellbase from project.internal to project #TASK-1100", version = "2.3.1", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220620, + deprecatedSince = "3.0.0") + +public class MoveCellbaseFromProjectInternalToProjectBugFix extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddClinicalStatusType_607.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddClinicalStatusType_607.java new file mode 100644 index 00000000000..8f3886c50e8 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddClinicalStatusType_607.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_clinical_status_type_TASK-607", + description = "Automatically close cases depending on the new clinical status type #TASK-607", version = "2.4.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220610, + deprecatedSince = "3.0.0") +public class AddClinicalStatusType_607 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddPrivateDueDate_803.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddPrivateDueDate_803.java new file mode 100644 index 00000000000..6dcab05694a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/AddPrivateDueDate_803.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_private_dueDate_TASK-803", + description = "Add private _dueDate to ClinicalAnalysis #TASK-803", version = "2.4.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220530, + deprecatedSince = "3.0.0") +public class AddPrivateDueDate_803 extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/IndexPanelSource.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/IndexPanelSource.java new file mode 100644 index 00000000000..af93d04de7d --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_0/catalog/IndexPanelSource.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "index_panel_source_TASK-473", + description = "Index panel source #TASK-473", version = "2.4.0", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220530, + deprecatedSince = "3.0.0") +public class IndexPanelSource extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java new file mode 100644 index 00000000000..0759d4c7bf1 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_10/catalog/AddMissingEndDateOnFinishedJobs.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_10.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_missing_endDate_on_finished_jobs" , + description = "Add missing end date on finished jobs", + version = "2.4.10", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20221026, + deprecatedSince = "3.0.0" +) +public class AddMissingEndDateOnFinishedJobs extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/SynchronizeCatalogStorageStatuses.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/SynchronizeCatalogStorageStatuses.java new file mode 100644 index 00000000000..e2fb99933cf --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/SynchronizeCatalogStorageStatuses.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_11; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "synchronize_catalog_storage_statuses", + description = "Synchronize catalog internal statuses from storage, #TASK-1304", version = "2.4.11", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20221117, + deprecatedSince = "3.0.0") +public class SynchronizeCatalogStorageStatuses extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/catalog/SignatureFittingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/catalog/SignatureFittingsMigration.java new file mode 100644 index 00000000000..ec0b49baae3 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_11/catalog/SignatureFittingsMigration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_11.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "signature_fittings" , + description = "Replace fitting for fittings in Signature", + version = "2.4.11", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20221109, + deprecatedSince = "3.0.0" +) +public class SignatureFittingsMigration extends MigrationTool { + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/AddClinicalDiscussion.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/AddClinicalDiscussion.java new file mode 100644 index 00000000000..8e5825e3de7 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/AddClinicalDiscussion.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_2.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "add_clinical_discussion_TASK-1472", + description = "Add ClinicalDiscussion #TASK-1472", version = "2.4.2", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220727, + deprecatedSince = "3.0.0") +public class AddClinicalDiscussion extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/RecoverProbandSamplesInCases.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/RecoverProbandSamplesInCases.java new file mode 100644 index 00000000000..aba0267d65b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_2/catalog/RecoverProbandSamplesInCases.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_2.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "recover_proband_samples_in_cases_TASK-1470", + description = "Recover lost samples in clinical collection #TASK-1470", version = "2.4.2", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220725, + patch = 2, + deprecatedSince = "3.0.0") +public class RecoverProbandSamplesInCases extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/MigrateToClinicalAcmg.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/MigrateToClinicalAcmg.java new file mode 100644 index 00000000000..7951810b105 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/MigrateToClinicalAcmg.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_3.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "migrate_to_clinical_acmg_TASK-1194", + description = "Migrate to ClinicalAcmg #TASK-1194", version = "2.4.3", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220809, + deprecatedSince = "3.0.0") +public class MigrateToClinicalAcmg extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/ShortenIndexedFieldVariables.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/ShortenIndexedFieldVariables.java new file mode 100644 index 00000000000..2d4e2cb14a8 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_3/catalog/ShortenIndexedFieldVariables.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_3.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "shorten_indexed_field_variables-1386", + description = "Shorten indexed field variables #TASK-1386", version = "2.4.3", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220809, + deprecatedSince = "3.0.0") +public class ShortenIndexedFieldVariables extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java new file mode 100644 index 00000000000..e26df11c03c --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/AvoidCaseStatusIdNullMigration.java @@ -0,0 +1,18 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_4.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "avoidCaseStatusIdNull-1938", + description = "Avoid nullable status id in Clinical Analysis #TASK-1938", version = "2.4.4", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220914, + deprecatedSince = "3.0.0") +public class AvoidCaseStatusIdNullMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java new file mode 100644 index 00000000000..07a19a12fdb --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/catalog/RemoveBiotypeAllowedKeysMigration.java @@ -0,0 +1,17 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_4.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "remove_biotype_allowed_keys_TASK-1849", + description = "Remove biotype and consequence type allowed keys from variable sets #TASK-1849", version = "2.4.4", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.CATALOG, + date = 20220905, + deprecatedSince = "3.0.0") +public class RemoveBiotypeAllowedKeysMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/storage/InvalidateVariantArchivesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/storage/InvalidateVariantArchivesMigration.java new file mode 100644 index 00000000000..069e1a86e6e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_4_4/storage/InvalidateVariantArchivesMigration.java @@ -0,0 +1,16 @@ +package org.opencb.opencga.app.migrations.v2.v2_4_4.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "invalidate_variant_archives_migration-633", + description = "Invalidate variant archives from variant storage hadoop #TASK-633", version = "2.4.4", + language = Migration.MigrationLanguage.JAVA, + domain = Migration.MigrationDomain.STORAGE, + date = 20220825, + deprecatedSince = "3.0.0") +public class InvalidateVariantArchivesMigration extends StorageMigrationTool { + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_6_0/storage/AddCellbaseDataRelease.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_6_0/storage/AddCellbaseDataRelease.java new file mode 100644 index 00000000000..06f1d871a80 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_6_0/storage/AddCellbaseDataRelease.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_6_0.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "add_cellbase_data_release" , + description = "Add default cellbase data release if missing", + version = "2.6.0", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + patch = 2, + date = 20230104, + deprecatedSince = "3.0.0" +) +public class AddCellbaseDataRelease extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java new file mode 100644 index 00000000000..ce425e3434b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_7_0/catalog/CancerRoleAndModeOfInheritanceMigration.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_7_0.catalog; + + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "panel_roles_and_modesOfInheritance" , + description = "Change panel cancer.role and modeOfInheritance for cancer.roles[] and modesOfInheritance[]", + version = "2.7.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20230117, + deprecatedSince = "3.0.0" +) +public class CancerRoleAndModeOfInheritanceMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_0/catalog/CalculatePedigreeGraphMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_0/catalog/CalculatePedigreeGraphMigration.java new file mode 100644 index 00000000000..0be802202c4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_0/catalog/CalculatePedigreeGraphMigration.java @@ -0,0 +1,19 @@ +package org.opencb.opencga.app.migrations.v2.v2_8_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "calculate_pedigree_graph" , + description = "Calculate Pedigree Graph for all the families", + version = "2.8.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20230313, + deprecatedSince = "3.0.0" +) +public class CalculatePedigreeGraphMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_6/storage/MarkUnknownLargestVariantLength.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_6/storage/MarkUnknownLargestVariantLength.java new file mode 100644 index 00000000000..e3b093906c4 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_8_6/storage/MarkUnknownLargestVariantLength.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_8_6.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; + +@Migration(id = "mark_unknown_largest_variant_length" , + description = "Mark as unknown largest variant length", + version = "2.8.6", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + patch = 1, + date = 20230927, + deprecatedSince = "3.0.0" +) +public class MarkUnknownLargestVariantLength extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java new file mode 100644 index 00000000000..7dbd590c27e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2/v2_9_0/catalog/RenameCellBaseToken2ApiKey.java @@ -0,0 +1,20 @@ +package org.opencb.opencga.app.migrations.v2.v2_9_0.catalog; + +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +@Migration(id = "rename_cellbase_token_2_api_key" , + description = "Rename CellBase Token to ApiKey", + version = "2.9.0", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20230829, + deprecatedSince = "3.0.0" +) +public class RenameCellBaseToken2ApiKey extends MigrationTool { + + @Override + protected void run() throws Exception { + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java new file mode 100644 index 00000000000..ae73bcdc578 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_4/catalog/FixStatusIndexesMigration.java @@ -0,0 +1,46 @@ +package org.opencb.opencga.app.migrations.v2_12_4.catalog; + +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "fix_status_indexes" , + description = "Replace 'status.name' indexes for 'status.id'", + version = "2.12.4", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20240328 +) +public class FixStatusIndexesMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + Document statusIndex = new Document() + .append("status.name", 1) + .append("studyUid", 1); + dropIndex(Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION), statusIndex); + + Document internalStatusIndex = new Document() + .append("internal.status.name", 1) + .append("studyUid", 1); + dropIndex(Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, + OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION), + internalStatusIndex); + + catalogManager.installIndexes(token); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java new file mode 100644 index 00000000000..4efa260d965 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/AddAllelesColumnToPhoenix.java @@ -0,0 +1,27 @@ +package org.opencb.opencga.app.migrations.v2_12_5.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; + +@Migration(id="add_missing_column_to_phoenix_TASK-6005", description = "Add missing ALLELES column to phoenix #TASK-6005", + version = "2.12.5", domain = Migration.MigrationDomain.STORAGE, date = 20240510 +) +public class AddAllelesColumnToPhoenix extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + for (String project : getVariantStorageProjects(organizationId)) { + VariantStorageEngine engine = getVariantStorageEngineByProject(project); + if (engine.getStorageEngineId().equals("hadoop")) { + logger.info("Adding missing columns (if any) for project " + project); + // Using same class for both migrations + Class aClass = Class.forName("org.opencb.opencga.storage.hadoop.variant.migration.v2_3_0.AddMissingColumns"); + Runnable runnable = (Runnable) aClass + .getConstructor(Object.class) + .newInstance(engine); + runnable.run(); + } + } + } +} \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java new file mode 100644 index 00000000000..132e1357f1b --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_5/storage/DetectIllegalConcurrentFileLoadingsMigration.java @@ -0,0 +1,360 @@ +package org.opencb.opencga.app.migrations.v2_12_5.storage; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.metadata.CatalogStorageMetadataSynchronizer; +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.db.api.FileDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.common.UriUtils; +import org.opencb.opencga.core.models.common.IndexStatus; +import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileInternalVariantIndex; +import org.opencb.opencga.core.models.file.FileUpdateParams; +import org.opencb.opencga.core.models.file.VariantIndexStatus; +import org.opencb.opencga.core.models.sample.SampleUpdateParams; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.FileMetadata; +import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; +import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; +import org.opencb.opencga.storage.core.variant.adaptors.VariantField; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; + +import java.nio.file.Paths; +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; + +@Migration(id = "illegal_concurrent_file_loadings" , + description = "Detect illegal concurrent file loadings and fix them by setting 'status' to 'INVALID' or 'READY'", + version = "2.12.5", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + date = 20240424 +) +public class DetectIllegalConcurrentFileLoadingsMigration extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + for (String project : getVariantStorageProjects(organizationId)) { + VariantStorageEngine engine = getVariantStorageEngineByProject(project); + if (!engine.getStorageEngineId().equals("hadoop")) { + continue; + } + logger.info("Checking project '{}'", project); + for (String study : engine.getMetadataManager().getStudyNames()) { + checkStudy(engine, study); + } + } + } + + private void checkStudy(VariantStorageEngine engine, String study) throws StorageEngineException, CatalogException { + VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); + boolean repeatMigration = params.getBoolean("repeat-migration"); + + logger.info("Checking study '{}'", study); + int studyId = metadataManager.getStudyId(study); + + Set> fileSets = getFileWithSharedSamples(engine, studyId); + Set fileIds = fileSets.stream().flatMap(Collection::stream).collect(Collectors.toSet()); + + if (fileSets.isEmpty()) { + logger.info("No concurrent file loadings found in study '{}'", study); + return; + } + + Map fileTasks = new HashMap<>(); + for (TaskMetadata taskMetadata : metadataManager.taskIterable(studyId)) { + if (taskMetadata.getType() == TaskMetadata.Type.LOAD) { + for (Integer fileId : taskMetadata.getFileIds()) { + if (fileIds.contains(fileId)) { + TaskMetadata old = fileTasks.put(fileId, taskMetadata); + if (old != null) { + throw new IllegalStateException("File '" + fileId + "' is being loaded by more than one task." + + " Tasks '" + old.getName() + "'(" + old.getId() + ") and" + + " '" + taskMetadata.getName() + "'(" + taskMetadata.getId() + ")"); + } + } + } + } + } + + Set> fileSetsToInvalidate = new HashSet<>(); + Set affectedFiles = new HashSet<>(); + Set affectedSamples = new HashSet<>(); + for (Set fileSet : fileSets) { + // Check if any task from this file set overlaps in time + List tasks = new ArrayList<>(); + for (Integer fileId : fileSet) { + TaskMetadata task = fileTasks.get(fileId); + if (task != null) { + tasks.add(task); + } + } + if (tasks.size() > 1) { + logger.info("Found {} tasks loading files {}", tasks.size(), fileSet); + for (int i = 0; i < tasks.size(); i++) { + TaskMetadata task1 = tasks.get(i); + Date task1start = task1.getStatus().firstKey(); + Date task1end = task1.getStatus().lastKey(); + for (int f = i + 1; f < tasks.size(); f++) { + TaskMetadata task2 = tasks.get(f); + Date task2start = task2.getStatus().firstKey(); + Date task2end = task2.getStatus().lastKey(); + if (task1start.before(task2end) && task1end.after(task2start)) { + fileSetsToInvalidate.add(fileSet); + affectedFiles.addAll(task1.getFileIds()); + + List task1Files = task1.getFileIds().stream().map(fileId -> "'" + metadataManager.getFileName(studyId, fileId) + "'(" + fileId + ")").collect(Collectors.toList()); + List task2Files = task2.getFileIds().stream().map(fileId -> "'" + metadataManager.getFileName(studyId, fileId) + "'(" + fileId + ")").collect(Collectors.toList()); + + logger.info("Tasks '{}'({}) and '{}'({}) overlap in time", task1.getName(), task1.getId(), task2.getName(), task2.getId()); + logger.info("Task1: {} - {} loading files {}", task1start, task1end, task1Files); + logger.info("Task2: {} - {} loading files {}", task2start, task2end, task2Files); + Set task1Samples = task1.getFileIds().stream().flatMap(file -> metadataManager.getSampleIdsFromFileId(studyId, file).stream()).collect(Collectors.toSet()); + Set task2Samples = task2.getFileIds().stream().flatMap(file -> metadataManager.getSampleIdsFromFileId(studyId, file).stream()).collect(Collectors.toSet()); + for (Integer task1Sample : task1Samples) { + if (task2Samples.contains(task1Sample)) { + String sampleName = metadataManager.getSampleName(studyId, task1Sample); + affectedSamples.add(task1Sample); + logger.info("Sample '{}'({}) is shared between tasks", sampleName, task1Sample); + } + } + } + } + } + } + + Set invalidFiles = new HashSet<>(); + List invalidSampleIndexes = new ArrayList<>(); + for (Integer sampleId : affectedSamples) { + String sampleName = metadataManager.getSampleName(studyId, sampleId); + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, sampleId); + if (!sampleMetadata.isMultiFileSample()) { + logger.warn("Sample '{}'({}) is not a multi-file sample but has multiple files loaded", sampleName, sampleId); + for (Integer file : sampleMetadata.getFiles()) { + if (metadataManager.isFileIndexed(studyId, file)) { + invalidFiles.add(file); + logger.info(" - Invalidating file '{}'({}). Must be deleted and then indexed.", + metadataManager.getFileName(studyId, file), file); + } + } + } else if (sampleMetadata.getSampleIndexStatus(Optional.of(sampleMetadata.getSampleIndexVersion()).orElse(-1)) == TaskMetadata.Status.READY) { + for (Integer fileId : sampleMetadata.getFiles()) { + if (affectedFiles.contains(fileId)) { + FileMetadata fileMetadata = metadataManager.getFileMetadata(studyId, fileId); + String fileName = fileMetadata.getName(); + + long actualCount = engine.get(new VariantQuery().study(study).sample(sampleName).file(fileName), + new QueryOptions(QueryOptions.INCLUDE, VariantField.ID).append(QueryOptions.COUNT, true)) + .getNumMatches(); + + File catalogFile = catalogManager.getFileManager().search(study, + new Query(FileDBAdaptor.QueryParams.URI.key(), UriUtils.createUriSafe(fileMetadata.getPath())), + new QueryOptions(), token).first(); + if (catalogFile == null) { + logger.warn("File '{}'({}) not found in catalog", fileName, fileId); + logger.warn("Sample '{}'({}) invalidated, as file '{}'({}) is not found in catalog", sampleName, sampleId, fileName, fileId); + logger.info(" - Invalidating sample index for sample '{}'({})", sampleName, sampleId); + invalidSampleIndexes.add(sampleId); + continue; + } + if (fileMetadata.getSamples().size() == 1 && catalogFile.getSampleIds().size() == 1) { + long expectedCount = 0; + for (Map.Entry entry : catalogFile.getQualityControl().getVariant() + .getVariantSetMetrics().getGenotypeCount().entrySet()) { + if (GenotypeClass.MAIN_ALT.test(entry.getKey())) { + expectedCount += entry.getValue(); + } + } + if (expectedCount == 0) { + expectedCount = catalogFile.getQualityControl().getVariant().getVariantSetMetrics().getVariantCount(); + } + if (expectedCount != actualCount) { + invalidSampleIndexes.add(sampleId); + logger.warn("Sample '{}'({}) was expected to have {} variants in the sample index of file '{}'({}) but has {}", + sampleName, sampleId, expectedCount, fileName, fileId, actualCount); + logger.info(" - Invalidating sample index for sample '{}'({})", sampleName, sampleId); + } + } else { + Map pipelineResult = (Map) catalogFile.getAttributes().get("storagePipelineResult"); + long loadedVariants = ((Number) ((Map) pipelineResult.get("loadStats")).get("loadedVariants")).longValue(); + if (loadedVariants != actualCount) { + invalidSampleIndexes.add(sampleId); + logger.warn("Sample '{}'({}) was expected to have {} variants in the sample index but has {}", + sampleName, sampleId, loadedVariants, actualCount); + logger.info(" - Invalidating sample index for sample '{}'({})", sampleName, sampleId); + } + } + } + } + } else { + invalidSampleIndexes.add(sampleId); + } + } + + if (params.getBoolean("dry-run")) { + if (invalidFiles.isEmpty() && invalidSampleIndexes.isEmpty()) { + logger.info("Dry-run mode. No files or samples to invalidate"); + } else { + logger.info("Dry-run mode. Skipping invalidation of files and samples"); + + Set invalidSamples = new HashSet<>(); + for (Integer fileId : invalidFiles) { + invalidSamples.addAll(metadataManager.getSampleIdsFromFileId(studyId, fileId)); + } + + logger.info("Affected files: {}", invalidFiles); + logger.info("Affected samples: {}", invalidSamples); + logger.info("Affected sample indexes: {}", invalidSampleIndexes); + } + } else { + ObjectMap event = new ObjectMap() + .append("patch", getAnnotation().patch()) + .append("description", "affected_invalid_sample") + .append("dateStr", TimeUtils.getTime()) + .append("date", Date.from(Instant.now())); + for (Integer sampleId : invalidSampleIndexes) { + invalidateSecondarySampleIndex(study, sampleId, event, metadataManager, studyId, repeatMigration); + } + Set invalidSamples = new HashSet<>(); + for (Integer fileId : invalidFiles) { + invalidateFileIndex(study, fileId, event, metadataManager, studyId, invalidSamples, repeatMigration); + } + for (Integer sampleId : invalidSamples) { + invalidateSampleIndex(study, sampleId, event, metadataManager, studyId, repeatMigration); + } + + Set allSampleIds = new HashSet<>(); + allSampleIds.addAll(affectedSamples); + allSampleIds.addAll(invalidSamples); + List allSampleNames = allSampleIds.stream() + .map(sampleId -> metadataManager.getSampleName(studyId, sampleId)) + .collect(Collectors.toList()); + + if (!allSampleNames.isEmpty()) { + new CatalogStorageMetadataSynchronizer(catalogManager, metadataManager) + .synchronizeCatalogSamplesFromStorage(study, allSampleNames, token); + } + + } + } + } + + private void invalidateFileIndex(String study, Integer fileId, ObjectMap event, VariantStorageMetadataManager metadataManager, int studyId, Set invalidSamples, boolean repeatMigration) throws StorageEngineException, CatalogException { + ObjectMap thisEvent = new ObjectMap(event); + String filePath = metadataManager.updateFileMetadata(studyId, fileId, fileMetadata -> { + invalidSamples.addAll(fileMetadata.getSamples()); + if (fileMetadata.getAttributes().containsKey("TASK-6078") && !repeatMigration) { + logger.info("File '{}'({}) already has the attribute 'TASK-6078'. Skip", + fileMetadata.getName(), fileMetadata.getId()); + } else { + Map oldStatus = new HashMap<>(fileMetadata.getStatus()); + fileMetadata.setIndexStatus(TaskMetadata.Status.INVALID); + fileMetadata.getAttributes().put("TASK-6078", thisEvent.append("oldStatus", oldStatus)); + } + }).getPath(); + String fileUri = Paths.get(filePath).toUri().toString(); + Query query = new Query(FileDBAdaptor.QueryParams.URI.key(), fileUri); + File file = catalogManager.getFileManager() + .search(study, query, new QueryOptions(), token) + .first(); + catalogManager.getFileManager().update(study, file.getId(), + new FileUpdateParams().setAttributes(new ObjectMap("TASK-6078", thisEvent)), QueryOptions.empty(), token); + catalogManager.getFileManager().updateFileInternalVariantIndex(study, file, new FileInternalVariantIndex() + .setStatus(new VariantIndexStatus(IndexStatus.INVALID, "Invalid status - TASK-6078 - affected_invalid_sample - " + + "File must be deleted and then indexed")), token); + } + + private void invalidateSecondarySampleIndex(String study, Integer sampleId, ObjectMap event, VariantStorageMetadataManager metadataManager, int studyId, boolean repeatMigration) throws StorageEngineException, CatalogException { + ObjectMap thisEvent = new ObjectMap(event); + String sampleName = metadataManager.updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + if (sampleMetadata.getAttributes().containsKey("TASK-6078") && !repeatMigration) { + logger.info("Sample '{}'({}) already has the attribute 'TASK-6078'. Skip", + sampleMetadata.getName(), sampleMetadata.getId()); + } else { + Map oldStatus = new HashMap<>(sampleMetadata.getStatus()); + Map oldAttributes = new HashMap<>(sampleMetadata.getAttributes()); + + for (Integer v : sampleMetadata.getSampleIndexVersions()) { + sampleMetadata.setSampleIndexStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { + sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getFamilyIndexVersions()) { + sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.NONE, v); + } + + sampleMetadata.setIndexStatus(TaskMetadata.Status.INVALID); + thisEvent.append("oldStatus", oldStatus); + thisEvent.append("oldAttributes", oldAttributes); + thisEvent.append("newStatus", sampleMetadata.getStatus()); + sampleMetadata.getAttributes().put("TASK-6078", thisEvent); + } + }).getName(); + catalogManager.getSampleManager().update(study, sampleName, + new SampleUpdateParams().setAttributes(new ObjectMap("TASK-6078", thisEvent)), QueryOptions.empty(), token); + } + + private void invalidateSampleIndex(String study, Integer sampleId, ObjectMap event, VariantStorageMetadataManager metadataManager, int studyId, boolean repeatMigration) throws StorageEngineException, CatalogException { + ObjectMap thisEvent = new ObjectMap(event); + String sampleName = metadataManager.updateSampleMetadata(studyId, sampleId, sampleMetadata -> { + if (sampleMetadata.getAttributes().containsKey("TASK-6078") && !repeatMigration) { + logger.info("Sample '{}'({}) already has the attribute 'TASK-6078'. Skip", + sampleMetadata.getName(), sampleMetadata.getId()); + } else { + Map oldStatus = new HashMap<>(sampleMetadata.getStatus()); + Map oldAttributes = new HashMap<>(sampleMetadata.getAttributes()); + sampleMetadata.setIndexStatus(TaskMetadata.Status.INVALID); + + for (Integer v : sampleMetadata.getSampleIndexVersions()) { + sampleMetadata.setSampleIndexStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getSampleIndexAnnotationVersions()) { + sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.NONE, v); + } + for (Integer v : sampleMetadata.getFamilyIndexVersions()) { + sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.NONE, v); + } + thisEvent.append("oldStatus", oldStatus); + thisEvent.append("oldAttributes", oldAttributes); + thisEvent.append("newStatus", sampleMetadata.getStatus()); + sampleMetadata.getAttributes().put("TASK-6078", thisEvent); + } + }).getName(); + catalogManager.getSampleManager().update(study, sampleName, + new SampleUpdateParams().setAttributes(new ObjectMap("TASK-6078", thisEvent)), QueryOptions.empty(), token); + } + + private Set> getFileWithSharedSamples(VariantStorageEngine engine, int studyId) throws StorageEngineException { + VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); + + Set> fileSets = new HashSet<>(); + // Check if there are any sample with more than one file + for (SampleMetadata sampleMetadata : metadataManager.sampleMetadataIterable(studyId)) { + if (sampleMetadata.getFiles().size() > 1) { + if (sampleMetadata.getSplitData() == VariantStorageEngine.SplitData.CHROMOSOME || sampleMetadata.getSplitData() == VariantStorageEngine.SplitData.REGION) { + logger.debug("Sample '{}' is split by chromosome or region. Skip", sampleMetadata.getName()); + continue; + } + ArrayList sampleFileIds = new ArrayList<>(sampleMetadata.getFiles()); + + sampleFileIds.removeIf(fileId -> !metadataManager.isFileIndexed(studyId, fileId)); + + if (sampleFileIds.size() > 1) { + logger.info("Sample '{}' has more than one indexed file with split data '{}'", sampleMetadata.getName(), sampleMetadata.getSplitData()); + + fileSets.add(new HashSet<>(sampleFileIds)); + } + } + } + return fileSets; + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_0_0/OrganizationMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java similarity index 70% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_0_0/OrganizationMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java index 0b9982c6f71..9b226c0ea1a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_0_0/OrganizationMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java @@ -1,12 +1,12 @@ -package org.opencb.opencga.app.migrations.v3_0_0; +package org.opencb.opencga.app.migrations.v3.v3_0_0; import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationException; import org.opencb.opencga.catalog.migration.MigrationTool; import org.opencb.opencga.core.config.Configuration; @Migration(id = "add_organizations", description = "Add new Organization layer #TASK-4389", version = "3.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20231212) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20231212, + deprecatedSince = "3.1.0") public class OrganizationMigration extends MigrationTool { public OrganizationMigration(Configuration configuration, String adminPassword, String userId) { @@ -14,7 +14,6 @@ public OrganizationMigration(Configuration configuration, String adminPassword, @Override protected void run() throws Exception { - throw MigrationException.outdatedMigration(getAnnotation(), "3.1.0"); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_1_0/AuthOriginSimplificationMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_1_0/AuthOriginSimplificationMigration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java index e913941bafd..e1b331fa987 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_1_0/AuthOriginSimplificationMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v3_1_0; +package org.opencb.opencga.app.migrations.v3.v3_1_0; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java new file mode 100644 index 00000000000..c3e49a2d85e --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java @@ -0,0 +1,51 @@ +package org.opencb.opencga.app.migrations.v3.v3_1_0; + +import com.mongodb.MongoNamespace; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.RenameCollectionOptions; +import com.mongodb.client.model.Updates; +import com.mongodb.client.result.UpdateResult; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.commons.datastore.mongodb.MongoDataStore; +import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.notes.Note; + +@Migration(id = "migrate_notes", description = "Migrate notes #TASK-5836", version = "3.1.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240315) +public class NoteMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + // First migrate to add the new values + MongoCollection collection = getMongoCollection(organizationId, "notes"); + Bson query = Filters.exists(NoteDBAdaptor.QueryParams.STUDY_UID.key(), false); + Bson update = Updates.combine( + Updates.set(NoteDBAdaptor.QueryParams.STUDY_UID.key(), -1L), + Updates.set(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.ORGANIZATION.name()), + Updates.set(NoteDBAdaptor.QueryParams.STUDY.key(), ""), + Updates.set(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PRIVATE.name()) + ); + UpdateResult updateResult = collection.updateMany(query, update); + if (updateResult.getModifiedCount() == 0) { + logger.info("Note data model could not be updated. Notes found in organization '{}': {}", organizationId, updateResult.getMatchedCount()); + } + + // Rename Note collection + RenameCollectionOptions options = new RenameCollectionOptions().dropTarget(true); + // Rename collection + String databaseName = dbAdaptorFactory.getMongoDataStore(organizationId).getDatabaseName(); + logger.info("Renaming notes collection for organization '{}' -> Database: '{}'", organizationId, databaseName); + MongoDataStore mongoDataStore = dbAdaptorFactory.getMongoDataStore(organizationId); + mongoDataStore.getDb().getCollection("notes").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION), options); + mongoDataStore.getDb().getCollection("notes_archive").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION), options); + mongoDataStore.getDb().getCollection("notes_deleted").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.DELETED_NOTE_COLLECTION), options); + + catalogManager.installIndexes(organizationId, token); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java new file mode 100644 index 00000000000..b7d79b73914 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java @@ -0,0 +1,41 @@ +package org.opencb.opencga.app.migrations.v3.v3_1_0; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.core.common.TimeUtils; + +@Migration(id = "addFailedLoginAttemptsMigration", description = "Add failedAttempts to User #TASK-6013", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240419) +public class UserBanMigration extends MigrationTool { + + @Override + protected void run() throws Exception { + String lastModified = TimeUtils.getTime(); + migrateCollection(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, + Filters.exists("internal.registrationDate", false), + Projections.include("_id", "internal", "account"), + (document, bulk) -> { + Document internal = document.get("internal", Document.class); + Document account = document.get("account", Document.class); + internal.put("failedAttempts", 0); + internal.put("registrationDate", account.get("creationDate")); + internal.put("lastModified", lastModified); + account.put("expirationDate", Constants.DEFAULT_USER_EXPIRATION_DATE); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), + Updates.combine( + Updates.set("internal", internal), + Updates.set("account", account)) + ) + ); + }); + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java new file mode 100644 index 00000000000..8e196973bba --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java @@ -0,0 +1,34 @@ +package org.opencb.opencga.app.migrations.v3.v3_2_0; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "add_jobParentId_scheduledStartTime", + description = "Add 'jobParentId' and 'scheduledStartTime' to existing jobs #TASK-6171 #TASK-6089", version = "3.2.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240506) +public class AddNewJobFieldsMigration extends MigrationTool { + + + @Override + protected void run() throws Exception { + for (String jobCollection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { + MongoCollection mongoCollection = getMongoCollection(jobCollection); + Bson query = Filters.exists("parentId", false); + Bson update = Updates.combine( + Updates.set("parentId", ""), + Updates.set("scheduledStartTime", "") + ); + mongoCollection.updateMany(query, update); + } + } + +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_1_0/NoteMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_1_0/NoteMigration.java deleted file mode 100644 index a842905fc70..00000000000 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3_1_0/NoteMigration.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.opencb.opencga.app.migrations.v3_1_0; - -import com.mongodb.MongoNamespace; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.RenameCollectionOptions; -import com.mongodb.client.model.Updates; -import com.mongodb.client.result.UpdateResult; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.io.IOManagerFactory; -import org.opencb.opencga.catalog.managers.CatalogManager; -import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.notes.Note; - -import java.util.Collections; -import java.util.List; - -@Migration(id = "migrate_notes", description = "Migrate notes #TASK-5836", version = "3.1.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240315) -public class NoteMigration extends MigrationTool { - - @Override - protected void run() throws Exception { - IOManagerFactory ioManagerFactory = new IOManagerFactory(); - dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory); - // First migrate to add the new values - MongoCollection collection = getMongoCollection(ParamConstants.ADMIN_ORGANIZATION, "notes"); - Bson query = Filters.exists(NoteDBAdaptor.QueryParams.STUDY_UID.key(), false); - Bson update = Updates.combine( - Updates.set(NoteDBAdaptor.QueryParams.STUDY_UID.key(), -1L), - Updates.set(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.ORGANIZATION.name()), - Updates.set(NoteDBAdaptor.QueryParams.STUDY.key(), ""), - Updates.set(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PRIVATE.name()) - ); - UpdateResult updateResult = collection.updateMany(query, update); - if (updateResult.getModifiedCount() == 0) { - // Check there are at least 2 organizations present - logger.info("Note data model could not be updated. Detected organizations are: {}", StringUtils.join(dbAdaptorFactory.getOrganizationIds(), ",")); - if (dbAdaptorFactory.getOrganizationIds().size() == 2) { - logger.info("Nothing to migrate"); - return; - } else { - throw new CatalogDBException("Notes could not be found to migrate."); - } - } - renameNoteCollection(Collections.singletonList(ParamConstants.ADMIN_ORGANIZATION)); - - dbAdaptorFactory.close(); - dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory); - // We run it a second time because the first time it will only rename the "opencga" org as OpenCGA will not be able to know - // which other organizations are present in the installation (trying to fetch the information from "note" instead of old "notes") - List organizationIds = dbAdaptorFactory.getOrganizationIds(); - organizationIds.remove(ParamConstants.ADMIN_ORGANIZATION); - renameNoteCollection(organizationIds); - - // Reload catalog manager to install missing indexes - catalogManager = new CatalogManager(configuration); - catalogManager.installIndexes(token); - } - - private void renameNoteCollection(List organizationIds) throws CatalogDBException { - RenameCollectionOptions options = new RenameCollectionOptions().dropTarget(true); - // Rename collection - for (String organizationId : organizationIds) { - String databaseName = dbAdaptorFactory.getMongoDataStore(organizationId).getDatabaseName(); - logger.info("Renaming notes collection for organization '{}' -> Database: '{}'", organizationId, databaseName); - MongoDataStore mongoDataStore = dbAdaptorFactory.getMongoDataStore(organizationId); - mongoDataStore.getDb().getCollection("notes").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION), options); - mongoDataStore.getDb().getCollection("notes_archive").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION), options); - mongoDataStore.getDb().getCollection("notes_deleted").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.DELETED_NOTE_COLLECTION), options); - } - } - -} diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index d15b4768e59..7457317446b 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml @@ -200,18 +200,6 @@ com.microsoft.azure azure-client-runtime - - com.microsoft.azure - azure - - - io.fabric8 - kubernetes-client - - - com.microsoft.azure - azure-batch - io.jsonwebtoken jjwt-jackson @@ -247,6 +235,12 @@ template.zip + + ../opencga-core/src/test/resources + + log4j2-test.xml + + diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java index 345ac2977b7..d93cf48c4a7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AuthenticationManager.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import javax.crypto.spec.SecretKeySpec; +import java.io.Closeable; import java.security.Key; import java.util.Collections; import java.util.Date; @@ -37,7 +38,7 @@ /** * @author Jacobo Coll <jacobo167@gmail.com> */ -public abstract class AuthenticationManager { +public abstract class AuthenticationManager implements Closeable { protected JwtManager jwtManager; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java index b0f525af072..f104f8fdf93 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/AzureADAuthenticationManager.java @@ -146,6 +146,15 @@ public AzureADAuthenticationManager(AuthenticationOrigin authenticationOrigin) t Configurator.setLevel("com.microsoft.aad.adal4j.AuthenticationAuthority", Level.WARN); } + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (authenticationOrigin.getType() != AuthenticationOrigin.AuthenticationType.AzureAD) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationOrigin.AuthenticationType.AzureAD + + "' but received '" + authenticationOrigin.getType() + "'."); + } + AzureADAuthenticationManager azureADAuthenticationManager = new AzureADAuthenticationManager(authenticationOrigin); + azureADAuthenticationManager.close(); + } + private OIDCProviderMetadata getProviderMetadata(String host) throws IOException, ParseException { URL providerConfigurationURL = new URL(host); InputStream stream = providerConfigurationURL.openStream(); @@ -420,4 +429,8 @@ public String createNonExpiringToken(String organizationId, String userId, Map claims, long expiration) { - return jwtManager.createJWTToken(organizationId, userId, claims, expiration); + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.OPENCGA, userId, claims, expiration); } @Override public String createNonExpiringToken(String organizationId, String userId, Map claims) { - return jwtManager.createJWTToken(organizationId, userId, claims, 0L); + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.OPENCGA, userId, claims, 0L); } @Override @@ -131,7 +142,7 @@ public OpenCGAResult resetPassword(String organizationId, String userId) throws String mailHost = this.emailConfig.getHost(); String mailPort = this.emailConfig.getPort(); try { - MailUtils.sendResetPasswordMail(email, newPassword, mailUser, mailPassword, mailHost, mailPort); + MailUtils.sendResetPasswordMail(email, newPassword, mailUser, mailPassword, mailHost, mailPort, userId); result = dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId).resetPassword(userId, email, newPassword); } catch (Exception e) { throw new CatalogException("Email could not be sent.", e); @@ -145,4 +156,8 @@ public static AuthenticationOrigin createOpencgaAuthenticationOrigin() { .setId(CatalogAuthenticationManager.OPENCGA) .setType(AuthenticationOrigin.AuthenticationType.OPENCGA); } + + @Override + public void close() { + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java index 8cbb28decca..fd0462e3a98 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/JwtManager.java @@ -18,6 +18,7 @@ import io.jsonwebtoken.*; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.models.JwtPayload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +30,8 @@ import java.util.List; import java.util.Map; +import static org.opencb.opencga.core.models.JwtPayload.AUTH_ORIGIN; + public class JwtManager { private SignatureAlgorithm algorithm; @@ -84,13 +87,18 @@ public JwtManager setPublicKey(Key publicKey) { return this; } - public String createJWTToken(String organizationId, String userId, Map claims, long expiration) { + public String createJWTToken(String organizationId, AuthenticationOrigin.AuthenticationType type, String userId, + Map claims, long expiration) { long currentTime = System.currentTimeMillis(); JwtBuilder jwtBuilder = Jwts.builder(); if (claims != null && !claims.isEmpty()) { jwtBuilder.setClaims(claims); } + if (type != null) { + jwtBuilder.addClaims(Collections.singletonMap(AUTH_ORIGIN, type)); + } + jwtBuilder.setSubject(userId) .setAudience(organizationId) .setIssuer("OpenCGA") @@ -115,14 +123,24 @@ public void validateToken(String token, Key publicKey) throws CatalogAuthenticat public JwtPayload getPayload(String token) throws CatalogAuthenticationException { Claims body = parseClaims(token, publicKey).getBody(); - return new JwtPayload(body.getSubject(), body.getAudience(), body.getIssuer(), body.getIssuedAt(), body.getExpiration(), token); + return new JwtPayload(body.getSubject(), body.getAudience(), getAuthOrigin(body), body.getIssuer(), body.getIssuedAt(), + body.getExpiration(), token); } public JwtPayload getPayload(String token, Key publicKey) throws CatalogAuthenticationException { Claims body = parseClaims(token, publicKey).getBody(); - return new JwtPayload(body.getSubject(), body.getAudience(), body.getIssuer(), body.getIssuedAt(), body.getExpiration(), token); + return new JwtPayload(body.getSubject(), body.getAudience(), getAuthOrigin(body), body.getIssuer(), body.getIssuedAt(), + body.getExpiration(), token); } + private AuthenticationOrigin.AuthenticationType getAuthOrigin(Claims claims) { + String o = claims.get(AUTH_ORIGIN, String.class); + if (o != null) { + return AuthenticationOrigin.AuthenticationType.valueOf(o); + } else { + return null; + } + } public String getAudience(String token) throws CatalogAuthenticationException { return getAudience(token, this.publicKey); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java index 516a54702c2..35fe615d161 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/LDAPAuthenticationManager.java @@ -24,8 +24,10 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.organizations.TokenConfiguration; import org.opencb.opencga.core.models.user.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -124,6 +126,24 @@ protected static String envToStringRedacted(Hashtable env) { return string; } + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (authenticationOrigin.getType() != AuthenticationType.LDAP) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationType.LDAP + "' but received '" + + authenticationOrigin.getType() + "'."); + } + ParamUtils.checkParameter(authenticationOrigin.getHost(), AuthenticationType.LDAP + " host."); + + TokenConfiguration defaultTokenConfig = TokenConfiguration.init(); + LDAPAuthenticationManager ldapAuthenticationManager = new LDAPAuthenticationManager(authenticationOrigin, + defaultTokenConfig.getAlgorithm(), defaultTokenConfig.getSecretKey(), defaultTokenConfig.getExpiration()); + DirContext dirContext = ldapAuthenticationManager.getDirContext(ldapAuthenticationManager.getDefaultEnv(), 1); + if (dirContext == null) { + throw new CatalogException("LDAP: Could not connect to the LDAP server using the provided configuration."); + } + ldapAuthenticationManager.closeDirContextAndSuppress(dirContext, new Exception()); + ldapAuthenticationManager.close(); + } + @Override public AuthenticationResponse authenticate(String organizationId, String userId, String password) throws CatalogAuthenticationException { @@ -213,12 +233,12 @@ public void newPassword(String organizationId, String userId, String newPassword @Override public String createToken(String organizationId, String userId, Map claims, long expiration) { - return jwtManager.createJWTToken(organizationId, userId, claims, expiration); + return jwtManager.createJWTToken(organizationId, AuthenticationType.LDAP, userId, claims, expiration); } @Override public String createNonExpiringToken(String organizationId, String userId, Map claims) { - return jwtManager.createJWTToken(organizationId, userId, claims, 0L); + return jwtManager.createJWTToken(organizationId, AuthenticationType.LDAP, userId, claims, 0L); } /* Private methods */ @@ -503,4 +523,9 @@ private String takeString(ObjectMap objectMap, String key, String defaultValue) objectMap.remove(key); return value; } + + @Override + public void close() { + executorService.shutdown(); + } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/SSOAuthenticationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/SSOAuthenticationManager.java new file mode 100644 index 00000000000..e676029e413 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/SSOAuthenticationManager.java @@ -0,0 +1,87 @@ +package org.opencb.opencga.catalog.auth.authentication; + +import io.jsonwebtoken.SignatureAlgorithm; +import org.apache.commons.lang3.NotImplementedException; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.models.user.AuthenticationResponse; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.security.Key; +import java.util.List; +import java.util.Map; + +public class SSOAuthenticationManager extends AuthenticationManager { + + public SSOAuthenticationManager(String algorithm, String secretKeyString, long expiration) { + super(expiration); + + SignatureAlgorithm signatureAlgorithm = SignatureAlgorithm.valueOf(algorithm); + Key secretKey = this.converStringToKeyObject(secretKeyString, signatureAlgorithm.getJcaName()); + this.jwtManager = new JwtManager(signatureAlgorithm.getValue(), secretKey); + + this.logger = LoggerFactory.getLogger(SSOAuthenticationManager.class); + } + + public static void validateAuthenticationOriginConfiguration(AuthenticationOrigin authenticationOrigin) throws CatalogException { + if (authenticationOrigin.getType() != AuthenticationOrigin.AuthenticationType.SSO) { + throw new CatalogException("Unknown authentication type. Expected type '" + AuthenticationOrigin.AuthenticationType.SSO + + "' but received '" + authenticationOrigin.getType() + "'."); + } + } + + @Override + public AuthenticationResponse authenticate(String organizationId, String userId, String password) + throws CatalogAuthenticationException { + throw new NotImplementedException("Authentication should be done through SSO"); + } + + @Override + public List getUsersFromRemoteGroup(String group) throws CatalogException { + throw new NotImplementedException("Operation not implemented"); + } + + @Override + public List getRemoteUserInformation(List userStringList) throws CatalogException { + throw new NotImplementedException("Operation not implemented"); + } + + @Override + public List getRemoteGroups(String token) throws CatalogException { + throw new NotImplementedException("Operation not implemented"); + } + + @Override + public void changePassword(String organizationId, String userId, String oldPassword, String newPassword) throws CatalogException { + throw new NotImplementedException("Change password should be done through SSO"); + } + + @Override + public OpenCGAResult resetPassword(String organizationId, String userId) throws CatalogException { + throw new NotImplementedException("Reset password should be done through SSO"); + } + + @Override + public void newPassword(String organizationId, String userId, String newPassword) throws CatalogException { + throw new NotImplementedException("Setting a new password should be done through SSO"); + } + + @Override + public String createToken(String organizationId, String userId, Map claims, long expiration) { + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.SSO, userId, claims, expiration); + } + + @Override + public String createNonExpiringToken(String organizationId, String userId, Map claims) { + return jwtManager.createJWTToken(organizationId, AuthenticationOrigin.AuthenticationType.SSO, userId, claims, 0L); + } + + @Override + public void close() throws IOException { + + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java index d1b1246e468..7172401fda7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authentication/azure/AuthenticationFactory.java @@ -2,16 +2,14 @@ import org.apache.commons.collections4.CollectionUtils; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.auth.authentication.AuthenticationManager; -import org.opencb.opencga.catalog.auth.authentication.AzureADAuthenticationManager; -import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; -import org.opencb.opencga.catalog.auth.authentication.LDAPAuthenticationManager; +import org.opencb.opencga.catalog.auth.authentication.*; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.OrganizationManager; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; -import org.opencb.opencga.core.config.Email; +import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.User; @@ -19,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,16 +29,15 @@ public final class AuthenticationFactory { private final Map> authenticationManagerMap; private final Logger logger = LoggerFactory.getLogger(AuthenticationFactory.class); private final DBAdaptorFactory catalogDBAdaptorFactory; + private final Configuration configuration; - public AuthenticationFactory(DBAdaptorFactory catalogDBAdaptorFactory) { + public AuthenticationFactory(DBAdaptorFactory catalogDBAdaptorFactory, Configuration configuration) { this.catalogDBAdaptorFactory = catalogDBAdaptorFactory; + this.configuration = configuration; authenticationManagerMap = new ConcurrentHashMap<>(); } public void configureOrganizationAuthenticationManager(Organization organization) throws CatalogException { - // TODO: Pass proper email values - Email email = new Email(); - Map tmpAuthenticationManagerMap = new HashMap<>(); long expiration = organization.getConfiguration().getToken().getExpiration(); @@ -60,10 +58,15 @@ public void configureOrganizationAuthenticationManager(Organization organization break; case OPENCGA: CatalogAuthenticationManager catalogAuthenticationManager = - new CatalogAuthenticationManager(catalogDBAdaptorFactory, email, algorithm, secretKey, expiration); + new CatalogAuthenticationManager(catalogDBAdaptorFactory, configuration.getEmail(), algorithm, + secretKey, expiration); tmpAuthenticationManagerMap.put(CatalogAuthenticationManager.INTERNAL, catalogAuthenticationManager); tmpAuthenticationManagerMap.put(CatalogAuthenticationManager.OPENCGA, catalogAuthenticationManager); break; + case SSO: + tmpAuthenticationManagerMap.put(authOrigin.getId(), new SSOAuthenticationManager(algorithm, secretKey, + expiration)); + break; default: logger.warn("Unexpected authentication origin type '{}' for id '{}' found in organization '{}'. " + "Authentication origin will be ignored.", authOrigin.getType(), organization.getId(), @@ -76,6 +79,18 @@ public void configureOrganizationAuthenticationManager(Organization organization if (tmpAuthenticationManagerMap.isEmpty()) { throw new CatalogException("No authentication origin found for organization '" + organization.getId() + "'"); } + if (authenticationManagerMap.containsKey(organization.getId())) { + for (AuthenticationManager authenticationManager : authenticationManagerMap.get(organization.getId()).values()) { + try { + logger.info("Closing previous authentication manager for organization '{}'", organization.getId()); + authenticationManager.close(); + } catch (IOException e) { + throw new CatalogException("Unable to close previous authentication manager for organization '" + organization.getId() + + "'.", e); + } + } + logger.info("Reloading new set of AuthenticationManagers for organization '{}'", organization.getId()); + } authenticationManagerMap.put(organization.getId(), tmpAuthenticationManagerMap); } @@ -147,4 +162,25 @@ public AuthenticationManager getOrganizationAuthenticationManager(String organiz return organizationAuthenticationManagers.get(authOriginId); } + public void validateAuthenticationOrigin(AuthenticationOrigin authenticationOrigin) throws CatalogException { + ParamUtils.checkParameter(authenticationOrigin.getId(), "authentication origin id"); + ParamUtils.checkObj(authenticationOrigin.getType(), "authentication origin type"); + switch (authenticationOrigin.getType()) { + case OPENCGA: + CatalogAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + case LDAP: + LDAPAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + case AzureAD: + AzureADAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + case SSO: + SSOAuthenticationManager.validateAuthenticationOriginConfiguration(authenticationOrigin); + break; + default: + throw new CatalogException("Unknown authentication origin type '" + authenticationOrigin.getType() + "'"); + } + } + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java index b58c117d94f..7022be4a0e7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java @@ -19,6 +19,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; @@ -79,7 +80,34 @@ public void checkCanEditProject(String organizationId, long projectId, String us if (isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { return; } - throw new CatalogAuthorizationException("Permission denied: Only the organization owner or administrators can update the project."); + try { + checkUserIsStudyAdminInAllStudiesOfProject(organizationId, projectId, userId); + } catch (CatalogException e) { + throw new CatalogAuthorizationException("Permission denied: Only the organization owner, administrators, or users who are " + + "study administrators in all studies within the project can update the project.", e); + } + } + + private void checkUserIsStudyAdminInAllStudiesOfProject(String organizationId, long projectUid, String userId) throws CatalogException { + Query query = new Query(StudyDBAdaptor.QueryParams.PROJECT_UID.key(), projectUid); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(StudyDBAdaptor.QueryParams.GROUPS.key(), StudyDBAdaptor.QueryParams.FQN.key())); + OpenCGAResult studyResult = dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId).get(query, options); + List nonAdminStudyIds = new ArrayList<>(); + for (Study study : studyResult.getResults()) { + for (Group group : study.getGroups()) { + if (group.getId().equals(ADMINS_GROUP)) { + if (!group.getUserIds().contains(userId)) { + nonAdminStudyIds.add(study.getFqn()); + } + break; + } + } + } + if (!nonAdminStudyIds.isEmpty()) { + throw new CatalogAuthorizationException("Permission denied: User " + userId + " is not an administrator of the following" + + " studies: " + String.join(", ", nonAdminStudyIds)); + } } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index e138ae26dbf..2ffe6fd1ee9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -68,6 +68,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_DESCRIPTION("internal.status.description", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), RELATED_FILES("relatedFiles", TEXT_ARRAY, ""), + RELATED_FILES_FILE_UID("relatedFiles.file.uid", LONG, ""), RELATED_FILES_RELATION("relatedFiles.relation", TEXT, ""), SIZE("size", INTEGER, ""), EXPERIMENT("experiment", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index b34721cc1f7..a6016da7441 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -126,6 +126,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_WEBHOOK("internal.webhook", OBJECT, ""), INTERNAL_EVENTS("internal.events", OBJECT, ""), + INTERNAL_KILL_JOB_REQUESTED("internal.killJobRequested", BOOLEAN, ""), OUT_DIR("outDir", OBJECT, ""), INPUT("input", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java index f9175b0a908..d8d8e8a0a51 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/OrganizationDBAdaptor.java @@ -17,6 +17,7 @@ public interface OrganizationDBAdaptor extends Iterable { String IS_ORGANIZATION_ADMIN_OPTION = "isOrgAdmin"; + String AUTH_ORIGINS_FIELD = "authenticationOrigins"; enum QueryParams implements QueryParam { UID("uid", LONG, ""), @@ -28,8 +29,10 @@ enum QueryParams implements QueryParam { INTERNAL("internal", OBJECT, ""), INTERNAL_MIGRATION_EXECUTIONS("internal.migrationExecutions", OBJECT, ""), CONFIGURATION("configuration", OBJECT, ""), - CONFIGURATION_AUTHENTICATION_ORIGINS("configuration.authenticationOrigins", OBJECT, ""), - CONFIGURATION_AUTHENTICATION_ORIGINS_OPTIONS("configuration.authenticationOrigins.options", OBJECT, ""), + CONFIGURATION_OPTIMIZATIONS("configuration.optimizations", OBJECT, ""), + CONFIGURATION_AUTHENTICATION_ORIGINS("configuration." + AUTH_ORIGINS_FIELD, OBJECT, ""), + CONFIGURATION_AUTHENTICATION_ORIGINS_ID("configuration." + AUTH_ORIGINS_FIELD + ".id", STRING, ""), + CONFIGURATION_AUTHENTICATION_ORIGINS_OPTIONS("configuration." + AUTH_ORIGINS_FIELD + ".options", OBJECT, ""), CONFIGURATION_TOKEN("configuration.token", OBJECT, ""), CREATION_DATE("creationDate", DATE, ""), MODIFICATION_DATE("modificationDate", DATE, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java index 8a6c7207968..3099262e917 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/UserDBAdaptor.java @@ -118,13 +118,15 @@ enum QueryParams implements QueryParam { NAME("name", TEXT_ARRAY, ""), EMAIL("email", TEXT_ARRAY, ""), ORGANIZATION("organization", TEXT_ARRAY, ""), + INTERNAL("internal", OBJECT, ""), + INTERNAL_FAILED_ATTEMPTS("internal.failedAttempts", INTEGER, ""), INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), ACCOUNT("account", TEXT_ARRAY, ""), ACCOUNT_AUTHENTICATION_ID("account.authentication.id", TEXT, ""), ACCOUNT_CREATION_DATE("account.creationDate", TEXT, ""), - SIZE("size", INTEGER_ARRAY, ""), - QUOTA("quota", INTEGER_ARRAY, ""), + ACCOUNT_EXPIRATION_DATE("account.expirationDate", TEXT, ""), + QUOTA("quota", OBJECT, ""), ATTRIBUTES("attributes", TEXT, ""), // "Format: where is [<|<=|>|>=|==|!=|~|!~]" PROJECTS("projects", TEXT_ARRAY, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index f7896320d10..445c1c6047a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -259,8 +259,9 @@ public List effectivePermissions(long studyUid, List resourceIdList } Document studyDocument = studyResult.first(); + boolean simplifyPermissions = simplifyPermissions(); Map> groupsMap = getGroupUsersMap(studyDocument); - Map> studyUserPermissionsMap = extractUserPermissionsMap(groupsMap, studyDocument); + Map> studyUserPermissionsMap = extractUserPermissionsMap(groupsMap, studyDocument, simplifyPermissions); // Retrieve ACL list for the resources requested MongoDBCollection collection = getMainCollection(entry); @@ -284,7 +285,8 @@ public List effectivePermissions(long studyUid, List resourceIdList throw new CatalogDBException("Resource id '" + resourceId + "' not found."); } Document resourceDocument = dataResultMap.get(resourceId); - Map> resourceUserPermissionsMap = extractUserPermissionsMap(groupsMap, resourceDocument); + Map> resourceUserPermissionsMap = extractUserPermissionsMap(groupsMap, resourceDocument, + simplifyPermissions); Acl acl = convertPermissionsToAcl(groupsMap, studyUserPermissionsMap, resourceUserPermissionsMap, resourceId, entry); aclList.add(acl); } @@ -357,7 +359,8 @@ private Acl convertPermissionsToAcl(Map> groupsMap, Map> extractUserPermissionsMap(Map> groupsMap, Document document) { + private Map> extractUserPermissionsMap(Map> groupsMap, Document document, + boolean simplifyPermissions) { Set allUsers = groupsMap.get(ParamConstants.MEMBERS_GROUP); // Map of userId - List of permissions @@ -383,8 +386,6 @@ private Map> extractUserPermissionsMap(Map userIdsWithPermissions = new HashSet<>(); // Personal ACLs diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java index 987a493fd38..fb8a3952f50 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java @@ -25,7 +25,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.study.StudyPermissions; @@ -200,35 +199,18 @@ private static int getPermissionType(Enums.Resource resource) throws CatalogPara /** * If query contains {@link ParamConstants#ACL_PARAM}, it will parse the value to generate the corresponding mongo query documents. * - * @param study Queried study document. - * @param query Original query. - * @param resource Affected resource. - * @param user User performing the query. - * @return A list of documents to satisfy the ACL query. - * @throws CatalogDBException when there is a DB error. - * @throws CatalogParameterException if there is any formatting error. - * @throws CatalogAuthorizationException if the user is not authorised to perform the query. - */ - public static List parseAclQuery(Document study, Query query, Enums.Resource resource, String user) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - return parseAclQuery(study, query, resource, user, null); - } - - /** - * If query contains {@link ParamConstants#ACL_PARAM}, it will parse the value to generate the corresponding mongo query documents. - * - * @param study Queried study document. - * @param query Original query. - * @param resource Affected resource. - * @param user User performing the query. - * @param configuration Configuration object. + * @param study Queried study document. + * @param query Original query. + * @param resource Affected resource. + * @param user User performing the query. + * @param simplifyPermissions Boolean indicating whether permission check can be simplified. * @return A list of documents to satisfy the ACL query. * @throws CatalogDBException when there is a DB error. * @throws CatalogParameterException if there is any formatting error. * @throws CatalogAuthorizationException if the user is not authorised to perform the query. */ public static List parseAclQuery(Document study, Query query, Enums.Resource resource, String user, - Configuration configuration) + boolean simplifyPermissions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { List aclDocuments = new LinkedList<>(); if (!query.containsKey(ParamConstants.ACL_PARAM)) { @@ -264,11 +246,6 @@ public static List parseAclQuery(Document study, Query query, Enums.Re + study.getString(StudyDBAdaptor.QueryParams.ID.key())); } - boolean simplifyPermissionCheck = false; - if (configuration != null && configuration.getOptimizations() != null) { - simplifyPermissionCheck = configuration.getOptimizations().isSimplifyPermissions(); - } - boolean isAnonymousPresent = false; boolean isRegisteredUsersPresent = false; List groups; @@ -302,7 +279,7 @@ public static List parseAclQuery(Document study, Query query, Enums.Re } Document queryDocument = getAuthorisedEntries(affectedUser, groups, permission, isRegisteredUsersPresent, isAnonymousPresent, - simplifyPermissionCheck); + simplifyPermissions); if (hasStudyPermissions) { // The user has permissions defined globally, so we also have to check the entries where the user/groups/members/* have no // permissions defined as the user will also be allowed to see them @@ -317,13 +294,8 @@ public static List parseAclQuery(Document study, Query query, Enums.Re return aclDocuments; } - public static Document getQueryForAuthorisedEntries(Document study, String user, String permission, Enums.Resource resource) - throws CatalogAuthorizationException, CatalogParameterException { - return getQueryForAuthorisedEntries(study, user, permission, resource, null); - } - public static Document getQueryForAuthorisedEntries(Document study, String user, String permission, Enums.Resource resource, - Configuration configuration) + boolean simplifyPermissions) throws CatalogAuthorizationException, CatalogParameterException { if (StringUtils.isEmpty(user)) { return new Document(); @@ -342,11 +314,6 @@ public static Document getQueryForAuthorisedEntries(Document study, String user, + study.getString(StudyDBAdaptor.QueryParams.ID.key())); } - boolean simplifyPermissionCheck = false; - if (configuration != null && configuration.getOptimizations() != null) { - simplifyPermissionCheck = configuration.getOptimizations().isSimplifyPermissions(); - } - String studyPermission = StudyPermissions.Permissions.getStudyPermission(permission, getPermissionType(resource)).name(); // 0. Check if anonymous has any permission defined (just for performance) @@ -375,8 +342,8 @@ public static Document getQueryForAuthorisedEntries(Document study, String user, } Document queryDocument = getAuthorisedEntries(user, groups, permission, isRegisteredUsersPresent, isAnonymousPresent, - simplifyPermissionCheck); - if (hasStudyPermissions && !simplifyPermissionCheck) { + simplifyPermissions); + if (hasStudyPermissions && !simplifyPermissions) { // The user has permissions defined globally, so we also have to check the entries where the user/groups/members/* have no // permissions defined as the user will also be allowed to see them queryDocument = new Document("$or", Arrays.asList( diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index a4073a8c192..ec066ba4fc6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -33,6 +33,7 @@ import org.opencb.commons.datastore.mongodb.MongoDBIterator; import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; import org.opencb.opencga.catalog.db.api.DBIterator; +import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.ClinicalAnalysisCatalogMongoDBIterator; @@ -151,6 +152,10 @@ static void fixFilesForRemoval(ObjectMap parameters, String key) { for (Object file : parameters.getAsList(key)) { if (file instanceof File) { fileParamList.add(new Document("uid", ((File) file).getUid())); + } else if (file instanceof Document) { + fileParamList.add(new Document("uid", ((Document) file).get("uid"))); + } else { + throw new IllegalArgumentException("Expected a File or Document object"); } } parameters.put(key, fileParamList); @@ -814,6 +819,25 @@ OpenCGAResult privateDelete(ClientSession clientSession, ClinicalAnalysis cli return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList()); } + void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document file) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + ObjectMap parameters = new ObjectMap(FILES.key(), Collections.singletonList(file)); + ObjectMap actionMap = new ObjectMap(FILES.key(), ParamUtils.BasicUpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + + Query query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.FILES_UID.key(), fileUid); + OpenCGAResult result = get(query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + for (ClinicalAnalysis clinicalAnalysis : result.getResults()) { + logger.debug("Removing file references from Clinical Analysis {}", clinicalAnalysis.getId()); + ClinicalAudit clinicalAudit = new ClinicalAudit("OPENCGA", ClinicalAudit.Action.UPDATE_CLINICAL_ANALYSIS, "File " + + file.getString(FileDBAdaptor.QueryParams.PATH.key()) + " was deleted. Remove file references from case.", + TimeUtils.getTime()); + transactionalUpdate(clientSession, clinicalAnalysis, parameters, null, Collections.singletonList(clinicalAudit), options); + } + } + @Override public OpenCGAResult restore(long id, QueryOptions queryOptions) throws CatalogDBException { return null; @@ -1292,17 +1316,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (queryCopy.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || queryCopy.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, queryCopy.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (queryCopy.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, queryCopy, Enums.Resource.CLINICAL_ANALYSIS, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - ClinicalAnalysisPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.CLINICAL_ANALYSIS, configuration)); + ClinicalAnalysisPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.CLINICAL_ANALYSIS, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, ClinicalAnalysisPermissions.VIEW.name(), - Enums.Resource.CLINICAL_ANALYSIS, configuration)); + Enums.Resource.CLINICAL_ANALYSIS, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index c6dd04863de..7f25b1ee7b9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -235,22 +235,21 @@ public OpenCGAResult update(long cohortId, ObjectMap parameters, QueryOptions qu @Override public OpenCGAResult update(long cohortUid, ObjectMap parameters, List variableSetList, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - Query query = new Query(QueryParams.UID.key(), cohortUid); - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, - Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.STUDY_UID.key(), - QueryParams.SAMPLES.key() + "." + QueryParams.ID.key())); - OpenCGAResult documentResult = get(query, options); - if (documentResult.getNumResults() == 0) { - throw new CatalogDBException("Could not update cohort. Cohort uid '" + cohortUid + "' not found."); - } - String cohortId = documentResult.first().getId(); - try { - return runTransaction(clientSession -> transactionalUpdate(clientSession, documentResult.first(), parameters, variableSetList, - queryOptions)); - } catch (CatalogDBException e) { - logger.error("Could not update cohort {}: {}", cohortId, e.getMessage(), e); - throw new CatalogDBException("Could not update cohort " + cohortId + ": " + e.getMessage(), e.getCause()); + return runTransaction(clientSession -> { + Query query = new Query(QueryParams.UID.key(), cohortUid); + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, + Arrays.asList(QueryParams.ID.key(), QueryParams.UID.key(), QueryParams.STUDY_UID.key(), + QueryParams.SAMPLES.key() + "." + QueryParams.ID.key())); + OpenCGAResult documentResult = get(clientSession, query, options); + if (documentResult.getNumResults() == 0) { + throw new CatalogDBException("Could not update cohort. Cohort uid '" + cohortUid + "' not found."); + } + return transactionalUpdate(clientSession, documentResult.first(), parameters, variableSetList, queryOptions); + }); + } catch (Exception e) { + logger.error("Could not update cohort {}: {}", cohortUid, e.getMessage(), e); + throw new CatalogDBException("Could not update cohort " + cohortUid + ": " + e.getMessage(), e); } } @@ -879,18 +878,19 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { + boolean simplifyPermissions = simplifyPermissions(); Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.COHORT, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - CohortPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.COHORT, configuration)); + CohortPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.COHORT, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, CohortPermissions.VIEW.name(), - Enums.Resource.COHORT, configuration)); + Enums.Resource.COHORT, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 2b7ed748772..9e8c35d00ab 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -1191,17 +1191,18 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.FAMILY, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - FamilyPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.FAMILY, configuration)); + FamilyPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.FAMILY, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, FamilyPermissions.VIEW.name(), - Enums.Resource.FAMILY, configuration)); + Enums.Resource.FAMILY, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 5c3b592e9b9..f214620bf39 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.managers.FileUtils; import org.opencb.opencga.catalog.managers.SampleManager; import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -785,7 +786,8 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon document.getSet().put(QueryParams.RELATED_FILES.key(), relatedFileDocument); break; case REMOVE: - document.getPullAll().put(QueryParams.RELATED_FILES.key(), relatedFileDocument); + List documentList = fixRelatedFilesForRemoval(relatedFileDocument); + document.getPull().put(QueryParams.RELATED_FILES.key(), documentList); break; case ADD: document.getAddToSet().put(QueryParams.RELATED_FILES.key(), relatedFileDocument); @@ -886,6 +888,18 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon return document; } + private List fixRelatedFilesForRemoval(List relatedFiles) { + if (CollectionUtils.isEmpty(relatedFiles)) { + return Collections.emptyList(); + } + + List relatedFilesCopy = new ArrayList<>(relatedFiles.size()); + for (Document relatedFile : relatedFiles) { + relatedFilesCopy.add(new Document("file", new Document("uid", relatedFile.get("file", Document.class).get("uid")))); + } + return relatedFilesCopy; + } + @Override public OpenCGAResult delete(File file) throws CatalogDBException { throw new UnsupportedOperationException("Use delete passing status field."); @@ -1007,7 +1021,9 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document fileDo Document tmpFile = iterator.next(); long tmpFileUid = tmpFile.getLong(PRIVATE_UID); + removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile); dbAdaptorFactory.getCatalogJobDBAdaptor().removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile); + dbAdaptorFactory.getClinicalAnalysisDBAdaptor().removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile); // Set status nestedPut(QueryParams.INTERNAL_STATUS.key(), getMongoDBDocument(new FileStatus(status), "status"), tmpFile); @@ -1035,6 +1051,28 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document fileDo } } + void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document fileDoc) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + File file = fileConverter.convertToDataModelType(fileDoc); + FileRelatedFile relatedFile = new FileRelatedFile(file, null); + ObjectMap parameters = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList(relatedFile)); + ObjectMap actionMap = new ObjectMap(QueryParams.RELATED_FILES.key(), ParamUtils.BasicUpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + + Query query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.RELATED_FILES_FILE_UID.key(), fileUid); + UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, studyUid, parameters, query, options); + Document updateDoc = updateDocument.toFinalUpdateDocument(); + if (!updateDoc.isEmpty()) { + Bson bsonQuery = parseQuery(query); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); + if (result.getNumUpdated() > 0) { + logger.debug("File '{}' removed from related files array from {} files.", file.getPath(), result.getNumUpdated()); + } + } + } + // OpenCGAResult privateDelete(ClientSession clientSession, File file, String status) throws CatalogDBException { // long tmpStartTime = startQuery(); // logger.debug("Deleting file {} ({})", file.getPath(), file.getUid()); @@ -1371,16 +1409,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { - andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.FILE, user, configuration)); + andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.FILE, user, + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, FilePermissions.VIEW_ANNOTATIONS.name(), - Enums.Resource.FILE, configuration)); + Enums.Resource.FILE, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, FilePermissions.VIEW.name(), - Enums.Resource.FILE, configuration)); + Enums.Resource.FILE, simplifyPermissions)); } } @@ -1479,13 +1519,6 @@ private Bson parseQuery(Query query, Document extraQuery, String user) addAutoOrQuery(queryParam.key(), queryParam.key(), myQuery, queryParam.type(), andBsonList); } break; - case FORMAT: - case BIOFORMAT: - // Replace the value for an uppercase string as we know it will always be in that way - String uppercaseValue = myQuery.getString(queryParam.key()).toUpperCase(); - myQuery.put(queryParam.key(), uppercaseValue); - addAutoOrQuery(queryParam.key(), queryParam.key(), myQuery, queryParam.type(), andBsonList); - break; case UUID: case EXTERNAL: case TYPE: @@ -1493,9 +1526,12 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case ID: case PATH: case RELEASE: + case FORMAT: + case BIOFORMAT: case TAGS: case SIZE: case SOFTWARE_NAME: + case RELATED_FILES_FILE_UID: case JOB_ID: addAutoOrQuery(queryParam.key(), queryParam.key(), myQuery, queryParam.type(), andBsonList); break; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 707375a5811..13dbb40741c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -1405,17 +1405,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.INDIVIDUAL, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - IndividualPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.INDIVIDUAL, configuration)); + IndividualPermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.INDIVIDUAL, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, IndividualPermissions.VIEW.name(), - Enums.Resource.INDIVIDUAL, configuration)); + Enums.Resource.INDIVIDUAL, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 58b3291a0d4..53c9a5f157e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -373,7 +373,7 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO String[] acceptedParams = {QueryParams.USER_ID.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); - String[] acceptedBooleanParams = {QueryParams.VISITED.key()}; + String[] acceptedBooleanParams = {QueryParams.VISITED.key(), QueryParams.INTERNAL_KILL_JOB_REQUESTED.key()}; filterBooleanParams(parameters, document.getSet(), acceptedBooleanParams); String[] acceptedStringListParams = {QueryParams.TAGS.key()}; @@ -814,13 +814,15 @@ private Bson parseQuery(Query query, Document extraQuery, QueryOptions options, if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { - andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.JOB, user, configuration)); + andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.JOB, user, + simplifyPermissions)); } else { // Get the document query needed to check the permissions as well andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, JobPermissions.VIEW.name(), - Enums.Resource.JOB, configuration)); + Enums.Resource.JOB, simplifyPermissions)); } query.remove(ParamConstants.ACL_PARAM); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java index 0b6f07d4269..7f41e38411d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptor.java @@ -28,10 +28,12 @@ import org.opencb.opencga.catalog.db.AbstractDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; @@ -703,6 +705,24 @@ protected Document getStudyDocument(ClientSession clientSession, long studyUid) return dataResult.first(); } + /** + * Method to obtain whether permissions should be simplified or not. + * + * @return true if permissions should be simplified, false otherwise. + * @throws CatalogDBException if there is any error obtaining the organization configuration. + */ + protected boolean simplifyPermissions() throws CatalogDBException { + Organization organization = dbAdaptorFactory.getCatalogOrganizationDBAdaptor() + .get(OrganizationManager.INCLUDE_ORGANIZATION_CONFIGURATION).first(); + if (organization.getConfiguration().getOptimizations() != null) { + return organization.getConfiguration().getOptimizations().isSimplifyPermissions(); + } else { + logger.warn("Organization '{}' configuration does not contain the 'optimizations.simplifyPermissions' field. Defaulting" + + " to false", organization.getId()); + return false; + } + } + public class NestedArrayUpdateDocument { private Query query; private Document set; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 47b6e96e406..621c537cddd 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -31,16 +31,16 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.managers.NoteManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; -import org.opencb.opencga.core.models.organizations.Organization; -import org.opencb.opencga.core.models.organizations.OrganizationSummary; import org.opencb.opencga.core.models.notes.Note; import org.opencb.opencga.core.models.notes.NoteCreateParams; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationSummary; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java index 4261a247c79..aa090f498d1 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptor.java @@ -24,6 +24,7 @@ import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.response.OpenCGAResult; @@ -137,11 +138,7 @@ private OpenCGAResult privateUpdate(ClientSession clientSession, S UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, parameters, queryOptions); Document organizationUpdate = updateDocument.toFinalUpdateDocument(); -// Query tmpQuery = new Query(QueryParams.ID.key(), organization.getId()); -// Bson queryBson = parseQuery(tmpQuery); - Bson queryBson = Filters.eq(QueryParams.ID.key(), organizationId); - - if (organizationUpdate.isEmpty()) { + if (organizationUpdate.isEmpty() && CollectionUtils.isEmpty(updateDocument.getNestedUpdateList())) { if (!parameters.isEmpty()) { logger.error("Non-processed update parameters: {}", parameters.keySet()); throw new CatalogDBException("Update could not be performed. Some fields could not be processed."); @@ -149,20 +146,52 @@ private OpenCGAResult privateUpdate(ClientSession clientSession, S throw new CatalogDBException("Nothing to be updated"); } - List events = new ArrayList<>(); - logger.debug("Update organization. Query: {}, Update: {}", queryBson.toBsonDocument(), organizationUpdate.toBsonDocument()); - // Update study admins need to be executed before the actual update because we need to fetch the previous owner/admins in case // of an update on these fields. updateStudyAdmins(clientSession, parameters, queryOptions); - DataResult updateResult = organizationCollection.update(clientSession, queryBson, organizationUpdate, null); - if (updateResult.getNumMatches() == 0) { - throw new CatalogDBException("Organization not found"); + List events = new ArrayList<>(); + DataResult updateResult; + if (!organizationUpdate.isEmpty()) { + Bson queryBson = Filters.eq(QueryParams.ID.key(), organizationId); + logger.debug("Update organization. Query: {}, Update: {}", queryBson.toBsonDocument(), organizationUpdate.toBsonDocument()); + + updateResult = organizationCollection.update(clientSession, queryBson, organizationUpdate, null); + + if (updateResult.getNumMatches() == 0) { + throw new CatalogDBException("Organization not found"); + } + if (updateResult.getNumUpdated() == 0) { + events.add(new Event(Event.Type.WARNING, organizationId, "Organization was already updated")); + } } - if (updateResult.getNumUpdated() == 0) { - events.add(new Event(Event.Type.WARNING, organizationId, "Organization was already updated")); + if (CollectionUtils.isNotEmpty(updateDocument.getNestedUpdateList())) { + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson bsonQuery = new Document(nestedDocument.getQuery()); + logger.debug("Update nested element from Organization. Query: {}, Update: {}", bsonQuery.toBsonDocument(), + nestedDocument.getSet()); + DataResult result = organizationCollection.update(clientSession, bsonQuery, nestedDocument.getSet(), null); + if (result.getNumMatches() == 0) { + throw new CatalogDBException("Couldn't update organization. Nothing could be found for query " + + bsonQuery.toBsonDocument()); + } + } + /* + for (NestedArrayUpdateDocument nestedDocument : updateDocument.getNestedUpdateList()) { + Bson nestedBsonQuery = parseQuery(nestedDocument.getQuery() + .append(QueryParams.UID.key(), interpretation.getUid())); + logger.debug("Update nested element from interpretation. Query: {}, Update: {}", + nestedBsonQuery.toBsonDocument(), nestedDocument.getSet()); + + update = interpretationCollection.update(clientSession, nestedBsonQuery, nestedDocument.getSet(), null); + + if (update.getNumMatches() == 0) { + throw CatalogDBException.uidNotFound("Interpretation", interpretationUid); + } + } + * */ } + logger.debug("Organization {} successfully updated", organizationId); @@ -235,9 +264,34 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, Obj String[] acceptedParams = { QueryParams.NAME.key() }; filterStringParams(parameters, document.getSet(), acceptedParams); - String[] acceptedObjectParams = { QueryParams.CONFIGURATION.key() }; + String[] acceptedObjectParams = { QueryParams.CONFIGURATION_OPTIMIZATIONS.key(), QueryParams.CONFIGURATION_TOKEN.key() }; filterObjectParams(parameters, document.getSet(), acceptedObjectParams); + // Authentication Origins action + if (parameters.containsKey(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key())) { + Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); + ParamUtils.UpdateAction operation = ParamUtils.UpdateAction.from(actionMap, OrganizationDBAdaptor.AUTH_ORIGINS_FIELD); + String[] authOriginsParams = {QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key()}; + switch (operation) { + case SET: + filterObjectParams(parameters, document.getSet(), authOriginsParams); + break; + case REMOVE: + fixAuthOriginsForRemoval(parameters); + filterObjectParams(parameters, document.getPull(), authOriginsParams); + break; + case ADD: + filterObjectParams(parameters, document.getAddToSet(), authOriginsParams); + break; + case REPLACE: + filterReplaceParams(parameters.getAsList(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key(), Map.class), document, + m -> String.valueOf(m.get("id")), QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS_ID.key()); + break; + default: + throw new IllegalStateException("Unknown operation " + operation); + } + } + String owner = parameters.getString(QueryParams.OWNER.key(), null); if (StringUtils.isNotEmpty(owner)) { // Check user exists @@ -312,6 +366,21 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, Obj return document; } + private void fixAuthOriginsForRemoval(ObjectMap parameters) { + if (parameters.get(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key()) == null) { + return; + } + List authOriginParamList = new LinkedList<>(); + for (Object authOrigin : parameters.getAsList(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key())) { + if (authOrigin instanceof AuthenticationOrigin) { + authOriginParamList.add(new Document("id", ((AuthenticationOrigin) authOrigin).getId())); + } else { + authOriginParamList.add(new Document("id", ((Map) authOrigin).get("id"))); + } + } + parameters.putNested(QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key(), authOriginParamList, false); + } + @Override public OpenCGAResult delete(Organization organization) throws CatalogDBException { return null; @@ -336,9 +405,6 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu } qOptions = filterQueryOptionsToIncludeKeys(qOptions, OrganizationManager.INCLUDE_ORGANIZATION_IDS.getAsStringList(QueryOptions.INCLUDE)); - if (!qOptions.getBoolean(IS_ORGANIZATION_ADMIN_OPTION)) { - qOptions = filterQueryOptionsToExcludeKeys(qOptions, Arrays.asList(QueryParams.CONFIGURATION.key())); - } return organizationCollection.iterator(clientSession, new Document(), null, null, qOptions); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index d373c68bb2b..d82535b9351 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -722,14 +722,15 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.DISEASE_PANEL, user, - configuration)); + simplifyPermissions)); } else { // Get the document query needed to check the permissions as well andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, PanelPermissions.VIEW.name(), - Enums.Resource.DISEASE_PANEL, configuration)); + Enums.Resource.DISEASE_PANEL, simplifyPermissions)); } query.remove(ParamConstants.ACL_PARAM); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index c1c8dfeb7bd..89d90aab932 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -542,7 +542,7 @@ public OpenCGAResult get(Query query, QueryOptions options, String user } @Override - public OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException { + public OpenCGAResult nativeGet(Query query, QueryOptions options) throws CatalogDBException { return nativeGet(null, query, options); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index a6d3b8521c0..198c9fce456 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -1327,17 +1327,18 @@ private Bson parseQuery(Query query, Document extraQuery, String user) if (query.containsKey(QueryParams.STUDY_UID.key()) && (StringUtils.isNotEmpty(user) || query.containsKey(ParamConstants.ACL_PARAM))) { Document studyDocument = getStudyDocument(null, query.getLong(QueryParams.STUDY_UID.key())); + boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.SAMPLE, user, - configuration)); + simplifyPermissions)); } else { if (containsAnnotationQuery(query)) { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, - SamplePermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.SAMPLE, configuration)); + SamplePermissions.VIEW_ANNOTATIONS.name(), Enums.Resource.SAMPLE, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, SamplePermissions.VIEW.name(), - Enums.Resource.SAMPLE, configuration)); + Enums.Resource.SAMPLE, simplifyPermissions)); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 21b592d1195..19a9a6cd1f3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -427,27 +427,35 @@ public OpenCGAResult nativeGet(Query query, QueryOptions options) throws Catalog @Override public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException { - Map userParameters = new HashMap<>(); + UpdateDocument document = new UpdateDocument(); - final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key()}; - filterStringParams(parameters, userParameters, acceptedParams); + final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.EMAIL.key(), ACCOUNT_EXPIRATION_DATE.key()}; + filterStringParams(parameters, document.getSet(), acceptedParams); if (parameters.containsKey(QueryParams.INTERNAL_STATUS_ID.key())) { - userParameters.put(QueryParams.INTERNAL_STATUS_ID.key(), parameters.get(QueryParams.INTERNAL_STATUS_ID.key())); - userParameters.put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); + document.getSet().put(QueryParams.INTERNAL_STATUS_ID.key(), parameters.get(QueryParams.INTERNAL_STATUS_ID.key())); + document.getSet().put(QueryParams.INTERNAL_STATUS_DATE.key(), TimeUtils.getTime()); } - final String[] acceptedLongParams = {QueryParams.QUOTA.key(), QueryParams.SIZE.key()}; - filterLongParams(parameters, userParameters, acceptedLongParams); + final String[] acceptedIntParams = {INTERNAL_FAILED_ATTEMPTS.key()}; + filterIntParams(parameters, document.getSet(), acceptedIntParams); + + final String[] acceptedObjectParams = {QueryParams.QUOTA.key()}; + filterObjectParams(parameters, document.getSet(), acceptedObjectParams); final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; - filterMapParams(parameters, userParameters, acceptedMapParams); + filterMapParams(parameters, document.getSet(), acceptedMapParams); + + if (!document.toFinalUpdateDocument().isEmpty()) { + document.getSet().put(INTERNAL_LAST_MODIFIED, TimeUtils.getTime()); + } - if (!userParameters.isEmpty()) { - return new OpenCGAResult(userCollection.update(parseQuery(query), new Document("$set", userParameters), null)); + Document userUpdate = document.toFinalUpdateDocument(); + if (userUpdate.isEmpty()) { + throw new CatalogDBException("Nothing to be updated."); } - return OpenCGAResult.empty(); + return new OpenCGAResult(userCollection.update(parseQuery(query), userUpdate, null)); } @Override @@ -476,11 +484,7 @@ public OpenCGAResult update(String userId, ObjectMap parameters) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { checkId(userId); Query query = new Query(QueryParams.ID.key(), userId); - OpenCGAResult update = update(query, parameters, QueryOptions.empty()); - if (update.getNumUpdated() != 1) { - throw new CatalogDBException("Could not update user " + userId); - } - return update; + return update(query, parameters, QueryOptions.empty()); } OpenCGAResult setStatus(Query query, String status) throws CatalogDBException { @@ -666,8 +670,6 @@ private Bson parseQuery(Query query) throws CatalogDBException { case EMAIL: case ORGANIZATION: case INTERNAL_STATUS_DATE: - case SIZE: - case QUOTA: case ACCOUNT_AUTHENTICATION_ID: case ACCOUNT_CREATION_DATE: case TOOL_ID: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java index 85306f33571..c50446677f7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java @@ -173,6 +173,8 @@ protected OpenCGAResult update(ClientSession session, Bson sourceQuery, L Document result = iterator.next(); entryList.add(result); + entryList.add(result); + long uid = result.get(PRIVATE_UID, Number.class).longValue(); int version = result.get(VERSION, Number.class).intValue(); String transactionId = result.getString(PRIVATE_TRANSACTION_ID); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java index 267666f77f0..c4db86b7408 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java @@ -107,7 +107,7 @@ public List convertRelatedFiles(List relatedFileList) if (ListUtils.isNotEmpty(relatedFileList)) { for (FileRelatedFile relatedFile : relatedFileList) { relatedFiles.add(new Document() - .append("relation", relatedFile.getRelation().name()) + .append("relation", relatedFile.getRelation() != null ? relatedFile.getRelation().name() : null) .append("file", new Document("uid", relatedFile.getFile().getUid())) ); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java index 8cb658885dc..3853e91607f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthenticationException.java @@ -61,6 +61,21 @@ public static CatalogAuthenticationException incorrectUserOrPassword(String doma return new CatalogAuthenticationException(domain + ": Incorrect user or password.", e); } + public static CatalogAuthenticationException userIsBanned(String userId) { + return new CatalogAuthenticationException("Too many login attempts. The account for user '" + userId + "' is banned." + + " Please, talk to your organization owner/administrator."); + } + + public static CatalogAuthenticationException userIsSuspended(String userId) { + return new CatalogAuthenticationException("The account for user '" + userId + "' is suspended. Please, talk to your organization" + + " owner/administrator."); + } + + public static CatalogAuthenticationException accountIsExpired(String userId, String expirationDate) { + return new CatalogAuthenticationException("The account for user '" + userId + "' expired on " + expirationDate + ". Please," + + " talk to your organization owner/administrator."); + } + public static CatalogAuthenticationException userNotAllowed(String domain) { return new CatalogAuthenticationException(domain + ": User not allowed to access the system."); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java index 28e0310609e..546d3117203 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/exceptions/CatalogAuthorizationException.java @@ -20,6 +20,28 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ public class CatalogAuthorizationException extends CatalogException { + + public enum ErrorCode { + KILL_JOB("kill", "Only the job owner or the study administrator can kill a job."); + + private final String permission; + private final String message; + + ErrorCode(String permission, String message) { + this.permission = permission; + this.message = message; + } + + public String getPermission() { + return permission; + } + + public String getMessage() { + return message; + } + } + + public CatalogAuthorizationException(String message) { super(message); } @@ -63,6 +85,16 @@ public static CatalogAuthorizationException deny(String userId, String permissio + resource + " { id: " + id + (name == null || name.isEmpty() ? "" : ", name: \"" + name + "\"") + " }"); } + public static CatalogAuthorizationException deny(String userId, String resource, String id, ErrorCode errorCode) { + return deny(userId, resource, id, errorCode, null); + } + + public static CatalogAuthorizationException deny(String userId, String resource, String id, ErrorCode errorCode, Throwable cause) { + return new CatalogAuthorizationException("Permission denied. " + + (userId == null || userId.isEmpty() ? "" : "User '" + userId + "'") + + " cannot " + errorCode.getPermission() + " " + resource + " { id: " + id + " }. " + errorCode.getMessage(), cause); + } + public static CatalogAuthorizationException deny(String userId, String description) { return new CatalogAuthorizationException("Permission denied. " + (userId == null || userId.isEmpty() ? "" : "User '" + userId + "'") + " cannot " + description); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java index 9ef3fd7f6e1..6c25baa36ff 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java @@ -215,6 +215,8 @@ public void audit(String organizationId, String operationId, String userId, Enum try { dbAdaptorFactory.getCatalogAuditDbAdaptor(organizationId).insertAuditRecords(this.auditRecordMap.get(operationId)); } catch (CatalogDBException e) { + // FIXME : How to raise attention on this silent error? + // This is a critical error that should not happen. logger.error("Could not audit operation '{}' -> Error: {}", operationId, e.getMessage(), e); } finally { this.auditRecordMap.get(operationId).clear(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 2308b7ad123..1409b1e9d4c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -20,6 +20,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; +import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.auth.authentication.azure.AuthenticationFactory; import org.opencb.opencga.catalog.auth.authorization.AuthorizationDBAdaptorFactory; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; @@ -35,6 +36,7 @@ import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.migration.MigrationManager; +import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.JwtUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.PasswordUtils; @@ -107,7 +109,7 @@ private void init() throws CatalogException { catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory); authorizationDBAdaptorFactory = new AuthorizationMongoDBAdaptorFactory((MongoDBAdaptorFactory) catalogDBAdaptorFactory, configuration); - authenticationFactory = new AuthenticationFactory(catalogDBAdaptorFactory); + authenticationFactory = new AuthenticationFactory(catalogDBAdaptorFactory, configuration); logger.debug("CatalogManager configureManager"); configureManagers(configuration); } @@ -259,6 +261,10 @@ private void privateInstall(String algorithm, String secretKey, String password, if (!PasswordUtils.isStrongPassword(password)) { throw new CatalogException("Invalid password. Check password strength for user "); } + if (StringUtils.isEmpty(secretKey)) { + logger.info("Generating secret key"); + secretKey = PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH); + } ParamUtils.checkParameter(secretKey, "secretKey"); ParamUtils.checkParameter(password, "password"); JwtUtils.validateJWTKey(algorithm, secretKey); @@ -267,7 +273,7 @@ private void privateInstall(String algorithm, String secretKey, String password, OrganizationConfiguration organizationConfiguration = new OrganizationConfiguration( Collections.singletonList(CatalogAuthenticationManager.createOpencgaAuthenticationOrigin()), - new Optimizations(), new TokenConfiguration(algorithm, secretKey, 3600L)); + Constants.DEFAULT_USER_EXPIRATION_DATE, new Optimizations(), new TokenConfiguration(algorithm, secretKey, 3600L)); organizationManager.create(new OrganizationCreateParams(ADMIN_ORGANIZATION, ADMIN_ORGANIZATION, null, null, organizationConfiguration, null), QueryOptions.empty(), null); @@ -301,7 +307,7 @@ public void installIndexes(String token) throws CatalogException { public void installIndexes(String organizationId, String token) throws CatalogException { JwtPayload payload = userManager.validateToken(token); - String userId = payload.getUserId(); + String userId = payload.getUserId(organizationId); if (!authorizationManager.isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) { throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin(); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 48ee0d6adda..37be3aeeba2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -33,10 +33,7 @@ import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.IOManager; @@ -1580,7 +1577,7 @@ public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions return delete(studyStr, fileIds, options, false, token); } - public OpenCGAResult delete(String studyStr, List fileIds, ObjectMap params, boolean ignoreException, String token) + public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions options, boolean ignoreException, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); @@ -1594,13 +1591,15 @@ public OpenCGAResult delete(String studyStr, List fileIds, ObjectMap par ObjectMap auditParams = new ObjectMap() .append("study", studyStr) .append("fileIds", fileIds) - .append("params", params) + .append("options", options) .append("ignoreException", ignoreException) .append("token", token); + QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); + // We need to avoid processing subfolders or subfiles of an already processed folder independently Set processedPaths = new HashSet<>(); - boolean physicalDelete = params.getBoolean(Constants.SKIP_TRASH, false); + boolean physicalDelete = queryOptions.getBoolean(Constants.SKIP_TRASH, false); auditManager.initAuditBatch(operationUuid); OpenCGAResult result = OpenCGAResult.empty(File.class); @@ -1888,10 +1887,10 @@ public OpenCGAResult unlink(@Nullable String studyId, String fileId, Strin study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; - } catch (CatalogException e) { + } catch (Exception e) { auditManager.audit(organizationId, userId, Enums.Action.UNLINK, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - throw e; + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", e.getMessage()))); + throw new CatalogException("Could not unlink file '" + fileId + "'", e); } } @@ -3231,6 +3230,16 @@ private void checkCanDeleteFile(String organizationId, Study study, String fileI // TODO: Validate no file/folder within any registered directory is not registered in OpenCGA } + Query clinicalQuery = new Query() + .append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) + .append(ClinicalAnalysisDBAdaptor.QueryParams.FILES_UID.key(), file.getUid()) + .append(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), true); + OpenCGAResult count = getClinicalAnalysisDBAdaptor(organizationId).count(clinicalQuery); + if (count.getNumMatches() > 0) { + throw new CatalogException("The file " + file.getName() + " is part of " + count.getNumMatches() + " clinical analyses"); + } + + // Check the original files are not being indexed at the moment if (!indexFiles.isEmpty()) { Query query = new Query(FileDBAdaptor.QueryParams.UID.key(), new ArrayList<>(indexFiles)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index 7a8f4ad0f73..b8b9ba55bae 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -39,9 +39,11 @@ import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.Execution; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.AclParams; import org.opencb.opencga.core.models.JwtPayload; @@ -216,9 +218,9 @@ public OpenCGAResult visit(String studyId, String jobId, String token) thro study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; - } catch (CatalogException e) { + } catch (Exception e) { auditManager.audit(organizationId, userId, Enums.Action.VISIT, Enums.Resource.JOB, jobId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -308,14 +310,14 @@ public OpenCGAResult create(String studyStr, Job job, QueryOptions options, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditCreate(organizationId, userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } - private void autoCompleteNewJob(String organizationId, Study study, Job job, String token) throws CatalogException { + private void autoCompleteNewJob(String organizationId, Study study, Job job, JwtPayload tokenPayload) throws CatalogException { ParamUtils.checkObj(job, "Job"); // Auto generate id @@ -333,6 +335,27 @@ private void autoCompleteNewJob(String organizationId, Study study, Job job, Str job.setInternal(JobInternal.init()); job.getInternal().setWebhook(new JobInternalWebhook(study.getNotification().getWebhook(), new HashMap<>())); + if (StringUtils.isNotEmpty(job.getParentId())) { + // Check parent job exists + try { + Job tmpJob = internalGet(organizationId, study.getUid(), job.getParentId(), INCLUDE_JOB_IDS, + tokenPayload.getUserId(organizationId)).first(); + job.setParentId(tmpJob.getId()); + } catch (CatalogException e) { + throw new CatalogException("Parent job '" + job.getParentId() + "' not found", e); + } + } else { + job.setParentId(""); + } + + if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { + ParamUtils.checkDateFormat(job.getScheduledStartTime(), FieldConstants.JOB_SCHEDULED_START_TIME); + Date date = TimeUtils.toDate(job.getScheduledStartTime()); + if (date.before(new Date())) { + throw new CatalogException("'" + FieldConstants.JOB_SCHEDULED_START_TIME + "' must be a future date"); + } + } + if (job.getDependsOn() != null && !job.getDependsOn().isEmpty()) { boolean uuidProvided = job.getDependsOn().stream().map(Job::getId).anyMatch(UuidUtils::isOpenCgaUuid); @@ -357,13 +380,61 @@ private void autoCompleteNewJob(String organizationId, Study study, Job job, Str } else { // We only check input files if the job does not depend on other job that might be creating the necessary file. - List inputFiles = getJobInputFilesFromParams(study.getFqn(), job, token); + List inputFiles = getJobInputFilesFromParams(study.getFqn(), job, tokenPayload.getToken()); job.setInput(inputFiles); } job.setAttributes(ParamUtils.defaultObject(job.getAttributes(), HashMap::new)); } + public OpenCGAResult kill(String studyStr, String jobId, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = catalogManager.getStudyManager().resolveId(studyStr, userId, organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("study", studyStr) + .append("jobId", jobId) + .append("token", token); + String jobUuid = ""; + try { + QueryOptions options = keepFieldInQueryOptions(INCLUDE_JOB_IDS, JobDBAdaptor.QueryParams.USER_ID.key()); + Job job = internalGet(organizationId, study.getUid(), jobId, options, userId).first(); + jobId = job.getId(); + jobUuid = job.getUuid(); + + try { + if (!job.getUserId().equals(userId)) { + // Check if the user is a study administrator + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId); + } + } catch (CatalogException e) { + throw CatalogAuthorizationException.deny(userId, "job", jobId, CatalogAuthorizationException.ErrorCode.KILL_JOB, e); + } + + if (!job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.PENDING) + && !job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.QUEUED) + && !job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.RUNNING)) { + throw new CatalogException("Cannot kill job '" + jobId + "' in status " + job.getInternal().getStatus().getId()); + } + + ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_KILL_JOB_REQUESTED.key(), true); + OpenCGAResult update = catalogDBAdaptorFactory.getCatalogJobDBAdaptor(organizationId).update(job.getUid(), params, + QueryOptions.empty()); + + auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return update; + } catch (Exception e) { + auditManager.audit(organizationId, userId, Enums.Action.KILL_JOB, Enums.Resource.JOB, jobId, jobUuid, study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); + throw e; + } + } + + public List getJobInputFilesFromParams(String study, Job job, String token) throws CatalogException { // Look for input files String fileParamSuffix = "file"; @@ -408,7 +479,8 @@ public List getJobInputFilesFromParams(String study, Job job, String token } public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums.Priority priority, String jobId, String jobDescription, - List jobDependsOn, List jobTags, String token) throws CatalogException { + List jobDependsOn, List jobTags, String jobScheduledStartTime, String token) + throws CatalogException { Job job = get(studyStr, jobRetry.getJob(), new QueryOptions(), token).first(); if (jobRetry.isForce() || job.getInternal().getStatus().getId().equals(Enums.ExecutionStatus.ERROR) @@ -422,8 +494,8 @@ public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums. if (StringUtils.isEmpty(jobDescription)) { jobDescription = "Retry from job '" + jobRetry.getJob() + "'"; } - return submit(studyStr, job.getTool().getId(), priority, params, jobId, jobDescription, jobDependsOn, jobTags, - attributes, token); + return submit(studyStr, job.getTool().getId(), priority, params, jobId, jobDescription, jobDependsOn, jobTags, job.getId(), + jobScheduledStartTime, job.isDryRun(), attributes, token); } else { throw new CatalogException("Unable to retry job with status " + job.getInternal().getStatus().getId()); } @@ -431,7 +503,7 @@ public OpenCGAResult retry(String studyStr, JobRetryParams jobRetry, Enums. public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority priority, Map params, String token) throws CatalogException { - return submit(studyStr, toolId, priority, params, null, null, null, null, token); + return submit(studyStr, toolId, priority, params, null, null, null, null, null, null, false, token); } public OpenCGAResult submitProject(String projectStr, String toolId, Enums.Priority priority, @@ -449,18 +521,20 @@ public OpenCGAResult submitProject(String projectStr, String toolId, Enums. if (studies.isEmpty()) { throw new CatalogException("Project '" + projectStr + "' not found!"); } - return submit(studies.get(0), toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, token); + return submit(studies.get(0), toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, null, null, false, token); } public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority priority, Map params, String jobId, - String jobDescription, List jobDependsOn, List jobTags, String token) - throws CatalogException { - return submit(studyStr, toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, null, token); + String jobDescription, List jobDependsOn, List jobTags, @Nullable String jobParentId, + @Nullable String scheduledStartTime, Boolean dryRun, String token) throws CatalogException { + return submit(studyStr, toolId, priority, params, jobId, jobDescription, jobDependsOn, jobTags, jobParentId, scheduledStartTime, + dryRun, null, token); } public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority priority, Map params, String jobId, - String jobDescription, List jobDependsOn, List jobTags, Map attributes, - String token) throws CatalogException { + String jobDescription, List jobDependsOn, List jobTags, + @Nullable String jobParentId, @Nullable String scheduledStartTime, Boolean dryRun, + Map attributes, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); String organizationId = studyFqn.getOrganizationId(); @@ -470,14 +544,16 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority ObjectMap auditParams = new ObjectMap() .append("study", studyStr) .append("toolId", toolId) - .append("priority", priority) + .append("jobPriority", priority) .append("params", params) .append("jobId", jobId) .append("jobDescription", jobDescription) .append("jobDependsOn", jobDependsOn) .append("jobTags", jobTags) + .append("jobParentId", jobParentId) + .append("jobScheduledStartTime", scheduledStartTime) + .append("jobDryRun", dryRun) .append("token", token); - Job job = new Job(); job.setId(jobId); job.setDescription(jobDescription); @@ -486,13 +562,16 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority job.setStudy(new JobStudyParam(study.getFqn())); job.setUserId(userId); job.setParams(params); + job.setParentId(jobParentId); + job.setScheduledStartTime(scheduledStartTime); job.setPriority(priority); + job.setDryRun(dryRun != null && dryRun); job.setDependsOn(jobDependsOn != null ? jobDependsOn.stream().map(j -> new Job().setId(j)).collect(Collectors.toList()) : Collections.emptyList()); job.setAttributes(attributes); try { - autoCompleteNewJob(organizationId, study, job, token); + autoCompleteNewJob(organizationId, study, job, tokenPayload); authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.EXECUTE_JOBS); @@ -522,9 +601,9 @@ public OpenCGAResult submit(String studyStr, String toolId, Enums.Priority return jobResult; } - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditCreate(organizationId, userId, Enums.Resource.JOB, job.getId(), "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); job.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED)); job.getInternal().getStatus().setDescription(e.toString()); @@ -589,7 +668,7 @@ private OpenCGAResult getJobToReuse(String organizationId, Study study, Str private boolean jobEligibleToReuse(String inputJobId, Job job) { boolean enabled = configuration.getAnalysis().getExecution().getOptions() - .getBoolean("jobs.reuse.enabled", true); + .getBoolean(Execution.JOBS_REUSE_ENABLED, Execution.JOBS_REUSE_ENABLED_DEFAULT); if (!enabled) { return false; } @@ -600,9 +679,9 @@ private boolean jobEligibleToReuse(String inputJobId, Job job) { } List availableTools = configuration.getAnalysis().getExecution().getOptions() - .getAsStringList("jobs.reuse.tools"); + .getAsStringList(Execution.JOBS_REUSE_TOOLS); if (availableTools.isEmpty()) { - availableTools = Collections.singletonList("variant-.*"); + availableTools = Execution.JOBS_REUSE_TOOLS_DEFAULT; } String toolId = job.getTool().getId(); boolean validTool = false; @@ -671,9 +750,9 @@ public OpenCGAResult search(String studyId, Query query, QueryOptions optio new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditSearch(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -703,9 +782,9 @@ public OpenCGAResult distinct(String studyId, List fields, Query quer new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditDistinct(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -765,9 +844,9 @@ public OpenCGAResult count(String studyId, Query query, String token) throw return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditCount(organizationId, userId, Enums.Resource.JOB, study.getId(), study.getUuid(), auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -799,9 +878,9 @@ public OpenCGAResult delete(String studyStr, List jobIds, ObjectMap para // If the user is the owner or the admin, we won't check if he has permissions for every single entry long studyId = study.getUid(); checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -833,14 +912,14 @@ public OpenCGAResult delete(String studyStr, List jobIds, ObjectMap para auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot delete job {}: {}", jobId, e.getMessage(), e); auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.FAMILY, jobId, jobUuid, - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } auditManager.finishAuditBatch(organizationId, operationUuid); @@ -887,9 +966,9 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole // If the user is the owner or the admin, we won't check if he has permissions for every single entry long studyId1 = study.getUid(); checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId1, userId); - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -909,7 +988,7 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { String errorMsg = "Cannot delete job " + job.getId() + ": " + e.getMessage(); Event event = new Event(Event.Type.ERROR, job.getId(), e.getMessage()); @@ -918,7 +997,7 @@ public OpenCGAResult delete(String studyId, Query query, ObjectMap params, boole logger.error(errorMsg, e); auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } auditManager.finishAuditBatch(organizationId, operationUuid); @@ -978,23 +1057,17 @@ public OpenCGAResult log(String studyId, String jobId, long offset, logFile = Paths.get(job.getStderr().getUri()); } else { // The log file hasn't yet been registered - if (!Arrays.asList(Enums.ExecutionStatus.PENDING, Enums.ExecutionStatus.QUEUED, Enums.ExecutionStatus.ABORTED) - .contains(job.getInternal().getStatus().getId()) && job.getOutDir() != null) { - logFile = Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".err"); - } else { - throw CatalogAuthorizationException.deny(userId, "see stderr log file of job '" + jobId + "'"); - } + logFile = Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".err"); } } else { if (job.getStdout() != null && job.getStdout().getUri() != null) { logFile = Paths.get(job.getStdout().getUri()); } else { - // The log file hasn't yet been registered - if (!Arrays.asList(Enums.ExecutionStatus.PENDING, Enums.ExecutionStatus.QUEUED, Enums.ExecutionStatus.ABORTED) - .contains(job.getInternal().getStatus().getId()) && job.getOutDir() != null) { + if (job.getOutDir() != null && job.getOutDir().getUri() != null) { + // The log file hasn't yet been registered logFile = Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".log"); } else { - throw CatalogAuthorizationException.deny(userId, "see stdout log file of job '" + jobId + "'"); + throw CatalogAuthorizationException.deny(userId, "view log file"); } } } @@ -1005,6 +1078,12 @@ public OpenCGAResult log(String studyId, String jobId, long offset, } catch (IOException e) { throw CatalogIOException.ioManagerException(logFile.toUri(), e); } + + if (!ioManager.exists(logFile.toUri())) { + String status = job.getInternal().getStatus().getId(); + throw new CatalogException("Job '" + jobId + "' in status '" + status + "'. Log file '" + type + "' not found"); + } + FileContent fileContent; if (tail) { fileContent = ioManager.tail(logFile, lines); @@ -1017,9 +1096,9 @@ public OpenCGAResult log(String studyId, String jobId, long offset, return new OpenCGAResult<>((int) (System.currentTimeMillis() - startTime), Collections.emptyList(), 1, Collections.singletonList(fileContent), 1); - } catch (CatalogException e) { + } catch (Exception e) { auditManager.audit(organizationId, userId, Enums.Action.VIEW_LOG, Enums.Resource.JOB, jobId, "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } } @@ -1062,9 +1141,9 @@ public OpenCGAResult update(String studyStr, Query query, JobUpdateParams u finalQuery.append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); iterator = getJobDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1078,14 +1157,14 @@ public OpenCGAResult update(String studyStr, Query query, JobUpdateParams u auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, job.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update job {}: {}", job.getId(), e.getMessage(), e); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } auditManager.finishAuditBatch(organizationId, operationId); @@ -1140,14 +1219,14 @@ public OpenCGAResult update(String studyStr, List jobIds, JobUpdate auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update job {}: {}", jobId, e.getMessage(), e); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } auditManager.finishAuditBatch(organizationId, operationId); @@ -1215,14 +1294,14 @@ public OpenCGAResult update(String studyStr, String jobId, JobUpdateParams auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Could not update job {}: {}", jobId, e.getMessage(), e); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1344,9 +1423,9 @@ public OpenCGAResult update(String studyId, Query query, ObjectMap paramete finalQuery.append(FamilyDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); iterator = getJobDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_JOB_IDS, userId); - } catch (CatalogException e) { + } catch (Exception e) { auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1363,14 +1442,14 @@ public OpenCGAResult update(String studyId, Query query, ObjectMap paramete auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, job.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update job {}: {}", job.getId(), e.getMessage()); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } @@ -1418,14 +1497,14 @@ public OpenCGAResult update(String studyId, String jobId, ObjectMap paramet auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update job {}: {}", jobId, e.getMessage()); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); throw e; } @@ -1482,14 +1561,14 @@ public OpenCGAResult update(String studyId, List jobIds, ObjectMap auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, job.getId(), job.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); - } catch (CatalogException e) { + } catch (Exception e) { Event event = new Event(Event.Type.ERROR, jobId, e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error("Cannot update job {}: {}", jobId, e.getMessage()); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.JOB, jobId, jobUuid, study.getId(), - study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e)); } } @@ -1807,10 +1886,10 @@ public OpenCGAResult> getAcls(String studyId, List< } jobAcls.setResults(resultList); jobAcls.setEvents(eventList); - } catch (CatalogException e) { + } catch (Exception e) { for (String jobId : jobList) { auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.JOB, jobId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e), new ObjectMap()); } if (!ignoreException) { @@ -1909,12 +1988,12 @@ public OpenCGAResult> updateAcl(String studyId, Lis new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } return queryResultList; - } catch (CatalogException e) { + } catch (Exception e) { if (jobStrList != null) { for (String jobId : jobStrList) { auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.JOB, jobId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, - e.getError()), new ObjectMap()); + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e), + new ObjectMap()); } } throw e; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java index f89d0595c30..90b83a92728 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/OrganizationManager.java @@ -1,6 +1,7 @@ package org.opencb.opencga.catalog.managers; import com.fasterxml.jackson.core.JsonProcessingException; +import io.jsonwebtoken.security.InvalidKeyException; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.Event; @@ -17,6 +18,8 @@ import org.opencb.opencga.catalog.exceptions.CatalogIOException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.io.CatalogIOManager; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.JwtUtils; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -24,16 +27,23 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.config.Optimizations; import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.organizations.*; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.util.*; +import java.util.stream.Collectors; + +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; public class OrganizationManager extends AbstractManager { @@ -257,33 +267,6 @@ public OpenCGAResult update(String organizationId, OrganizationUpd // We set the proper values for the audit organizationId = organization.getId(); - // Avoid duplicated authentication origin ids. - // Validate all mandatory authentication origin fields are filled in. - if (updateParams.getConfiguration() != null && updateParams.getConfiguration().getAuthenticationOrigins() != null) { - String authOriginsPrefixKey = OrganizationDBAdaptor.QueryParams.CONFIGURATION_AUTHENTICATION_ORIGINS.key(); - boolean internal = false; - Set authenticationOriginIds = new HashSet<>(); - for (AuthenticationOrigin authenticationOrigin : updateParams.getConfiguration().getAuthenticationOrigins()) { - if (authenticationOrigin.getType().equals(AuthenticationOrigin.AuthenticationType.OPENCGA)) { - if (internal) { - throw new CatalogException("Found duplicated authentication origin of type OPENCGA."); - } - internal = true; - // Set id to INTERNAL - authenticationOrigin.setId(CatalogAuthenticationManager.OPENCGA); - } - ParamUtils.checkIdentifier(authenticationOrigin.getId(), authOriginsPrefixKey + ".id"); - ParamUtils.checkObj(authenticationOrigin.getType(), authOriginsPrefixKey + ".type"); - if (authenticationOriginIds.contains(authenticationOrigin.getId())) { - throw new CatalogException("Found duplicated authentication origin id '" + authenticationOrigin.getId() + "'."); - } - authenticationOriginIds.add(authenticationOrigin.getId()); - } - if (!internal) { - throw new CatalogException("Missing mandatory AuthenticationOrigin of type OPENCGA."); - } - } - result = getOrganizationDBAdaptor(organizationId).update(organizationId, updateMap, options); auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organization.getId(), organization.getUuid(), "", @@ -309,6 +292,243 @@ public OpenCGAResult update(String organizationId, OrganizationUpd return result; } + public OpenCGAResult updateUser(@Nullable String organizationId, String userId, OrganizationUserUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("userId", userId) + .append("updateParams", updateParams) + .append("options", options) + .append("token", token); + + options = ParamUtils.defaultObject(options, QueryOptions::new); + String myOrganizationId = StringUtils.isNotEmpty(organizationId) ? organizationId : tokenPayload.getOrganization(); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(myOrganizationId, tokenPayload.getUserId(myOrganizationId)); + ParamUtils.checkObj(updateParams, "OrganizationUserUpdateParams"); + getUserDBAdaptor(myOrganizationId).checkId(userId); + + if (StringUtils.isNotEmpty(updateParams.getEmail())) { + ParamUtils.checkEmail(updateParams.getEmail()); + } + if (updateParams.getQuota() != null) { + if (updateParams.getQuota().getDiskUsage() < 0) { + throw new CatalogException("Disk usage cannot be negative"); + } + if (updateParams.getQuota().getCpuUsage() < 0) { + throw new CatalogException("CPU usage cannot be negative"); + } + if (updateParams.getQuota().getMaxDisk() < 0) { + throw new CatalogException("Max disk cannot be negative"); + } + if (updateParams.getQuota().getMaxCpu() < 0) { + throw new CatalogException("Max CPU cannot be negative"); + } + } + if (updateParams.getAccount() != null && StringUtils.isNotEmpty(updateParams.getAccount().getExpirationDate())) { + ParamUtils.checkDateIsNotExpired(updateParams.getAccount().getExpirationDate(), "expirationDate"); + } + + ObjectMap updateMap; + try { + updateMap = updateParams.getUpdateMap(); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse OrganizationUserUpdateParams object: " + e.getMessage(), e); + } + OpenCGAResult updateResult = getUserDBAdaptor(myOrganizationId).update(userId, updateMap); + auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated user + OpenCGAResult result = getUserDBAdaptor(myOrganizationId).get(userId, options); + updateResult.setResults(result.getResults()); + } + + return updateResult; + } catch (CatalogException e) { + auditManager.auditUpdate(myOrganizationId, tokenPayload.getUserId(myOrganizationId), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + + public OpenCGAResult updateConfiguration(String organizationId, OrganizationConfiguration updateParams, + QueryOptions options, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + String userId = tokenPayload.getUserId(organizationId); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("updateParams", updateParams) + .append("options", options) + .append("token", token); + + QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); + OpenCGAResult result = OpenCGAResult.empty(OrganizationConfiguration.class); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId); + + ParamUtils.checkObj(updateParams, "OrganizationConfiguration"); + ObjectMap updateConfigurationMap; + try { + updateConfigurationMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(updateParams)); + } catch (JsonProcessingException e) { + throw new CatalogException("Could not parse OrganizationConfiguration object: " + e.getMessage(), e); + } + + OpenCGAResult internalResult = get(organizationId, INCLUDE_ORGANIZATION_CONFIGURATION, token); + if (internalResult.getNumResults() == 0) { + throw new CatalogException("Organization '" + organizationId + "' not found"); + } + Organization organization = internalResult.first(); + + // We set the proper values for the audit + organizationId = organization.getId(); + + if (CollectionUtils.isNotEmpty(updateParams.getAuthenticationOrigins())) { + // Check action + ParamUtils.UpdateAction authOriginsAction = null; + Map map = queryOptions.getMap(Constants.ACTIONS); + if (map == null || map.get(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD) == null) { + // Write default option + authOriginsAction = ParamUtils.UpdateAction.ADD; + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, authOriginsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + } else { + authOriginsAction = ParamUtils.UpdateAction.valueOf(map.get(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD).toString()); + } + + Set currentAuthOriginIds = organization.getConfiguration().getAuthenticationOrigins() + .stream() + .map(AuthenticationOrigin::getId) + .collect(Collectors.toSet()); + + Set updateAuthOriginIds = new HashSet<>(); + StringBuilder authOriginUpdateBuilder = new StringBuilder(); + List authenticationOrigins = updateParams.getAuthenticationOrigins(); + for (int i = 0; i < authenticationOrigins.size(); i++) { + AuthenticationOrigin authenticationOrigin = authenticationOrigins.get(i); + ParamUtils.checkParameter(authenticationOrigin.getId(), "AuthenticationOrigin id"); + ParamUtils.checkObj(authenticationOrigin.getType(), "AuthenticationOrigin type"); + if (updateAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogParameterException("Found duplicated authentication origin id '" + authenticationOrigin.getId() + + "'."); + } + // Check authOrigin OPENCGA-OPENCGA + if ((authenticationOrigin.getType().equals(AuthenticationOrigin.AuthenticationType.OPENCGA) + && !CatalogAuthenticationManager.OPENCGA.equals(authenticationOrigin.getId())) + || (!authenticationOrigin.getType().equals(AuthenticationOrigin.AuthenticationType.OPENCGA) + && CatalogAuthenticationManager.OPENCGA.equals(authenticationOrigin.getId()))) { + throw new CatalogParameterException("AuthenticationOrigin type '" + AuthenticationOrigin.AuthenticationType.OPENCGA + + "' must go together with id '" + CatalogAuthenticationManager.OPENCGA + "'."); + } + updateAuthOriginIds.add(authenticationOrigin.getId()); + if (i > 0) { + authOriginUpdateBuilder.append(", "); + } + authOriginUpdateBuilder.append(authenticationOrigin.getType()).append(": ").append(authenticationOrigin.getId()); + + if (authOriginsAction != ParamUtils.UpdateAction.REMOVE) { + // Validate configuration is correct and can be used + authenticationFactory.validateAuthenticationOrigin(authenticationOrigin); + } + } + + switch (authOriginsAction) { + case ADD: + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (currentAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogException("Authentication origin '" + authenticationOrigin.getId() + "' already exists. " + + "Please, set the authOriginsAction to 'REPLACE' to replace the current configuration."); + } + } + logger.debug("Adding new list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + case SET: + boolean userAuthOriginFound = false; + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (tokenPayload.getAuthOrigin().equals(authenticationOrigin.getType())) { + userAuthOriginFound = true; + } + } + if (!userAuthOriginFound) { + throw new CatalogException("User authentication origin not found in the list of authentication origins. " + + "Please, add an AuthenticationOrigin of type '" + tokenPayload.getAuthOrigin() + "' to the list."); + } + logger.debug("Set new list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + case REMOVE: + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (!currentAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogException("Authentication origin '" + authenticationOrigin.getId() + "' does not exist. " + + "The current available authentication origin ids are: '" + + StringUtils.join(currentAuthOriginIds, ", ") + "'."); + } + if (tokenPayload.getAuthOrigin().equals(authenticationOrigin.getType())) { + throw new CatalogException("Removing the authentication origin '" + tokenPayload.getAuthOrigin() + "' " + + "not allowed. Your user account uses that AuthenticationOrigin."); + } + } + logger.debug("Removing list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + case REPLACE: + for (AuthenticationOrigin authenticationOrigin : updateParams.getAuthenticationOrigins()) { + if (!currentAuthOriginIds.contains(authenticationOrigin.getId())) { + throw new CatalogException("Authentication origin '" + authenticationOrigin.getId() + "' not found." + + "Please, set the authOriginsAction to 'ADD' to add the new AuthenticationOrigin."); + } + } + logger.debug("Replace list of Authentication Origins: {}.", authOriginUpdateBuilder); + break; + default: + throw new CatalogParameterException("Unknown authentication origins action " + authOriginsAction); + } + } + + if (updateParams.getToken() != null) { + try { + JwtUtils.validateJWTKey(updateParams.getToken().getAlgorithm(), updateParams.getToken().getSecretKey()); + } catch (InvalidKeyException e) { + throw new CatalogParameterException("Invalid secret key - algorithm for JWT token: " + e.getMessage(), e); + } + if (updateParams.getToken().getExpiration() <= 0) { + throw new CatalogParameterException("Invalid expiration for JWT token. It must be a positive number."); + } + } + + ObjectMap updateMap = new ObjectMap(OrganizationDBAdaptor.QueryParams.CONFIGURATION.key(), updateConfigurationMap); + OpenCGAResult update = getOrganizationDBAdaptor(organizationId).update(organizationId, updateMap, queryOptions); + result.append(update); + auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organization.getId(), organization.getUuid(), "", + "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + organization = null; + if (queryOptions.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated organization + organization = getOrganizationDBAdaptor(organizationId).get(INCLUDE_ORGANIZATION_CONFIGURATION).first(); + result.setResults(Collections.singletonList(organization.getConfiguration())); + } + + if (CollectionUtils.isNotEmpty(updateParams.getAuthenticationOrigins()) || updateParams.getToken() != null) { + if (organization == null) { + organization = getOrganizationDBAdaptor(organizationId).get(INCLUDE_ORGANIZATION_CONFIGURATION).first(); + } + authenticationFactory.configureOrganizationAuthenticationManager(organization); + } + + } catch (Exception e) { + auditManager.auditUpdate(organizationId, userId, Enums.Resource.ORGANIZATION, organizationId, organizationId, "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "Update organization", + e.getMessage()))); + throw e; + } + return result; + } + private void validateOrganizationForCreation(Organization organization, String userId) throws CatalogParameterException { ParamUtils.checkParameter(organization.getId(), OrganizationDBAdaptor.QueryParams.ID.key()); @@ -327,6 +547,12 @@ private void validateOrganizationForCreation(Organization organization, String u if (organization.getConfiguration() == null) { organization.setConfiguration(new OrganizationConfiguration()); } + validateOrganizationConfiguration(organization); + + organization.setAttributes(ParamUtils.defaultObject(organization.getAttributes(), HashMap::new)); + } + + private void validateOrganizationConfiguration(Organization organization) throws CatalogParameterException { if (CollectionUtils.isNotEmpty(organization.getConfiguration().getAuthenticationOrigins())) { for (AuthenticationOrigin authenticationOrigin : organization.getConfiguration().getAuthenticationOrigins()) { ParamUtils.checkParameter(authenticationOrigin.getId(), "AuthenticationOrigin id"); @@ -340,7 +566,11 @@ private void validateOrganizationForCreation(Organization organization, String u || StringUtils.isEmpty(organization.getConfiguration().getToken().getSecretKey())) { organization.getConfiguration().setToken(TokenConfiguration.init()); } - organization.setAttributes(ParamUtils.defaultObject(organization.getAttributes(), HashMap::new)); + organization.getConfiguration().setDefaultUserExpirationDate(ParamUtils.defaultString( + organization.getConfiguration().getDefaultUserExpirationDate(), Constants.DEFAULT_USER_EXPIRATION_DATE)); + if (organization.getConfiguration().getOptimizations() == null) { + organization.getConfiguration().setOptimizations(new Optimizations(false)); + } } Set getOrganizationOwnerAndAdmins(String organizationId) throws CatalogException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index a782ea23ed2..beda16bfa2e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -220,8 +220,13 @@ private void validateProjectForCreation(String organizationId, Project project) throw new CatalogParameterException("Missing mandatory organism information"); } try { + //TODO: Should the datarelease be undefined? When undefined, it'd be read from cellbase meta endpoints. + String defaultDataRelease = project.getOrganism().getAssembly().equalsIgnoreCase("grch38") + ? ParamConstants.CELLBASE_DATA_RELEASE_GRCH38 + : null; CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(), - new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION)); + new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, + defaultDataRelease, ParamConstants.CELLBASE_APIKEY)); cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, project.getOrganism().getScientificName(), project.getOrganism().getAssembly(), true); project.setCellbase(cellBaseConfiguration); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java index d882015eae5..4c9f03d12e1 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java @@ -40,6 +40,7 @@ import org.opencb.opencga.core.models.JwtPayload; import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.GroupUpdateParams; import org.opencb.opencga.core.models.user.*; @@ -59,8 +60,8 @@ */ public class UserManager extends AbstractManager { - static final QueryOptions INCLUDE_ACCOUNT = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( - UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.ACCOUNT.key())); + static final QueryOptions INCLUDE_ACCOUNT_AND_INTERNAL = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( + UserDBAdaptor.QueryParams.ID.key(), UserDBAdaptor.QueryParams.ACCOUNT.key(), UserDBAdaptor.QueryParams.INTERNAL.key())); protected static Logger logger = LoggerFactory.getLogger(UserManager.class); private final CatalogIOManager catalogIOManager; private final AuthenticationFactory authenticationFactory; @@ -118,6 +119,9 @@ public OpenCGAResult create(User user, String password, String token) thro throw new CatalogException("Creating '" + OPENCGA + "' user is forbidden in any organization."); } + Organization organization = getOrganizationDBAdaptor(organizationId).get(OrganizationManager.INCLUDE_ORGANIZATION_CONFIGURATION) + .first(); + ObjectMap auditParams = new ObjectMap("user", user); // Initialise fields @@ -130,7 +134,15 @@ public OpenCGAResult create(User user, String password, String token) thro } user.setAccount(ParamUtils.defaultObject(user.getAccount(), Account::new)); user.getAccount().setCreationDate(TimeUtils.getTime()); - user.getAccount().setExpirationDate(ParamUtils.defaultString(user.getAccount().getExpirationDate(), "")); + if (StringUtils.isEmpty(user.getAccount().getExpirationDate())) { + // By default, user accounts will be valid for 1 year when they are created. + user.getAccount().setExpirationDate(organization.getConfiguration().getDefaultUserExpirationDate()); + Date date = TimeUtils.add1YeartoDate(new Date()); + user.getAccount().setExpirationDate(TimeUtils.getTime(date)); + } else { + // Validate expiration date is not over + ParamUtils.checkDateIsNotExpired(user.getAccount().getExpirationDate(), "account.expirationDate"); + } user.setInternal(new UserInternal(new UserStatus(UserStatus.READY))); user.setQuota(ParamUtils.defaultObject(user.getQuota(), UserQuota::new)); user.setProjects(ParamUtils.defaultObject(user.getProjects(), Collections::emptyList)); @@ -253,7 +265,8 @@ public JwtPayload validateToken(String token) throws CatalogException { if (ParamConstants.ANONYMOUS_USER_ID.equals(jwtPayload.getUserId())) { authOrigin = CatalogAuthenticationManager.OPENCGA; } else { - OpenCGAResult userResult = getUserDBAdaptor(jwtPayload.getOrganization()).get(jwtPayload.getUserId(), INCLUDE_ACCOUNT); + OpenCGAResult userResult = getUserDBAdaptor(jwtPayload.getOrganization()).get(jwtPayload.getUserId(), + INCLUDE_ACCOUNT_AND_INTERNAL); if (userResult.getNumResults() == 0) { throw new CatalogException("User '" + jwtPayload.getUserId() + "' could not be found."); } @@ -631,7 +644,7 @@ public OpenCGAResult update(String userId, ObjectMap parameters, QueryOpti userId = getValidUserId(userId, payload); for (String s : parameters.keySet()) { - if (!s.matches("name|email|attributes")) { + if (!s.matches("name|email")) { throw new CatalogDBException("Parameter '" + s + "' can't be changed"); } } @@ -739,7 +752,7 @@ public OpenCGAResult resetPassword(String userId, String token) throws CatalogEx JwtPayload jwtPayload = validateToken(token); String organizationId = jwtPayload.getOrganization(); try { - authorizationManager.checkIsOpencgaAdministrator(jwtPayload, "reset password"); + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, jwtPayload.getUserId()); String authOrigin = getAuthenticationOriginId(organizationId, userId); OpenCGAResult writeResult = authenticationFactory.resetPassword(organizationId, authOrigin, userId); @@ -778,16 +791,55 @@ public AuthenticationResponse login(String organizationId, String username, Stri } } - OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_ACCOUNT); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(username, INCLUDE_ACCOUNT_AND_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { + User user = userOpenCGAResult.first(); + // Only local OPENCGA users that are not superadmins can be automatically banned or their accounts be expired + boolean userCanBeBanned = !ParamConstants.ADMIN_ORGANIZATION.equals(organizationId) + && CatalogAuthenticationManager.OPENCGA.equals(user.getAccount().getAuthentication().getId()); + // We check + if (userCanBeBanned) { + // Check user is not banned, suspended or has an expired account + if (UserStatus.BANNED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsBanned(username); + } + Date date = TimeUtils.toDate(user.getAccount().getExpirationDate()); + if (date == null) { + throw new CatalogException("Unexpected null expiration date for user '" + username + "'."); + } + if (date.before(new Date())) { + throw CatalogAuthenticationException.accountIsExpired(username, user.getAccount().getExpirationDate()); + } + } + if (UserStatus.SUSPENDED.equals(user.getInternal().getStatus().getId())) { + throw CatalogAuthenticationException.userIsSuspended(username); + } authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); try { response = authenticationFactory.authenticate(organizationId, authId, username, password); } catch (CatalogAuthenticationException e) { + if (userCanBeBanned) { + // We can only lock the account if it is not the root user + int failedAttempts = userOpenCGAResult.first().getInternal().getFailedAttempts(); + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), failedAttempts + 1); + if (failedAttempts >= (configuration.getMaxLoginAttempts() - 1)) { + // Ban the account + updateParams.append(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), UserStatus.BANNED); + } + getUserDBAdaptor(organizationId).update(username, updateParams); + } + auditManager.auditUser(organizationId, username, Enums.Action.LOGIN, username, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } + + // If it was a local user and the counter of failed attempts was greater than 0, we reset it + if (userCanBeBanned && userOpenCGAResult.first().getInternal().getFailedAttempts() > 0) { + // Reset login failed attempts counter + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); + getUserDBAdaptor(organizationId).update(username, updateParams); + } } else { // We attempt to login the user with the different authentication managers for (Map.Entry entry @@ -895,6 +947,70 @@ public AuthenticationResponse refreshToken(String token) throws CatalogException return response; } + public OpenCGAResult changeStatus(String organizationId, String userId, String status, QueryOptions options, String token) + throws CatalogException { + JwtPayload tokenPayload = validateToken(token); + String userIdOrganization = StringUtils.isNotEmpty(organizationId) ? organizationId : tokenPayload.getOrganization(); + + ObjectMap auditParams = new ObjectMap() + .append("organizationId", organizationId) + .append("userId", userId) + .append("status", status) + .append("options", options) + .append("token", token); + try { + authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(userIdOrganization, tokenPayload.getUserId(userIdOrganization)); + options = ParamUtils.defaultObject(options, QueryOptions::new); + + // Validate user exists + getUserDBAdaptor(userIdOrganization).checkId(userId); + + // Validate status is valid + if (!UserStatus.READY.equals(status) && !UserStatus.SUSPENDED.equals(status)) { + throw new CatalogParameterException("Invalid status '" + status + "'. Valid values are: " + UserStatus.READY + ", " + + UserStatus.SUSPENDED); + } + + if (UserStatus.SUSPENDED.equals(status)) { + // Get organization information + Set ownerAndAdmins = catalogManager.getOrganizationManager().getOrganizationOwnerAndAdmins(userIdOrganization); + if (ownerAndAdmins.contains(userId)) { + if (tokenPayload.getUserId().equals(userId)) { + // The user is trying to suspend himself + throw new CatalogAuthorizationException("You can't suspend your own account."); + } + if (!authorizationManager.isAtLeastOrganizationOwner(userIdOrganization, tokenPayload.getUserId(userIdOrganization))) { + // One of the admins is trying to suspend the owner or one of the admins + throw new CatalogAuthorizationException("Only the owner of the organization can suspend administrators."); + } + } + } + + // Update user status and reset failed attempts to 0 + ObjectMap updateParams = new ObjectMap(UserDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), status); + if (UserStatus.READY.equals(status)) { + updateParams.put(UserDBAdaptor.QueryParams.INTERNAL_FAILED_ATTEMPTS.key(), 0); + } + OpenCGAResult result = getUserDBAdaptor(userIdOrganization).update(userId, updateParams); + + auditManager.auditUpdate(organizationId, tokenPayload.getUserId(userIdOrganization), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch updated user + OpenCGAResult tmpResult = getUserDBAdaptor(userIdOrganization).get(userId, options); + result.setResults(tmpResult.getResults()); + } + + return result; + } catch (Exception e) { + auditManager.auditUpdate(organizationId, tokenPayload.getUserId(userIdOrganization), Enums.Resource.USER, userId, "", "", "", + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(-1, "Could not update user status", + e.getMessage()))); + throw e; + } + } + /** * This method will be only callable by the system. It generates a new session id for the user. * @@ -940,7 +1056,7 @@ public String getNonExpiringToken(String organizationId, String userId, Map userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_ACCOUNT); + OpenCGAResult userOpenCGAResult = getUserDBAdaptor(organizationId).get(user, INCLUDE_ACCOUNT_AND_INTERNAL); if (userOpenCGAResult.getNumResults() == 1) { String authId = userOpenCGAResult.first().getAccount().getAuthentication().getId(); return authenticationFactory.getOrganizationAuthenticationManager(organizationId, authId); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java index 7fb3b1dabf8..3c5ddea26d3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/Migration.java @@ -54,6 +54,11 @@ */ boolean manual() default false; + /** + * @return the version when the migration was deprecated. + */ + String deprecatedSince() default ""; + enum MigrationDomain { CATALOG, STORAGE diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java index e59e1e659e3..000c46d57dc 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationException.java @@ -21,8 +21,9 @@ public static MigrationException offlineMigrationException(Migration migration) + "cannot be accessed and run try again with '--offline' flag."); } - public static MigrationException outdatedMigration(Migration migration, String version) { - return new MigrationException("Migration '" + migration.id() + "' is outdated since version " + version + ". Please, update the " - + "migration from the previous latest version."); + public static MigrationException deprecatedMigration(Migration migration) { + return new MigrationException("Migration '" + migration.id() + "' can't be run since version '" + + migration.deprecatedSince() + "'. Please, run this migration from a previous OpenCGA version."); } + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index 7809f4fb0f7..fa8fd7b9ee0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -125,14 +125,20 @@ public void runMigration(String version, Collection d public void runMigration(String version, Collection domains, Collection languages, boolean offline, String appHome, ObjectMap params, String token) throws CatalogException, IOException { - // Migrate all organizations - for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { - if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { - runMigration(organizationId, version, domains, languages, offline, appHome, params, token); + runMigration(ParamConstants.ADMIN_ORGANIZATION, version, domains, languages, offline, appHome, params, token); + + // ***** Starts code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ + try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory())) { + for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { + // ***** Finish code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ + + // Migrate all organizations + // for (String organizationId : dbAdaptorFactory.getOrganizationIds()) { + if (!ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + runMigration(organizationId, version, domains, languages, offline, appHome, params, token); + } } } - // Lastly, migrate the admin organization - runMigration(ParamConstants.ADMIN_ORGANIZATION, version, domains, languages, offline, appHome, params, token); } public void runMigration(String organizationId, String version, Collection domains, @@ -622,6 +628,10 @@ private MigrationRun run(String organizationId, Class r String token) throws MigrationException { Migration annotation = getMigrationAnnotation(runnableMigration); + if (StringUtils.isNotEmpty(annotation.deprecatedSince())) { + throw MigrationException.deprecatedMigration(annotation); + } + MigrationTool migrationTool; try { migrationTool = runnableMigration.newInstance(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java index 47a701f0ed2..219e1178652 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationTool.java @@ -4,6 +4,7 @@ import com.mongodb.client.MongoCursor; import com.mongodb.client.model.IndexOptions; import com.mongodb.client.model.WriteModel; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.commons.ProgressLogger; @@ -78,6 +79,10 @@ public final void setup(Configuration configuration, CatalogManager catalogManag public final void execute() throws MigrationException { try { + Migration annotation = getAnnotation(); + if (StringUtils.isNotEmpty(annotation.deprecatedSince())) { + throw MigrationException.deprecatedMigration(annotation); + } run(); } catch (MigrationException e) { throw e; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java index ec210c0cd79..d56f598dabb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java @@ -17,8 +17,10 @@ package org.opencb.opencga.catalog.utils; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.study.Group; @@ -48,7 +50,7 @@ private CatalogDemo() { */ public static void createDemoDatabase(CatalogManager catalogManager, String organizationId, String adminPassword, boolean force) throws CatalogException { - catalogManager.installCatalogDB("HS256", catalogManager.getConfiguration().getAdmin().getSecretKey(), adminPassword, + catalogManager.installCatalogDB("HS256", PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH), adminPassword, "opencga@admin.com", force); String token = catalogManager.getUserManager().loginAsAdmin(adminPassword).getToken(); try { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java index 543502a6f87..8b1f4ac747e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/Constants.java @@ -111,4 +111,6 @@ public class Constants { */ public static final String JOB_DELETED_OUTPUT_DIRECTORY = "deletedOutputFiles"; + public static final String DEFAULT_USER_EXPIRATION_DATE = "21000101000000"; + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java index 7d6eb326acc..5e93414f08d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/ParamUtils.java @@ -124,9 +124,9 @@ public static void checkValidUserId(String userId) throws CatalogParameterExcept if (userId.equals(ParamConstants.ANONYMOUS_USER_ID) || userId.equals(ParamConstants.REGISTERED_USERS)) { throw new CatalogParameterException("User id cannot be one of the reserved OpenCGA users."); } - if (!userId.matches("^[A-Za-z0-9]([-_.]?[A-Za-z0-9])*$")) { + if (!userId.matches("^[A-Za-z0-9]([-_.@]?[A-Za-z0-9])*$")) { throw new CatalogParameterException("Invalid user id. Id needs to start by any character and might contain single '-', '_', " - + "'.', symbols followed by any character or number."); + + "'.' or '@' symbols followed by any character or number."); } } @@ -174,6 +174,22 @@ public static void checkDateFormat(String creationDate, String param) throws Cat } } + public static void checkDateIsNotExpired(String dateStr, String param) throws CatalogParameterException { + if (StringUtils.isEmpty(dateStr)) { + throw CatalogParameterException.isNull(param); + } else { + // Validate date can be parsed and has the proper format + Date date = TimeUtils.toDate(dateStr); + if (date == null || dateStr.length() != 14) { + throw new CatalogParameterException("Unexpected '" + param + "' format. Expected format is 'yyyyMMddHHmmss'"); + } + if (date.before(TimeUtils.getDate())) { + throw new CatalogParameterException("The date for '" + param + "' introduced is already expired. Please, introduce a valid" + + " date in the future."); + } + } + } + public static String checkDateOrGetCurrentDate(String date, String param) throws CatalogParameterException { if (StringUtils.isEmpty(date)) { return TimeUtils.getTime(); diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index 6cabeed184d..557a660716a 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -37,8 +37,8 @@ {"collections": ["job"], "fields": {"outDir.uid": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"visited": 1, "studyUid": 1}, "options": {}} -{"collections": ["job"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["job"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"internal.status.id": 1, "_priority": 1, "_creationDate": 1}, "options": {}} {"collections": ["job"], "fields": {"_priority": 1, "_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_priority": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} @@ -65,8 +65,8 @@ {"collections": ["file"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"jobId": 1, "studyUid": 1}, "options": {}} -{"collections": ["file"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["file"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["file"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} {"collections": ["file"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} @@ -104,8 +104,8 @@ {"collections": ["sample", "sample_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["sample", "sample_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["sample", "sample_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"internal.rga.status": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} @@ -138,8 +138,8 @@ {"collections": ["individual", "individual_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["individual", "individual_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["individual", "individual_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"uuid": 1}, "options": {"unique": true}} {"collections": ["cohort"], "fields": {"uid": 1}, "options": {"unique": true}} @@ -158,8 +158,8 @@ {"collections": ["cohort"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["cohort"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["cohort"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["family", "family_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -182,8 +182,8 @@ {"collections": ["family", "family_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["family", "family_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["family", "family_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["panel", "panel_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -201,11 +201,11 @@ {"collections": ["panel", "panel_archive"], "fields": {"categories.name": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} -{"collections": ["panel", "panel_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} -{"collections": ["panel", "panel_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["clinical"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} {"collections": ["clinical"], "fields": {"uuid": 1}, "options": {"unique": true}} @@ -231,7 +231,7 @@ {"collections": ["clinical"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["clinical"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} {"collections": ["clinical"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -246,7 +246,7 @@ {"collections": ["interpretation", "interpretation_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "release": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java index 2e9887f85fb..c9d2ca8aec4 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authentication/JwtSessionManagerTest.java @@ -53,7 +53,7 @@ public void setUp() throws Exception { @Test public void testCreateJWTToken() throws Exception { - jwtToken = jwtSessionManager.createJWTToken(organizationId, "testUser", Collections.emptyMap(), 60L); + jwtToken = jwtSessionManager.createJWTToken(organizationId, null, "testUser", Collections.emptyMap(), 60L); } @Test @@ -81,7 +81,7 @@ public void testInvalidSecretKey() throws CatalogAuthenticationException { @Test public void testNonExpiringToken() throws CatalogException { - String nonExpiringToken = jwtSessionManager.createJWTToken(organizationId, "System", null, -1L); + String nonExpiringToken = jwtSessionManager.createJWTToken(organizationId, null, "System", null, -1L); assertEquals(jwtSessionManager.getUser(nonExpiringToken), "System"); assertNull(jwtSessionManager.getExpiration(nonExpiringToken)); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java index 3af2d35be82..2de27beb3da 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractClinicalManagerTest.java @@ -16,6 +16,7 @@ package org.opencb.opencga.catalog.managers; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.experimental.categories.Category; @@ -23,6 +24,7 @@ import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.test.GenericTest; import org.opencb.opencga.catalog.exceptions.CatalogException; @@ -35,6 +37,7 @@ import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import java.io.IOException; @@ -46,6 +49,8 @@ import java.util.Collections; import java.util.List; +import static org.opencb.commons.datastore.core.QueryOptions.INCLUDE; + @Category(MediumTests.class) public class AbstractClinicalManagerTest extends GenericTest { @@ -57,7 +62,10 @@ public class AbstractClinicalManagerTest extends GenericTest { public final static String PROBAND_ID2 = "manuel_individual"; public final static String CA_ID3 = "clinical-analysis-3"; - public final static String PROBAND_ID3 = "HG005_individual"; + public final static String PROBAND_ID3 = "HG005"; + + public final static String CA_ID4 = "clinical-analysis-4"; + public final static String PROBAND_ID4 = "HG105"; @Rule public ExpectedException thrown = ExpectedException.none(); @@ -135,7 +143,7 @@ public void setUpCatalogManager() throws IOException, CatalogException, URISynta catalogUploadFile("/biofiles/exomiser.vcf.gz"); //--------------------------------------------------------------------- - // Chinese trio (clinicalAnalysis3) + // Chinese trio: FAMILY (clinicalAnalysis3) //--------------------------------------------------------------------- Individual hg006Individual = new Individual().setId("HG006_individual") @@ -154,7 +162,7 @@ public void setUpCatalogManager() throws IOException, CatalogException, URISynta .setFather(hg006Individual) .setMother(hg007Individual) .setSex(SexOntologyTermAnnotation.initMale()) - .setSamples(Collections.singletonList(new Sample().setId("HG005"))); + .setSamples(Collections.singletonList(new Sample().setId(PROBAND_ID3))); Individual hg004Individual = new Individual().setId("HG004_individual") .setFather(hg006Individual) @@ -162,14 +170,14 @@ public void setUpCatalogManager() throws IOException, CatalogException, URISynta .setSex(SexOntologyTermAnnotation.initFemale()) .setSamples(Collections.singletonList(new Sample().setId("HG004"))); - Family chineseFamily = new Family("chinese_family", "chinese_family", null, null, + Family chineseFamily = new Family("chinese_trio_family", "chinese_trio_family", null, null, Arrays.asList(hg005Individual, hg006Individual, hg007Individual, hg004Individual), "", 4, Collections.emptyList(), Collections.emptyMap()); catalogManager.getFamilyManager().create(studyFqn, chineseFamily, QueryOptions.empty(), token).first(); auxClinicalAnalysis = new ClinicalAnalysis() .setId(CA_ID3) - .setDescription("My description - exomiser - trio") + .setDescription("My description - exomiser - trio - family") .setType(ClinicalAnalysis.Type.FAMILY) .setDueDate("20180510100000") .setDisorder(getDisorder()) @@ -183,6 +191,60 @@ public void setUpCatalogManager() throws IOException, CatalogException, URISynta catalogUploadFile("/biofiles/HG005.1k.vcf.gz"); catalogUploadFile("/biofiles/HG006.1k.vcf.gz"); catalogUploadFile("/biofiles/HG007.1k.vcf.gz"); + + //--------------------------------------------------------------------- + // Chinese trio: SINGLE (clinicalAnalysis4) + //--------------------------------------------------------------------- + + Individual hg106Individual = new Individual().setId("HG106_individual") + .setPhenotypes(Collections.emptyList()) + .setSex(SexOntologyTermAnnotation.initMale()) + .setSamples(Collections.singletonList(new Sample().setId("HG106"))); + + Individual hg107Individual = new Individual().setId("HG107_individual") + .setPhenotypes(Collections.emptyList()) + .setSex(SexOntologyTermAnnotation.initFemale()) + .setSamples(Collections.singletonList(new Sample().setId("HG107"))); + + Individual hg105Individual = new Individual().setId("HG105_individual") + .setDisorders(Collections.singletonList(getDisorder())) + .setPhenotypes(getPhenotypes()) + .setFather(hg106Individual) + .setMother(hg107Individual) + .setSex(SexOntologyTermAnnotation.initMale()) + .setSamples(Collections.singletonList(new Sample().setId(PROBAND_ID4))); + + Individual hg104Individual = new Individual().setId("HG104_individual") + .setFather(hg106Individual) + .setMother(hg107Individual) + .setSex(SexOntologyTermAnnotation.initFemale()) + .setSamples(Collections.singletonList(new Sample().setId("HG104"))); + + Family chineseSingle = new Family("chinese_trio_single", "chinese_trio_single", null, null, + Arrays.asList(hg105Individual, hg106Individual, hg107Individual, hg104Individual), "", 4, Collections.emptyList(), + Collections.emptyMap()); + catalogManager.getFamilyManager().create(studyFqn, chineseSingle, QueryOptions.empty(), token).first(); + + auxClinicalAnalysis = new ClinicalAnalysis() + .setId(CA_ID4) + .setDescription("My description - exomiser - trio - single") + .setType(ClinicalAnalysis.Type.SINGLE) + .setDueDate("20180510100000") + .setDisorder(getDisorder()) + .setProband(hg105Individual) + .setFamily(chineseSingle); + + catalogManager.getClinicalAnalysisManager().create(studyFqn, auxClinicalAnalysis, QueryOptions.empty(), token) + .first(); + + catalogUploadFile("/biofiles/HG104.1k.vcf.gz"); + catalogUploadFile("/biofiles/HG105.1k.vcf.gz"); + catalogUploadFile("/biofiles/HG106.1k.vcf.gz"); + catalogUploadFile("/biofiles/HG107.1k.vcf.gz"); + + + OpenCGAResult sampleResult = catalogManager.getSampleManager().search(studyFqn, new Query(), new QueryOptions(INCLUDE, "id"), token); + Assert.assertEquals(12, sampleResult.getNumResults()); } private void catalogUploadFile(String path) throws IOException, CatalogException { diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java index 21d0322207b..73cab79045b 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java @@ -16,10 +16,12 @@ package org.opencb.opencga.catalog.managers; -import com.mongodb.BasicDBObject; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -37,6 +39,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.Optimizations; import org.opencb.opencga.core.models.Acl; import org.opencb.opencga.core.models.AclEntry; import org.opencb.opencga.core.models.AclEntryList; @@ -49,6 +52,7 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.job.*; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; import org.opencb.opencga.core.models.project.DataStore; @@ -57,20 +61,22 @@ import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.study.*; -import org.opencb.opencga.core.models.user.Account; -import org.opencb.opencga.core.models.user.AuthenticationResponse; -import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import javax.naming.NamingException; import java.io.IOException; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.*; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; @Category(MediumTests.class) public class CatalogManagerTest extends AbstractManagerTest { @@ -254,6 +260,111 @@ public void anonymousUserLoginTest() throws CatalogException { catalogManager.getUserManager().loginAnonymous(org2); } + @Test + public void incrementLoginAttemptsTest() throws CatalogException { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + User user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + assertEquals(1, user.getInternal().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + for (int i = 2; i < 5; i++) { + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + assertEquals(i, user.getInternal().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + } + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + assertEquals(5, user.getInternal().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, "incorrect")); + assertTrue(incorrect.getMessage().contains("banned")); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + assertEquals(5, user.getInternal().getFailedAttempts()); + assertEquals(UserStatus.BANNED, user.getInternal().getStatus().getId()); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("banned")); + + // Remove ban from user + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.READY, QueryOptions.empty(), ownerToken); + user = catalogManager.getUserManager().get(organizationId, normalUserId1, QueryOptions.empty(), ownerToken).first(); + assertEquals(0, user.getInternal().getFailedAttempts()); + assertEquals(UserStatus.READY, user.getInternal().getStatus().getId()); + + String token = catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void changeUserStatusTest() throws CatalogException { + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), normalToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), studyAdminToken1)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.BANNED, QueryOptions.empty(), ownerToken)); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken1); + catalogManager.getUserManager().changeStatus(organizationId, normalUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), opencgaToken); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken); + CatalogAuthorizationException authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgOwnerUserId, UserStatus.SUSPENDED, QueryOptions.empty(), ownerToken)); + assertTrue(authException.getMessage().contains("own account")); + + authException = assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(authException.getMessage().contains("suspend administrators")); + + CatalogAuthenticationException incorrect = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD)); + assertTrue(incorrect.getMessage().contains("suspended")); + + catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, UserStatus.READY, QueryOptions.empty(), orgAdminToken2); + String token = catalogManager.getUserManager().login(organizationId, orgAdminUserId1, TestParamConstants.PASSWORD).getToken(); + assertNotNull(token); + + CatalogParameterException paramException = assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, orgAdminUserId1, "NOT_A_STATUS", QueryOptions.empty(), orgAdminToken2)); + assertTrue(paramException.getMessage().contains("Invalid status")); + + CatalogDBException dbException = assertThrows(CatalogDBException.class, () -> catalogManager.getUserManager().changeStatus(organizationId, "notAUser", UserStatus.SUSPENDED, QueryOptions.empty(), orgAdminToken2)); + assertTrue(dbException.getMessage().contains("not exist")); + } + + @Test + public void loginExpiredAccountTest() throws CatalogException { + // Expire account of normalUserId1 + ObjectMap params = new ObjectMap(UserDBAdaptor.QueryParams.ACCOUNT_EXPIRATION_DATE.key(), TimeUtils.getTime()); + catalogManager.getUserManager().getUserDBAdaptor(organizationId).update(normalUserId1, params); + + CatalogAuthenticationException authException = assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getUserManager().login(organizationId, normalUserId1, TestParamConstants.PASSWORD)); + assertTrue(authException.getMessage().contains("expired")); + + // Ensure it doesn't matter whether opencga account is expired or not + catalogManager.getUserManager().getUserDBAdaptor(ParamConstants.ADMIN_ORGANIZATION).update(ParamConstants.OPENCGA_USER_ID, params); + String token = catalogManager.getUserManager().login(ParamConstants.ADMIN_ORGANIZATION, ParamConstants.OPENCGA_USER_ID, TestParamConstants.ADMIN_PASSWORD).getToken(); + assertNotNull(token); + } + + @Test + public void updateUserTest() throws JsonProcessingException, CatalogException { + UserUpdateParams userUpdateParams = new UserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com"); + ObjectMap updateParams = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + User user = catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken1).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, normalToken2)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, opencgaToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, ownerToken)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, orgAdminToken1)); + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams, INCLUDE_RESULT, studyAdminToken1)); + + userUpdateParams = new UserUpdateParams() + .setEmail("notAnEmail"); + ObjectMap updateParams2 = new ObjectMap(getUpdateObjectMapper().writeValueAsString(userUpdateParams)); + assertThrows(CatalogParameterException.class, () -> catalogManager.getUserManager().update(normalUserId1, updateParams2, INCLUDE_RESULT, normalToken1)); + } + @Test public void testGetUserInfo() throws CatalogException { // OpenCGA administrator @@ -353,10 +464,6 @@ public void testModifyUser() throws CatalogException, InterruptedException, IOEx String newEmail = "new@email.ac.uk"; params.put("name", newName); - ObjectMap attributes = new ObjectMap("myBoolean", true); - attributes.put("value", 6); - attributes.put("object", new BasicDBObject("id", 1234)); - params.put("attributes", attributes); Thread.sleep(10); @@ -373,10 +480,6 @@ public void testModifyUser() throws CatalogException, InterruptedException, IOEx assertEquals(userPost.getEmail(), newEmail); catalogManager.getUserManager().login(organizationId, orgOwnerUserId, newPassword); - for (Map.Entry entry : attributes.entrySet()) { - assertEquals(userPost.getAttributes().get(entry.getKey()), entry.getValue()); - } - catalogManager.getUserManager().changePassword(organizationId, orgOwnerUserId, newPassword, TestParamConstants.PASSWORD); catalogManager.getUserManager().login(organizationId, orgOwnerUserId, TestParamConstants.PASSWORD); @@ -427,6 +530,15 @@ private String getAdminToken() throws CatalogException, IOException { return catalogManager.getUserManager().loginAsAdmin("admin").getToken(); } + @Test + public void createUserUsingMailAsId() throws CatalogException { + catalogManager.getUserManager().create(new User().setId("hello.mail@mymail.org").setName("Hello"), TestParamConstants.PASSWORD, ownerToken); + AuthenticationResponse login = catalogManager.getUserManager().login(organizationId, "hello.mail@mymail.org", TestParamConstants.PASSWORD); + assertNotNull(login); + User user = catalogManager.getUserManager().get(organizationId, "hello.mail@mymail.org", new QueryOptions(), login.getToken()).first(); + assertEquals("hello.mail@mymail.org", user.getId()); + } + @Test public void getGroupsTest() throws CatalogException { Group group = new Group("groupId", Arrays.asList(normalUserId2, normalUserId3)).setSyncedFrom(new Group.Sync("ldap", "bio")); @@ -656,6 +768,82 @@ public void testModifyProject() throws CatalogException { catalogManager.getProjectManager().update(project1, options, null, ownerToken); } + @Test + public void updateProjectPermissionTest() throws CatalogException { + ObjectMap params = new ObjectMap() + .append(ProjectDBAdaptor.QueryParams.DESCRIPTION.key(), "my new description"); + Project project = catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, ownerToken).first(); + assertEquals("my new description", project.getDescription()); + + params.put(ProjectDBAdaptor.QueryParams.DESCRIPTION.key(), "my new description 2"); + project = catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken2).first(); + assertEquals("my new description 2", project.getDescription()); + + params.put(ProjectDBAdaptor.QueryParams.DESCRIPTION.key(), "my new description 3"); + project = catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, studyAdminToken1).first(); + assertEquals("my new description 3", project.getDescription()); + + // Make normalUser1 admin of first study + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(normalUserId1)), ownerToken); + // And remove normalUser1 from the admins group of the second study (just in case) + catalogManager.getStudyManager().updateGroup(studyFqn2, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.REMOVE, + new GroupUpdateParams(Collections.singletonList(normalUserId1)), ownerToken); + + CatalogAuthorizationException catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, normalToken1)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, normalToken2)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + + // Remove orgAdminUser1 from the administrators group of the organization + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.QueryParams.ADMINS.key(), ParamUtils.BasicUpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + catalogManager.getOrganizationManager().update(organizationId, new OrganizationUpdateParams() + .setAdmins(Collections.singletonList(orgAdminUserId1)), options, ownerToken); + + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + + // Create a third study + catalogManager.getStudyManager().create(project1, new Study().setId("study_3"), null, ownerToken); + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains("study_3")); + + // Add orgAdminUser1 to the administrators group of the third study + catalogManager.getStudyManager().updateGroup("study_3", ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), ownerToken); + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains("study_3")); + + // Add orgAdminUser1 to the administrators group of the second study + catalogManager.getStudyManager().updateGroup(studyFqn2, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), ownerToken); + catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, + () -> catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1)); + assertTrue(catalogAuthorizationException.getCause().getMessage().contains(studyFqn)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains(studyFqn2)); + assertFalse(catalogAuthorizationException.getCause().getMessage().contains("study_3")); + + // Add orgAdminUser1 to the administrators group of the remaining study + catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, + new GroupUpdateParams(Collections.singletonList(orgAdminUserId1)), ownerToken); + catalogManager.getProjectManager().update(project1, params, INCLUDE_RESULT, orgAdminToken1); + } + @Test public void updatePrivateParamsFromProjectTest() throws CatalogException { catalogManager.getProjectManager().setDatastoreVariant(projectFqn1, new DataStore(), opencgaToken); @@ -1087,6 +1275,28 @@ public void testCreateJob() throws CatalogException { QueryOptions.empty(), ownerToken); } + @Test + public void testKillJob() throws CatalogException { + Job job = catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); + assertFalse(job.getInternal().isKillJobRequested()); + + catalogManager.getJobManager().kill(studyFqn, job.getId(), ownerToken); + job = catalogManager.getJobManager().get(studyFqn, job.getId(), QueryOptions.empty(), ownerToken).first(); + assertTrue(job.getInternal().isKillJobRequested()); + + for (String status : Arrays.asList(Enums.ExecutionStatus.DONE, Enums.ExecutionStatus.ABORTED, Enums.ExecutionStatus.ERROR)) { + job = catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); + catalogManager.getJobManager().update(studyFqn, job.getId(), + new ObjectMap(JobDBAdaptor.QueryParams.INTERNAL_STATUS.key(), new Enums.ExecutionStatus(status)), + QueryOptions.empty(), ownerToken); + // Try to kill job that is already finished + String jobId = job.getId(); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getJobManager().kill(studyFqn, jobId, ownerToken)); + assertTrue(catalogException.getMessage().contains("status")); + } + + } + @Test public void testCreateJobAndReuse() throws CatalogException { String project1 = catalogManager.getProjectManager().create("testCreateJobAndReuse_project1", "", "", "Homo sapiens", @@ -1107,7 +1317,7 @@ public void testCreateJobAndReuse() throws CatalogException { // Same params, different order, empty jobId OpenCGAResult result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 1), - "", "", Collections.emptyList(), Collections.emptyList(), ownerToken); + "", "", Collections.emptyList(), Collections.emptyList(), null, null, false, ownerToken); assertEquals(job1, result.first().getId()); assertEquals(1, result.getEvents().size()); assertEquals("reuse", result.getEvents().get(0).getId()); @@ -1118,13 +1328,13 @@ public void testCreateJobAndReuse() throws CatalogException { // Same params, but with jobId result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 2), "MyJobId", "", - Collections.emptyList(), Collections.emptyList(), ownerToken); + Collections.emptyList(), Collections.emptyList(), null, null, false, ownerToken); assertNotEquals(job1, result.first().getId()); assertEquals("MyJobId", result.first().getId()); // Same params, but with dependencies result = catalogManager.getJobManager().submit(study1, toolId, null, new ObjectMap("key2", 2).append("key", 2), "", "", - Collections.singletonList(job1), Collections.emptyList(), ownerToken); + Collections.singletonList(job1), Collections.emptyList(), null, null, false, ownerToken); assertNotEquals(job1, result.first().getId()); } @@ -1132,13 +1342,13 @@ public void testCreateJobAndReuse() throws CatalogException { public void submitJobWithDependenciesFromDifferentStudies() throws CatalogException { Job first = catalogManager.getJobManager().submit(studyFqn, "command-subcommand", null, Collections.emptyMap(), ownerToken).first(); Job second = catalogManager.getJobManager().submit(studyFqn2, "command-subcommand2", null, Collections.emptyMap(), null, "", - Collections.singletonList(first.getUuid()), null, ownerToken).first(); + Collections.singletonList(first.getUuid()), null, null, null, false, ownerToken).first(); assertEquals(first.getId(), second.getDependsOn().get(0).getId()); thrown.expect(CatalogException.class); thrown.expectMessage("not found"); catalogManager.getJobManager().submit(studyFqn2, "command-subcommand2", null, Collections.emptyMap(), null, "", - Collections.singletonList(first.getId()), null, ownerToken); + Collections.singletonList(first.getId()), null, null, null, false, ownerToken); } @Test @@ -1204,9 +1414,9 @@ public void submitJobWithDependencies() throws CatalogException { Job job2 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file2"), ownerToken).first(); Job job3 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file3"), null, null, - Arrays.asList(job1.getId(), job2.getId()), null, ownerToken).first(); + Arrays.asList(job1.getId(), job2.getId()), null, null, null, false, ownerToken).first(); Job job4 = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, new ObjectMap("param", "file4"), null, null, - Arrays.asList(job1.getUuid(), job2.getUuid()), null, ownerToken).first(); + Arrays.asList(job1.getUuid(), job2.getUuid()), null, null, null, false, ownerToken).first(); assertEquals(2, job3.getDependsOn().size()); assertEquals(job1.getUuid(), job3.getDependsOn().get(0).getUuid()); @@ -1565,6 +1775,88 @@ public void updateSampleCohortTest() throws Exception { } } + @Test + public void updateSampleCohortWithThreadsTest() throws Exception { + Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), INCLUDE_RESULT, ownerToken).first(); + Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), INCLUDE_RESULT, ownerToken).first(); + catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort1") + .setSamples(Arrays.asList(sampleId1)), null, ownerToken).first(); + catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort2") + .setSamples(Arrays.asList(sampleId2, sampleId3)), null, ownerToken).first(); + + ExecutorService executorService = Executors.newFixedThreadPool(10, + new ThreadFactoryBuilder() + .setNameFormat("executor-service-%d") + .build()); + + StopWatch stopWatch = StopWatch.createStarted(); + List> sampleIds = new ArrayList<>(5); + List innerArray = new ArrayList<>(50); + for (int i = 0; i < 250; i++) { + if (i % 50 == 0) { + System.out.println("i = " + i); + } + + String sampleId = "SAMPLE_AUTO_" + i; + executorService.submit(() -> { + try { + catalogManager.getSampleManager().create(studyFqn, new Sample().setId(sampleId), QueryOptions.empty(), ownerToken); + } catch (CatalogException e) { + throw new RuntimeException(e); + } + }); + if (innerArray.size() == 50) { + sampleIds.add(new ArrayList<>(innerArray)); + innerArray.clear(); + } + innerArray.add(sampleId); + } + sampleIds.add(new ArrayList<>(innerArray)); + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + + System.out.println("Creating 250 samples took " + stopWatch.getTime(TimeUnit.SECONDS) + " seconds"); + + stopWatch.stop(); + stopWatch.reset(); + stopWatch.start(); + executorService = Executors.newFixedThreadPool(3); + int execution = 0; + Map actionMap = new HashMap<>(); + actionMap.put(CohortDBAdaptor.QueryParams.SAMPLES.key(), ParamUtils.BasicUpdateAction.SET); + QueryOptions queryOptions = new QueryOptions(); + queryOptions.put(Constants.ACTIONS, actionMap); + for (List innerSampleIds : sampleIds) { + Cohort myCohort1 = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", null, ownerToken).first(); + List sampleReferenceParamList = new ArrayList<>(myCohort1.getNumSamples() + innerSampleIds.size()); + sampleReferenceParamList.addAll(myCohort1.getSamples().stream().map(s -> new SampleReferenceParam().setId(s.getId())).collect(Collectors.toList())); + sampleReferenceParamList.addAll(innerSampleIds.stream().map(s -> new SampleReferenceParam().setId(s)).collect(Collectors.toList())); + int executionId = execution++; + executorService.submit(() -> { + try { + catalogManager.getCohortManager().update(studyFqn, "MyCohort1", + new CohortUpdateParams().setSamples(sampleReferenceParamList), + queryOptions, ownerToken); + System.out.println("Execution: " + executionId); + } catch (CatalogException e) { + throw new RuntimeException(e); + } + }); + } + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + System.out.println("Attaching 250 samples took " + stopWatch.getTime(TimeUnit.SECONDS) + " seconds"); + + // Ensure persistence + Query sampleQuery = new Query(SampleDBAdaptor.QueryParams.COHORT_IDS.key(), "MyCohort1"); + OpenCGAResult search = catalogManager.getSampleManager().search(studyFqn, sampleQuery, SampleManager.INCLUDE_SAMPLE_IDS, ownerToken); + Cohort myCohort1 = catalogManager.getCohortManager().get(studyFqn, "MyCohort1", null, ownerToken).first(); + assertEquals(search.getNumResults(), myCohort1.getNumSamples()); + Set sampleIdSet = search.getResults().stream().map(Sample::getId).collect(Collectors.toSet()); + assertTrue(myCohort1.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()).containsAll(sampleIdSet)); + } + @Test public void deleteSampleCohortTest() throws Exception { Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), INCLUDE_RESULT, ownerToken).first(); @@ -2147,7 +2439,7 @@ public void getEffectivePermissions() throws CatalogException { Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user5", "user6", ParamConstants.ANONYMOUS_USER_ID), Arrays.asList("user4", "user7"), aclList.getResults().get(2)); - catalogManager.getConfiguration().getOptimizations().setSimplifyPermissions(true); + catalogManager.getOrganizationManager().updateConfiguration(organizationId, new OrganizationConfiguration().setOptimizations(new Optimizations(true)), null, ownerToken); aclList = catalogManager.getAdminManager().getEffectivePermissions(studyFqn, Arrays.asList(s_7Id, s_8Id, s_9Id), Enums.Resource.SAMPLE.name(), ownerToken); assertEquals(3, aclList.getNumResults()); assertPermissions(s_7Id, Arrays.asList(orgOwnerUserId, orgAdminUserId1, orgAdminUserId2, studyAdminUserId1, normalUserId1, normalUserId2, normalUserId3, "user4", "user5", "user6", "user7", ParamConstants.ANONYMOUS_USER_ID), diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index 65af29ca14a..5e655998abd 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -40,11 +40,17 @@ import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.*; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -189,6 +195,32 @@ public void testLinkFileWithoutReadPermissions() throws IOException, CatalogExce fileManager.link(studyFqn, new FileLinkParams().setUri(file.getPath()), false, ownerToken); } + @Test + public void filterByFormatTest() throws CatalogException { + Query query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "PLAIN"); + OpenCGAResult search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); + + query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "plain"); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(0, search.getNumResults()); + + // Case sensitive search in lower case + query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "~/^pla/"); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(0, search.getNumResults()); + + // Case sensitive in upper case + query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "~/^PLA/"); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); + + // Case insensitive search + query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), "~/^pla/i"); + search = catalogManager.getFileManager().search(studyFqn, query, QueryOptions.empty(), ownerToken); + assertEquals(3, search.getNumResults()); + } + @Test public void createDirectoryTest() throws CatalogException { FileCreateParams params = new FileCreateParams() @@ -677,6 +709,7 @@ public void testMoveFiles() throws CatalogException { file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello2.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam2")), INCLUDE_RESULT, ownerToken).first(); fileMap.put(file.getUid(), file); file = catalogManager.getFileManager().update(studyFqn, "A/B/C/D/hello3.txt", new FileUpdateParams().setSampleIds(Collections.singletonList("sam3")), INCLUDE_RESULT, ownerToken).first(); + fileMap.put(file.getUid(), file); Map sampleVersionMap = new HashMap<>(); @@ -766,6 +799,7 @@ public void testMoveFiles() throws CatalogException { catalogManager.getFileManager().move(studyFqn, "A/C/D/hello3.txt", "A/C/D/otherName.txt", QueryOptions.empty(), ownerToken); assertThrows(CatalogException.class, () -> catalogManager.getFileManager().get(studyFqn, "A/C/D/hello3.txt", QueryOptions.empty(), ownerToken)); file = catalogManager.getFileManager().get(studyFqn, "A/C/D/otherName.txt", QueryOptions.empty(), ownerToken).first(); + assertTrue(file.getId().endsWith(":otherName.txt")); assertTrue(file.getPath().endsWith("/otherName.txt")); assertTrue(file.getUri().toString().endsWith("/otherName.txt")); @@ -1027,7 +1061,7 @@ public void testUnlinkFile() throws CatalogException, IOException { // We send the unlink command again thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); + thrown.expectMessage("not unlink"); fileManager.unlink(studyFqn, "myDirectory/data/test/folder/test_0.5K.txt", ownerToken); } @@ -2035,6 +2069,97 @@ public void deleteFolderTest4() throws CatalogException, IOException { } } + @Test + public void deleteFileInClinicalAnalysis() throws CatalogException, IOException { + // START DATA PREPARATION FOR TEST !!! + String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); + File file = fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, ownerToken).first(); + + Family family1 = DummyModelUtils.getDummyFamily("familyId1"); + catalogManager.getFamilyManager().create(studyFqn, family1, QueryOptions.empty(), ownerToken); + + // Associate BAM file to sample + String sampleId = family1.getMembers().get(0).getSamples().get(0).getId(); + catalogManager.getFileManager().update(studyFqn, file.getId(), new FileUpdateParams().setSampleIds(Collections.singletonList(sampleId)), + QueryOptions.empty(), ownerToken); + + Panel myPanel = DummyModelUtils.getDummyPanel("myPanel"); + catalogManager.getPanelManager().create(studyFqn, myPanel, QueryOptions.empty(), ownerToken); + + Family copy = JacksonUtils.copy(family1, Family.class); + for (Individual member : copy.getMembers()) { + // Only use the first sample + member.setSamples(Collections.singletonList(member.getSamples().get(0))); + } + + ClinicalAnalysis clinicalAnalysis1 = DummyModelUtils.getDummyClinicalAnalysis(copy.getMembers().get(0), copy, Collections.singletonList(myPanel)); + clinicalAnalysis1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis1, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, clinicalAnalysis1.getFiles().size()); + assertEquals(file.getPath(), clinicalAnalysis1.getFiles().get(0).getPath()); + assertFalse(clinicalAnalysis1.isLocked()); + + ClinicalAnalysis clinicalAnalysis2 = DummyModelUtils.getDummyClinicalAnalysis(copy.getMembers().get(0), copy, Collections.singletonList(myPanel)); + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis2, INCLUDE_RESULT, ownerToken).first(); + assertEquals(1, clinicalAnalysis2.getFiles().size()); + assertEquals(file.getPath(), clinicalAnalysis2.getFiles().get(0).getPath()); + assertFalse(clinicalAnalysis2.isLocked()); + + // Lock clinicalAnalysis2 + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis2.getId(), + new ClinicalAnalysisUpdateParams().setLocked(true), INCLUDE_RESULT, ownerToken).first(); + assertTrue(clinicalAnalysis2.isLocked()); + // END DATA PREPARATION FOR TEST !!! + + // Mark as pending delete + catalogManager.getFileManager().getFileDBAdaptor(organizationId).update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().unlink(studyFqn, file.getId(), ownerToken)); + assertTrue(catalogException.getMessage().contains("Could not unlink")); + assertTrue(catalogException.getCause().getMessage().contains("clinical analyses")); + + // Unlock clinicalAnalysis2 + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis2.getId(), + new ClinicalAnalysisUpdateParams().setLocked(false), INCLUDE_RESULT, ownerToken).first(); + assertFalse(clinicalAnalysis2.isLocked()); + + // Unlink file + catalogManager.getFileManager().unlink(studyFqn, file.getId(), ownerToken); + + Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId, QueryOptions.empty(), ownerToken).first(); + assertEquals(0, sample.getFileIds().size()); + + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); + assertEquals(2, search.getNumResults()); + for (ClinicalAnalysis clinicalAnalysis : search.getResults()) { + assertEquals(0, clinicalAnalysis.getFiles().size()); + assertEquals("OPENCGA", clinicalAnalysis.getAudit().get(clinicalAnalysis.getAudit().size() - 1).getAuthor()); + assertTrue(clinicalAnalysis.getAudit().get(clinicalAnalysis.getAudit().size() - 1).getMessage().contains("was deleted. Remove file references from case")); + } + } + + @Test + public void deleteFileUserInRelatedFilesTest() throws CatalogException { + fileManager.update(studyFqn, "data/test/folder/test_1K.txt.gz", + new FileUpdateParams().setRelatedFiles(Collections.singletonList( + new SmallRelatedFileParams("data/test/folder/test_0.5K.txt", FileRelatedFile.Relation.PART_OF_PAIR))), + null, ownerToken); + File file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), ownerToken).first(); + assertFalse(file.getRelatedFiles().isEmpty()); + assertEquals(1, file.getRelatedFiles().size()); + assertEquals("data/test/folder/test_0.5K.txt", file.getRelatedFiles().get(0).getFile().getPath()); + + file = fileManager.get(studyFqn, "data/test/folder/test_0.5K.txt", FileManager.INCLUDE_FILE_IDS, ownerToken).first(); + + // Mark as pending delete + catalogManager.getFileManager().getFileDBAdaptor(organizationId).update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); + // Delete test_0.5K file + QueryOptions options = new QueryOptions(Constants.SKIP_TRASH, true); + fileManager.delete(studyFqn, Collections.singletonList("data/test/folder/test_0.5K.txt"), options, ownerToken); + + // Ensure there are no more references to test_0.5K file + file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), ownerToken).first(); + assertTrue(file.getRelatedFiles().isEmpty()); + } + private File createBasicDirectoryFileTestEnvironment(List folderFiles) throws CatalogException { File folder = fileManager.createFolder(studyFqn, Paths.get("folder").toString(), false, null, QueryOptions.empty(), ownerToken).first(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java index 929ac84e55e..8755cc50c4a 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/IndividualManagerTest.java @@ -1120,8 +1120,10 @@ public void viewSampleFilesFromIndividualTest() throws CatalogException { // Create individual catalogManager.getIndividualManager().create(studyFqn, new Individual().setId("individual"), Collections.singletonList(sample.getId()), + QueryOptions.empty(), ownerToken); Individual individual = catalogManager.getIndividualManager().get(studyFqn, "individual", QueryOptions.empty(), ownerToken).first(); + assertEquals(1, individual.getSamples().get(0).getFileIds().size()); assertEquals("variant-test-file.vcf.gz", individual.getSamples().get(0).getFileIds().get(0)); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java index b408fec74c1..ca9323b422d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/OrganizationManagerTest.java @@ -3,28 +3,33 @@ import org.apache.commons.collections4.CollectionUtils; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.catalog.auth.authentication.CatalogAuthenticationManager; import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.AuthenticationOrigin; +import org.opencb.opencga.core.config.Optimizations; import org.opencb.opencga.core.models.organizations.*; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserQuota; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import static org.junit.Assert.*; @@ -39,48 +44,125 @@ public void ensureAuthOriginExistsTest() throws CatalogException { } @Test - public void ensureAuthOriginCannotBeRemovedTest() throws CatalogException { - OrganizationUpdateParams updateParams = new OrganizationUpdateParams().setConfiguration(new OrganizationConfiguration( - Collections.emptyList(), null, new TokenConfiguration())); - thrown.expect(CatalogException.class); - thrown.expectMessage("OPENCGA"); - catalogManager.getOrganizationManager().update(organizationId, updateParams, INCLUDE_RESULT, ownerToken); + public void updateConfigurationAuthorizationTest() throws CatalogException { + OrganizationConfiguration configuration = new OrganizationConfiguration() + .setOptimizations(new Optimizations(true)); + + // Owner can update configuration + catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, QueryOptions.empty(), ownerToken); + + // Admin can update configuration + catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, QueryOptions.empty(), orgAdminToken1); + + // Admin can update configuration + catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, QueryOptions.empty(), orgAdminToken2); + + // Study admin cannot update configuration + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, QueryOptions.empty(), studyAdminToken1)); + + // Normal user cannot update configuration + assertThrows(CatalogAuthorizationException.class, () -> catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, QueryOptions.empty(), normalToken1)); } @Test - public void avoidDuplicatedOPENCGAAuthOriginTest() throws CatalogException { - AuthenticationOrigin authOrigin = CatalogAuthenticationManager.createOpencgaAuthenticationOrigin(); - AuthenticationOrigin authOrigin2 = CatalogAuthenticationManager.createOpencgaAuthenticationOrigin(); - OrganizationUpdateParams updateParams = new OrganizationUpdateParams().setConfiguration(new OrganizationConfiguration( - Arrays.asList(authOrigin, authOrigin2), null, new TokenConfiguration())); - - thrown.expect(CatalogException.class); - thrown.expectMessage("OPENCGA"); - catalogManager.getOrganizationManager().update(organizationId, updateParams, null, ownerToken); + public void ensureAuthOriginCannotBeRemovedTest() throws CatalogException { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); + + OrganizationConfiguration configuration = new OrganizationConfiguration() + .setAuthenticationOrigins(organization.getConfiguration().getAuthenticationOrigins()); + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getOrganizationManager() + .updateConfiguration(organizationId, configuration, options, ownerToken)); + assertTrue(catalogException.getMessage().contains("user account uses")); } @Test - public void avoidDuplicatedAuthOriginIdTest() throws CatalogException { - AuthenticationOrigin authOrigin = CatalogAuthenticationManager.createOpencgaAuthenticationOrigin(); - AuthenticationOrigin authOrigin2 = CatalogAuthenticationManager.createOpencgaAuthenticationOrigin(); - authOrigin2.setType(AuthenticationOrigin.AuthenticationType.LDAP); - OrganizationUpdateParams updateParams = new OrganizationUpdateParams().setConfiguration(new OrganizationConfiguration( - Arrays.asList(authOrigin, authOrigin2), null, new TokenConfiguration())); - - thrown.expect(CatalogException.class); - thrown.expectMessage("origin id"); - catalogManager.getOrganizationManager().update(organizationId, updateParams, null, ownerToken); + public void authOriginActionTest() throws CatalogException { + OrganizationConfiguration configuration = new OrganizationConfiguration() + .setAuthenticationOrigins(Collections.singletonList(new AuthenticationOrigin("myId", + AuthenticationOrigin.AuthenticationType.SSO, null, null))); + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.ADD); + QueryOptions options = new QueryOptions() + .append(ParamConstants.INCLUDE_RESULT_PARAM, true) + .append(Constants.ACTIONS, actionMap); + + OrganizationConfiguration configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, options, ownerToken).first(); + assertEquals(2, configurationResult.getAuthenticationOrigins().size()); + for (AuthenticationOrigin authenticationOrigin : configurationResult.getAuthenticationOrigins()) { + if (authenticationOrigin.getId().equals("myId")) { + assertEquals(AuthenticationOrigin.AuthenticationType.SSO, authenticationOrigin.getType()); + } else { + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, authenticationOrigin.getType()); + } + } + + // Remove authOrigin + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.REMOVE); + options.put(Constants.ACTIONS, actionMap); + configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, options, + ownerToken).first(); + assertEquals(1, configurationResult.getAuthenticationOrigins().size()); + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, configurationResult.getAuthenticationOrigins().get(0).getType()); + + // Set authOrigin + List authenticationOriginList = new ArrayList<>(); + authenticationOriginList.add(new AuthenticationOrigin("myId", AuthenticationOrigin.AuthenticationType.SSO, null, null)); + authenticationOriginList.add(configurationResult.getAuthenticationOrigins().get(0)); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.SET); + options.put(Constants.ACTIONS, actionMap); + configuration.setAuthenticationOrigins(authenticationOriginList); + configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, options, + ownerToken).first(); + assertEquals(2, configurationResult.getAuthenticationOrigins().size()); + for (AuthenticationOrigin authenticationOrigin : configurationResult.getAuthenticationOrigins()) { + if (authenticationOrigin.getId().equals("myId")) { + assertEquals(AuthenticationOrigin.AuthenticationType.SSO, authenticationOrigin.getType()); + } else { + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, authenticationOrigin.getType()); + } + } + + // Add existing authOrigin + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.ADD); + options.put(Constants.ACTIONS, actionMap); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getOrganizationManager() + .updateConfiguration(organizationId, configuration, options, ownerToken)); + assertTrue(catalogException.getMessage().contains("REPLACE")); + + // Replace existing authOrigin + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, ParamUtils.UpdateAction.REPLACE); + options.put(Constants.ACTIONS, actionMap); + configuration.setAuthenticationOrigins(Collections.singletonList( + new AuthenticationOrigin(CatalogAuthenticationManager.OPENCGA, AuthenticationOrigin.AuthenticationType.OPENCGA, null, new ObjectMap("key", "value")))); + configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, configuration, options, ownerToken).first(); + assertEquals(2, configurationResult.getAuthenticationOrigins().size()); + for (AuthenticationOrigin authenticationOrigin : configurationResult.getAuthenticationOrigins()) { + if (authenticationOrigin.getId().equals("myId")) { + assertEquals(AuthenticationOrigin.AuthenticationType.SSO, authenticationOrigin.getType()); + } else { + assertEquals(AuthenticationOrigin.AuthenticationType.OPENCGA, authenticationOrigin.getType()); + assertTrue(authenticationOrigin.getOptions().containsKey("key")); + assertEquals("value", authenticationOrigin.getOptions().get("key")); + } + } } @Test - public void updateAuthOriginTest() throws CatalogException { - AuthenticationOrigin authOrigin = CatalogAuthenticationManager.createOpencgaAuthenticationOrigin(); - OrganizationUpdateParams updateParams = new OrganizationUpdateParams().setConfiguration(new OrganizationConfiguration( - Collections.singletonList(authOrigin), null, new TokenConfiguration())); - - Organization organization = catalogManager.getOrganizationManager().update(organizationId, updateParams, INCLUDE_RESULT, ownerToken).first(); - assertEquals(authOrigin.getId(), organization.getConfiguration().getAuthenticationOrigins().get(0).getId()); - assertEquals(authOrigin.getType(), organization.getConfiguration().getAuthenticationOrigins().get(0).getType()); + public void tokenUpdateTest() throws CatalogException { + TokenConfiguration tokenConfiguration = TokenConfiguration.init(); + OrganizationConfiguration configuration = new OrganizationConfiguration().setToken(tokenConfiguration); + OrganizationConfiguration configurationResult = catalogManager.getOrganizationManager().updateConfiguration(organizationId, + configuration, INCLUDE_RESULT, ownerToken).first(); + assertEquals(tokenConfiguration.getSecretKey(), configurationResult.getToken().getSecretKey()); + + assertThrows(CatalogAuthenticationException.class, () -> catalogManager.getOrganizationManager().get(organizationId, null, ownerToken)); } @Test @@ -150,6 +232,71 @@ public void organizationInfoProjectsAuthTest() throws CatalogException { assertEquals(projectFqn1, organization.getProjects().get(0).getFqn()); } + @Test + public void validateUserUpdateParamsTest() { + OrganizationUserUpdateParams expiredDateParam = new OrganizationUserUpdateParams() + .setAccount(new OrganizationUserUpdateParams.Account("20200101100000")); + CatalogParameterException exception = assertThrows(CatalogParameterException.class, () -> catalogManager.getOrganizationManager() + .updateUser(organizationId, normalUserId1, expiredDateParam, INCLUDE_RESULT, ownerToken)); + assertTrue(exception.getMessage().contains("expired")); + + OrganizationUserUpdateParams invalidMailParam = new OrganizationUserUpdateParams() + .setEmail("invalidEmail"); + exception = assertThrows(CatalogParameterException.class, () -> catalogManager.getOrganizationManager().updateUser(organizationId, + normalUserId1, invalidMailParam, INCLUDE_RESULT, ownerToken)); + assertTrue(exception.getMessage().contains("not valid")); + } + + @Test + public void updateUserInformationTest() throws CatalogException { + Date date = TimeUtils.getDate(); + Calendar cl = Calendar.getInstance(); + cl.setTime(date); + cl.add(Calendar.YEAR, 1); + String expirationTime = TimeUtils.getTime(cl.getTime()); + + OrganizationUserUpdateParams userUpdateParams = new OrganizationUserUpdateParams() + .setName("newName") + .setEmail("mail@mail.com") + .setAccount(new OrganizationUserUpdateParams.Account(expirationTime)) + .setQuota(new UserQuota(1000, 1000000, 1000, 1000000)) + .setAttributes(Collections.singletonMap("key1", "value1")); + updateAndAssertChanges(organizationId, userUpdateParams, opencgaToken); + + userUpdateParams = new OrganizationUserUpdateParams() + .setName("newName2") + .setEmail("mai2l@mail.com") + .setAccount(new OrganizationUserUpdateParams.Account(expirationTime)) + .setQuota(new UserQuota(1001, 1010000, 1010, 1100000)) + .setAttributes(Collections.singletonMap("key2", "value2")); + updateAndAssertChanges(null, userUpdateParams, ownerToken); + + userUpdateParams = new OrganizationUserUpdateParams() + .setName("newName3") + .setEmail("mai3l@mail.com") + .setAccount(new OrganizationUserUpdateParams.Account(expirationTime)) + .setQuota(new UserQuota(3001, 1010300, 1013, 1300000)) + .setAttributes(Collections.singletonMap("key3", "value3")); + updateAndAssertChanges(null, userUpdateParams, orgAdminToken1); + + thrown.expect(CatalogAuthorizationException.class); + catalogManager.getOrganizationManager().updateUser(organizationId, normalUserId1, userUpdateParams, INCLUDE_RESULT, normalToken1); + } + + private void updateAndAssertChanges(String orgId, OrganizationUserUpdateParams userUpdateParams, String token) throws CatalogException { + User user = catalogManager.getOrganizationManager().updateUser(orgId, normalUserId1, userUpdateParams, INCLUDE_RESULT, token).first(); + assertEquals(userUpdateParams.getName(), user.getName()); + assertEquals(userUpdateParams.getEmail(), user.getEmail()); + assertEquals(userUpdateParams.getAccount().getExpirationDate(), user.getAccount().getExpirationDate()); + assertEquals(userUpdateParams.getQuota().getCpuUsage(), user.getQuota().getCpuUsage()); + assertEquals(userUpdateParams.getQuota().getDiskUsage(), user.getQuota().getDiskUsage()); + assertEquals(userUpdateParams.getQuota().getMaxCpu(), user.getQuota().getMaxCpu()); + assertEquals(userUpdateParams.getQuota().getMaxDisk(), user.getQuota().getMaxDisk()); + for (String key : userUpdateParams.getAttributes().keySet()) { + assertEquals(userUpdateParams.getAttributes().get(key), user.getAttributes().get(key)); + } + } + @Test public void migrationExecutionProjectionTest() throws CatalogException { Organization organization = catalogManager.getOrganizationManager().get(organizationId, null, ownerToken).first(); diff --git a/opencga-catalog/src/test/resources/biofiles/HG104.1k.vcf.gz b/opencga-catalog/src/test/resources/biofiles/HG104.1k.vcf.gz new file mode 100644 index 00000000000..4237b056dfa Binary files /dev/null and b/opencga-catalog/src/test/resources/biofiles/HG104.1k.vcf.gz differ diff --git a/opencga-catalog/src/test/resources/biofiles/HG105.1k.vcf.gz b/opencga-catalog/src/test/resources/biofiles/HG105.1k.vcf.gz new file mode 100644 index 00000000000..4bb5c469f01 Binary files /dev/null and b/opencga-catalog/src/test/resources/biofiles/HG105.1k.vcf.gz differ diff --git a/opencga-catalog/src/test/resources/biofiles/HG106.1k.vcf.gz b/opencga-catalog/src/test/resources/biofiles/HG106.1k.vcf.gz new file mode 100644 index 00000000000..47cec223765 Binary files /dev/null and b/opencga-catalog/src/test/resources/biofiles/HG106.1k.vcf.gz differ diff --git a/opencga-catalog/src/test/resources/biofiles/HG107.1k.vcf.gz b/opencga-catalog/src/test/resources/biofiles/HG107.1k.vcf.gz new file mode 100644 index 00000000000..065f94379ea Binary files /dev/null and b/opencga-catalog/src/test/resources/biofiles/HG107.1k.vcf.gz differ diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 7b84c12702c..2d7cbe0861a 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -6,9 +6,7 @@ databasePrefix: "opencga_test" workspace: "/tmp/opencga/sessions" jobDir: "/tmp/opencga/JOBS" - -admin: - secretKey: "asidnadh19rh230qncfascd1.rffzasf.asd.ad.12ddeASDAsd12e1d.adsx" +maxLoginAttempts: 5 audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. @@ -72,29 +70,6 @@ catalog: password: "" options: authenticationDatabase: "" - searchEngine: ## Solr configuration, by default is the same than storage - hosts: - - "http://localhost:8983/solr/" - user: "" - password: "" - options: - mode: "core" - timeout: 30000 - insertBatchSize: 2000 - -authentication: - expiration: 1000 -#LDAP configuration example -# authenticationOrigins: -# - id: ldap # Any id -# type: LDAP # At the moment, we only support LDAP -# host: ldap://localhost:9000 -# options: -# usersSearch: dc=ge,dc=co,dc=uk # Base search to look for the users -# groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Base search to look for the groups - -optimizations: - simplifyPermissions: false server: rest: diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index 2582a5f2003..043c1c5788d 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index 398851faed5..3c3198d89f8 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -19,19 +19,19 @@ #' #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | -#' | runBwa | /{apiVersion}/analysis/alignment/bwa/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runCoverageIndex | /{apiVersion}/analysis/alignment/coverage/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | coverageQcGeneCoverageStatsRun | /{apiVersion}/analysis/alignment/coverage/qc/geneCoverageStats/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runBwa | /{apiVersion}/analysis/alignment/bwa/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runCoverageIndex | /{apiVersion}/analysis/alignment/coverage/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | coverageQcGeneCoverageStatsRun | /{apiVersion}/analysis/alignment/coverage/qc/geneCoverageStats/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | queryCoverage | /{apiVersion}/analysis/alignment/coverage/query | file[*], study, region, gene, offset, onlyExons, range, windowSize, splitResults | #' | ratioCoverage | /{apiVersion}/analysis/alignment/coverage/ratio | file1[*], file2[*], study, skipLog2, region, gene, offset, onlyExons, windowSize, splitResults | #' | statsCoverage | /{apiVersion}/analysis/alignment/coverage/stats | file[*], gene[*], study, threshold | -#' | runDeeptools | /{apiVersion}/analysis/alignment/deeptools/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runFastqc | /{apiVersion}/analysis/alignment/fastqc/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runIndex | /{apiVersion}/analysis/alignment/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runPicard | /{apiVersion}/analysis/alignment/picard/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runQc | /{apiVersion}/analysis/alignment/qc/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runDeeptools | /{apiVersion}/analysis/alignment/deeptools/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runFastqc | /{apiVersion}/analysis/alignment/fastqc/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runIndex | /{apiVersion}/analysis/alignment/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runPicard | /{apiVersion}/analysis/alignment/picard/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runQc | /{apiVersion}/analysis/alignment/qc/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | query | /{apiVersion}/analysis/alignment/query | limit, skip, count, file[*], study, region, gene, offset, onlyExons, minMappingQuality, maxNumMismatches, maxNumHits, properlyPaired, maxInsertSize, skipUnmapped, skipDuplicated, regionContained, forceMDField, binQualities, splitResults | -#' | runSamtools | /{apiVersion}/analysis/alignment/samtools/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runSamtools | /{apiVersion}/analysis/alignment/samtools/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -49,6 +49,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data BWA parameters. runBwa=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/bwa", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -60,6 +63,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Coverage computation parameters. runCoverageIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/coverage/index", subcategoryId=NULL, action="run", params=params, @@ -72,6 +78,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gene coverage stats parameters for a given BAM file and a list of genes. coverageQcGeneCoverageStatsRun=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/coverage/qc/geneCoverageStats", subcategoryId=NULL, action="run", params=params, @@ -125,6 +134,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Deeptools parameters. Supported Deeptools commands: bamCoverage, bamCompare. runDeeptools=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/deeptools", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -137,6 +149,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data FastQC parameters. runFastqc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/fastqc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -148,6 +163,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Alignment index params. runIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/index", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -159,6 +177,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Picard parameters. Supported Picard commands: CollectHsMetrics, CollectWgsMetrics, BedToIntervalList. runPicard=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/picard", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -170,6 +191,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Alignment quality control (QC) parameters. It computes: stats, flag stats and fastqc metrics. The BAM file ID is mandatory and in order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc. runQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -206,6 +230,9 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Samtools parameters. Supported Samtools commands: sort, index, view, stats, flagstat, dict, faidx, depth, plot-bamstats. runSamtools=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/samtools", subcategoryId=NULL, action="run", params=params, httpMethod="POST", diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index 195b5ad36f3..5b3e3d890ce 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,6 +1,6 @@ # ############################################################################## ## OrganizationClient -setGeneric("organizationClient", function(OpencgaR, id, organization, endpointName, params=NULL, ...) +setGeneric("organizationClient", function(OpencgaR, id, organization, user, endpointName, params=NULL, ...) standardGeneric("organizationClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index f9d9f038ae1..6fb3c4dfffa 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -27,16 +27,16 @@ #' | distinctInterpretation | /{apiVersion}/analysis/clinical/interpretation/distinct | study, id, uuid, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release, field[*] | #' | searchInterpretation | /{apiVersion}/analysis/clinical/interpretation/search | include, exclude, limit, skip, sort, study, id, uuid, clinicalAnalysisId, analystId, methodName, panels, primaryFindings, secondaryFindings, creationDate, modificationDate, status, internalStatus, release | #' | infoInterpretation | /{apiVersion}/analysis/clinical/interpretation/{interpretations}/info | include, exclude, interpretations[*], study, version, deleted | -#' | runInterpreterCancerTiering | /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterExomiser | /{apiVersion}/analysis/clinical/interpreter/exomiser/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterTeam | /{apiVersion}/analysis/clinical/interpreter/team/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterTiering | /{apiVersion}/analysis/clinical/interpreter/tiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInterpreterZetta | /{apiVersion}/analysis/clinical/interpreter/zetta/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | load | /{apiVersion}/analysis/clinical/load | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runInterpreterCancerTiering | /{apiVersion}/analysis/clinical/interpreter/cancerTiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterExomiser | /{apiVersion}/analysis/clinical/interpreter/exomiser/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterTeam | /{apiVersion}/analysis/clinical/interpreter/team/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterTiering | /{apiVersion}/analysis/clinical/interpreter/tiering/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInterpreterZetta | /{apiVersion}/analysis/clinical/interpreter/zetta/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | load | /{apiVersion}/analysis/clinical/load | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | aggregationStatsRga | /{apiVersion}/analysis/clinical/rga/aggregationStats | limit, skip, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study, field[*] | #' | queryRgaGene | /{apiVersion}/analysis/clinical/rga/gene/query | include, exclude, limit, skip, count, includeIndividual, skipIndividual, limitIndividual, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | summaryRgaGene | /{apiVersion}/analysis/clinical/rga/gene/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | -#' | runRgaIndex | /{apiVersion}/analysis/clinical/rga/index/run | study, jobId, jobDescription, jobDependsOn, jobTags, auxiliarIndex, body[*] | +#' | runRgaIndex | /{apiVersion}/analysis/clinical/rga/index/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, auxiliarIndex, body[*] | #' | queryRgaIndividual | /{apiVersion}/analysis/clinical/rga/individual/query | include, exclude, limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | summaryRgaIndividual | /{apiVersion}/analysis/clinical/rga/individual/summary | limit, skip, count, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | #' | queryRgaVariant | /{apiVersion}/analysis/clinical/rga/variant/query | include, exclude, limit, skip, count, includeIndividual, skipIndividual, limitIndividual, sampleId, individualId, sex, phenotypes, disorders, numParents, geneId, geneName, chromosome, start, end, transcriptId, variants, dbSnps, knockoutType, filter, type, clinicalSignificance, populationFrequency, consequenceType, study | @@ -204,6 +204,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Cancer tiering interpretation analysis params. runInterpreterCancerTiering=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/cancerTiering", subcategoryId=NULL, action="run", params=params, @@ -216,6 +219,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Exomizer interpretation analysis params. runInterpreterExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/exomiser", subcategoryId=NULL, action="run", params=params, @@ -228,6 +234,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data TEAM interpretation analysis params. runInterpreterTeam=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/team", subcategoryId=NULL, action="run", params=params, @@ -240,6 +249,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Tiering interpretation analysis params. runInterpreterTiering=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/tiering", subcategoryId=NULL, action="run", params=params, @@ -252,6 +264,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Zetta interpretation analysis params. runInterpreterZetta=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical/interpreter/zetta", subcategoryId=NULL, action="run", params=params, @@ -264,6 +279,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Parameters to load clinical analysis in OpenCGA catalog from a file. load=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="clinical", subcategoryId=NULL, action="load", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -370,6 +388,9 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param auxiliarIndex Index auxiliar collection to improve performance assuming RGA is completely indexed. #' @param data Recessive Gene Analysis index params. runRgaIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, @@ -552,7 +573,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param approximateCountSamplingSize Sampling size to get the approximate count. Larger values increase accuracy but also increase execution time. #' @param savedFilter Use a saved filter at User level. #' @param includeInterpretation Interpretation ID to include the fields related to this interpretation. - #' @param id List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + #' @param id List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format organization@project:study. @@ -579,7 +600,7 @@ setMethod("clinicalClient", "OpencgaR", function(OpencgaR, annotationSet, clinic #' @param familyProband Specify the proband child to use for the family segregation. #' @param gene List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter. #' @param ct List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'. - #' @param xref List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ... + #' @param xref List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ... #' @param biotype List of biotypes, e.g. protein_coding. #' @param proteinSubstitution Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. #' @param conservation Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1. diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index 41ff659c258..da3baba3e62 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -24,11 +24,11 @@ #' | bioformats | /{apiVersion}/files/bioformats | | #' | create | /{apiVersion}/files/create | study, parents, body[*] | #' | distinct | /{apiVersion}/files/distinct | study, id, uuid, name, path, uri, type, bioformat, format, external, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release, field[*] | -#' | fetch | /{apiVersion}/files/fetch | jobId, jobDescription, jobDependsOn, jobTags, study, body[*] | +#' | fetch | /{apiVersion}/files/fetch | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body[*] | #' | formats | /{apiVersion}/files/formats | | #' | link | /{apiVersion}/files/link | study, parents, body[*] | -#' | runLink | /{apiVersion}/files/link/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runPostlink | /{apiVersion}/files/postlink/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runLink | /{apiVersion}/files/link/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runPostlink | /{apiVersion}/files/postlink/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | search | /{apiVersion}/files/search | include, exclude, limit, skip, count, flattenAnnotations, study, id, uuid, name, path, uri, type, bioformat, format, external, status, internalStatus, internalVariantIndexStatus, softwareName, directory, creationDate, modificationDate, description, tags, size, sampleIds, jobId, annotation, acl, deleted, release | #' | upload | /{apiVersion}/files/upload | file, fileName, fileFormat, bioformat, checksum, study, relativeFilePath, description, parents | #' | acl | /{apiVersion}/files/{files}/acl | files[*], study, member, silent | @@ -130,6 +130,9 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, file #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Fetch parameters. fetch=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -156,6 +159,9 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, file #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data File parameters. runLink=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory="link", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -167,6 +173,9 @@ setMethod("fileClient", "OpencgaR", function(OpencgaR, annotationSet, file, file #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data File parameters. runPostlink=fetchOpenCGA(object=OpencgaR, category="files", categoryId=NULL, subcategory="postlink", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index df80a2c29cc..32df3f06506 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -22,13 +22,14 @@ #' | updateAcl | /{apiVersion}/jobs/acl/{members}/update | members[*], action[*], body[*] | #' | create | /{apiVersion}/jobs/create | study, body[*] | #' | distinct | /{apiVersion}/jobs/distinct | study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field[*] | -#' | retry | /{apiVersion}/jobs/retry | jobId, jobDescription, jobDependsOn, jobTags, study, body[*] | +#' | retry | /{apiVersion}/jobs/retry | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, study, body[*] | #' | search | /{apiVersion}/jobs/search | include, exclude, limit, skip, count, study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted | #' | top | /{apiVersion}/jobs/top | limit, study, internalStatus, priority, userId, toolId | #' | acl | /{apiVersion}/jobs/{jobs}/acl | jobs[*], member, silent | #' | delete | /{apiVersion}/jobs/{jobs}/delete | study, jobs[*] | #' | info | /{apiVersion}/jobs/{jobs}/info | include, exclude, jobs[*], study, deleted | #' | update | /{apiVersion}/jobs/{jobs}/update | include, exclude, jobs[*], study, includeResult, body | +#' | kill | /{apiVersion}/jobs/{job}/kill | job[*], study | #' | headLog | /{apiVersion}/jobs/{job}/log/head | job[*], study, offset, lines, type | #' | tailLog | /{apiVersion}/jobs/{job}/log/tail | job[*], study, lines, type | #' @@ -88,6 +89,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data job. retry=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, @@ -169,6 +171,13 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi update=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=jobs, subcategory=NULL, subcategoryId=NULL, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/jobs/{job}/kill: + #' Send a signal to kill a pending or running job. + #' @param job Job ID or UUID. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + kill=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=job, subcategory=NULL, subcategoryId=NULL, + action="kill", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/jobs/{job}/log/head: #' Show the first lines of a log file (up to a limit). #' @param job Job ID or UUID. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index ecd586a9ed7..bd581f1ba5e 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -20,33 +20,33 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | configureCellbase | /{apiVersion}/operation/cellbase/configure | project, annotationUpdate, annotationSaveId, body | -#' | aggregateVariant | /{apiVersion}/operation/variant/aggregate | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | deleteVariantAnnotation | /{apiVersion}/operation/variant/annotation/delete | jobId, jobDescription, jobDependsOn, jobTags, project, annotationId | -#' | indexVariantAnnotation | /{apiVersion}/operation/variant/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | saveVariantAnnotation | /{apiVersion}/operation/variant/annotation/save | jobId, jobDescription, jobDependsOn, jobTags, project, body | +#' | aggregateVariant | /{apiVersion}/operation/variant/aggregate | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | deleteVariantAnnotation | /{apiVersion}/operation/variant/annotation/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, annotationId | +#' | indexVariantAnnotation | /{apiVersion}/operation/variant/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | saveVariantAnnotation | /{apiVersion}/operation/variant/annotation/save | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, body | #' | configureVariant | /{apiVersion}/operation/variant/configure | project, study, body | -#' | deleteVariant | /{apiVersion}/operation/variant/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | aggregateVariantFamily | /{apiVersion}/operation/variant/family/aggregate | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | indexVariantFamily | /{apiVersion}/operation/variant/family/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | indexVariant | /{apiVersion}/operation/variant/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | launcherVariantIndex | /{apiVersion}/operation/variant/index/launcher | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | runVariantJulie | /{apiVersion}/operation/variant/julie/run | jobId, jobDescription, jobDependsOn, jobTags, project, body[*] | -#' | repairVariantMetadata | /{apiVersion}/operation/variant/metadata/repair | jobId, jobDescription, jobDependsOn, jobTags, body | -#' | synchronizeVariantMetadata | /{apiVersion}/operation/variant/metadata/synchronize | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | pruneVariant | /{apiVersion}/operation/variant/prune | jobId, jobDescription, jobDependsOn, jobTags, body | -#' | deleteVariantSample | /{apiVersion}/operation/variant/sample/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | indexVariantSample | /{apiVersion}/operation/variant/sample/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | deleteVariant | /{apiVersion}/operation/variant/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | aggregateVariantFamily | /{apiVersion}/operation/variant/family/aggregate | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | indexVariantFamily | /{apiVersion}/operation/variant/family/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | indexVariant | /{apiVersion}/operation/variant/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | launcherVariantIndex | /{apiVersion}/operation/variant/index/launcher | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | runVariantJulie | /{apiVersion}/operation/variant/julie/run | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, body[*] | +#' | repairVariantMetadata | /{apiVersion}/operation/variant/metadata/repair | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body | +#' | synchronizeVariantMetadata | /{apiVersion}/operation/variant/metadata/synchronize | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | pruneVariant | /{apiVersion}/operation/variant/prune | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body | +#' | deleteVariantSample | /{apiVersion}/operation/variant/sample/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | indexVariantSample | /{apiVersion}/operation/variant/sample/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' | variantSampleIndexConfigure | /{apiVersion}/operation/variant/sample/index/configure | study, skipRebuild, body | -#' | deleteVariantScore | /{apiVersion}/operation/variant/score/delete | jobId, jobDescription, jobDependsOn, jobTags, study, name, resume, force | -#' | indexVariantScore | /{apiVersion}/operation/variant/score/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | -#' | variantSecondaryAnnotationIndex | /{apiVersion}/operation/variant/secondary/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | variantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | deleteVariantScore | /{apiVersion}/operation/variant/score/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, name, resume, force | +#' | indexVariantScore | /{apiVersion}/operation/variant/score/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | +#' | variantSecondaryAnnotationIndex | /{apiVersion}/operation/variant/secondary/annotation/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | variantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' | configureVariantSecondarySampleIndex | /{apiVersion}/operation/variant/secondary/sample/index/configure | study, skipRebuild, body | -#' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, project, study, body | -#' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, study, samples | -#' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, study, body | +#' | secondaryIndexVariant | /{apiVersion}/operation/variant/secondaryIndex | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, project, study, body | +#' | deleteVariantSecondaryIndex | /{apiVersion}/operation/variant/secondaryIndex/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, samples | +#' | deleteVariantStats | /{apiVersion}/operation/variant/stats/delete | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | indexVariantStats | /{apiVersion}/operation/variant/stats/index | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | deleteVariantStudy | /{apiVersion}/operation/variant/study/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, body | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -72,6 +72,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant aggregate params. aggregateVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", @@ -83,6 +86,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param annotationId Annotation identifier. deleteVariantAnnotation=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -95,6 +101,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant annotation index params. @@ -108,6 +117,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param data Variant annotation save params. saveVariantAnnotation=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -128,6 +140,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant delete file params. deleteVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", @@ -139,6 +154,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant aggregate family params. aggregateVariantFamily=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -151,6 +169,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant family index params. indexVariantFamily=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -163,6 +184,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant index params. indexVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", @@ -174,6 +198,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data . launcherVariantIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -186,6 +213,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project project. #' @param data Julie tool params. Specify list of cohorts from multiple studies with {study}:{cohort}. runVariantJulie=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -198,6 +228,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant storage metadata repair params. repairVariantMetadata=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/metadata", subcategoryId=NULL, action="repair", params=params, httpMethod="POST", @@ -209,6 +242,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant storage metadata synchronize params. synchronizeVariantMetadata=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -221,6 +257,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant prune params. Use dry-run to just generate a report with the orphan variants. pruneVariant=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant", subcategoryId=NULL, action="prune", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -231,6 +270,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant delete sample params. deleteVariantSample=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -243,6 +285,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant sample index params. indexVariantSample=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -264,6 +309,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param name Unique name of the score within the study. #' @param resume Resume a previously failed remove. @@ -278,6 +326,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant score index params. scoreName: Unique name of the score within the study. cohort1: Cohort used to compute the score. Use the cohort 'ALL' if all samples from the study where used to compute the score. cohort2: Second cohort used to compute the score, typically to compare against the first cohort. If only one cohort was used to compute the score, leave empty. inputColumns: Indicate which columns to load from the input file. Provide the column position (starting in 0) for the column with the score with 'SCORE=n'. Optionally, the PValue column with 'PVALUE=n'. The, to indicate the variant associated with the score, provide either the columns ['CHROM', 'POS', 'REF', 'ALT'], or the column 'VAR' containing a variant representation with format 'chr:start:ref:alt'. e.g. 'CHROM=0,POS=1,REF=3,ALT=4,SCORE=5,PVALUE=6' or 'VAR=0,SCORE=1,PVALUE=2'. resume: Resume a previously failed indexation. indexVariantScore=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -290,6 +341,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant secondary annotation index params. @@ -303,6 +357,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant sample index params. variantSecondarySampleIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -324,6 +381,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param project Project [organization@]project where project can be either the ID or the alias. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant secondary annotation index params. @@ -337,6 +397,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param samples Samples to remove. Needs to provide all the samples in the secondary index. deleteVariantSecondaryIndex=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, @@ -350,6 +413,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats delete params. deleteVariantStats=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/stats", subcategoryId=NULL, action="delete", params=params, httpMethod="POST", @@ -362,6 +428,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats params. indexVariantStats=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, subcategory="variant/stats", subcategoryId=NULL, action="index", params=params, httpMethod="POST", @@ -373,6 +442,9 @@ setMethod("operationClient", "OpencgaR", function(OpencgaR, endpointName, params #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param data Variant delete study params. deleteVariantStudy=fetchOpenCGA(object=OpencgaR, category="operation", categoryId=NULL, diff --git a/opencga-client/src/main/R/R/Organization-methods.R b/opencga-client/src/main/R/R/Organization-methods.R index 3efaf5eec2f..da241e16fb5 100644 --- a/opencga-client/src/main/R/R/Organization-methods.R +++ b/opencga-client/src/main/R/R/Organization-methods.R @@ -24,6 +24,9 @@ #' | searchNotes | /{apiVersion}/organizations/notes/search | include, exclude, creationDate, modificationDate, id, scope, visibility, uuid, userId, tags, version | #' | deleteNotes | /{apiVersion}/organizations/notes/{id}/delete | id[*], includeResult | #' | updateNotes | /{apiVersion}/organizations/notes/{id}/update | include, exclude, id[*], includeResult, body[*] | +#' | userUpdateStatus | /{apiVersion}/organizations/user/{user}/status/update | include, exclude, user[*], organization, includeResult, body[*] | +#' | updateUser | /{apiVersion}/organizations/user/{user}/update | include, exclude, user[*], organization, includeResult, body[*] | +#' | updateConfiguration | /{apiVersion}/organizations/{organization}/configuration/update | include, exclude, organization[*], includeResult, authenticationOriginsAction, body[*] | #' | info | /{apiVersion}/organizations/{organization}/info | include, exclude, organization[*] | #' | update | /{apiVersion}/organizations/{organization}/update | include, exclude, organization[*], includeResult, adminsAction, body[*] | #' @@ -33,7 +36,7 @@ #' [*]: Required parameter #' @export -setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, endpointName, params=NULL, ...) { +setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, user, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/organizations/create: @@ -87,6 +90,41 @@ setMethod("organizationClient", "OpencgaR", function(OpencgaR, id, organization, updateNotes=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="notes", subcategoryId=id, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/organizations/user/{user}/status/update: + #' Update the user status. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param user User ID. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the User fields to be updated. + userUpdateStatus=fetchOpenCGA(object=OpencgaR, category="organizations/user", categoryId=user, + subcategory="status", subcategoryId=NULL, action="update", params=params, httpMethod="POST", + as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/user/{user}/update: + #' Update the user information. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param user User ID. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing the User fields to be updated. + updateUser=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=NULL, subcategory="user", + subcategoryId=user, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/organizations/{organization}/configuration/update: + #' Update the Organization configuration attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param organization Organization id. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param authenticationOriginsAction Action to be performed if the array of authenticationOrigins is being updated. Allowed values: ['ADD SET REMOVE REPLACE'] + #' @param data JSON containing the params to be updated. + updateConfiguration=fetchOpenCGA(object=OpencgaR, category="organizations", categoryId=organization, + subcategory="configuration", subcategoryId=NULL, action="update", params=params, httpMethod="POST", + as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/organizations/{organization}/info: #' Return the organization information. #' @param include Fields included in the response, whole JSON path must be provided. diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index b23b8043f55..36c512a86f2 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -22,7 +22,7 @@ #' | updateAcl | /{apiVersion}/panels/acl/{members}/update | study, members[*], action[*], body[*] | #' | create | /{apiVersion}/panels/create | include, exclude, study, includeResult, body | #' | distinct | /{apiVersion}/panels/distinct | study, id, uuid, name, internalStatus, disorders, variants, genes, source, regions, categories, tags, deleted, status, creationDate, modificationDate, acl, release, snapshot, field[*] | -#' | importPanels | /{apiVersion}/panels/import | study, jobId, jobDependsOn, jobDescription, jobTags, body | +#' | importPanels | /{apiVersion}/panels/import | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body | #' | search | /{apiVersion}/panels/search | include, exclude, limit, skip, count, study, id, uuid, name, internalStatus, disorders, variants, genes, source, regions, categories, tags, deleted, status, creationDate, modificationDate, acl, release, snapshot | #' | acl | /{apiVersion}/panels/{panels}/acl | panels[*], study, member, silent | #' | delete | /{apiVersion}/panels/{panels}/delete | study, panels[*] | @@ -90,6 +90,9 @@ setMethod("panelClient", "OpencgaR", function(OpencgaR, members, panels, endpoin #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Panel parameters. importPanels=fetchOpenCGA(object=OpencgaR, category="panels", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="import", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index 95104111011..ddbc75ca374 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -34,7 +34,7 @@ #' | updateNotes | /{apiVersion}/studies/{study}/notes/{id}/update | include, exclude, study[*], id[*], includeResult, body[*] | #' | permissionRules | /{apiVersion}/studies/{study}/permissionRules | study[*], entity[*] | #' | updatePermissionRules | /{apiVersion}/studies/{study}/permissionRules/update | study[*], entity[*], action, body[*] | -#' | runTemplates | /{apiVersion}/studies/{study}/templates/run | study[*], jobId, jobDependsOn, jobDescription, jobTags, body[*] | +#' | runTemplates | /{apiVersion}/studies/{study}/templates/run | study[*], jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | uploadTemplates | /{apiVersion}/studies/{study}/templates/upload | file, study[*] | #' | deleteTemplates | /{apiVersion}/studies/{study}/templates/{templateId}/delete | study[*], templateId[*] | #' | update | /{apiVersion}/studies/{study}/update | include, exclude, study[*], includeResult, body[*] | @@ -221,6 +221,9 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, stud #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Template loader parameters. runTemplates=fetchOpenCGA(object=OpencgaR, category="studies", categoryId=study, subcategory="templates", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index 95fba46fac9..5413a7604b5 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -25,39 +25,39 @@ #' | runCircos | /{apiVersion}/analysis/variant/circos/run | study, body[*] | #' | deleteCohortStats | /{apiVersion}/analysis/variant/cohort/stats/delete | study, cohort | #' | infoCohortStats | /{apiVersion}/analysis/variant/cohort/stats/info | study, cohort[*] | -#' | runCohortStats | /{apiVersion}/analysis/variant/cohort/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runExomiser | /{apiVersion}/analysis/variant/exomiser/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runExport | /{apiVersion}/analysis/variant/export/run | include, exclude, project, study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runCohortStats | /{apiVersion}/analysis/variant/cohort/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runExomiser | /{apiVersion}/analysis/variant/exomiser/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runExport | /{apiVersion}/analysis/variant/export/run | include, exclude, project, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | genotypesFamily | /{apiVersion}/analysis/variant/family/genotypes | study, family, clinicalAnalysis, modeOfInheritance[*], penetrance, disorder | -#' | runFamilyQc | /{apiVersion}/analysis/variant/family/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | deleteFile | /{apiVersion}/analysis/variant/file/delete | jobId, jobDescription, jobDependsOn, jobTags, study, file, resume | -#' | runGatk | /{apiVersion}/analysis/variant/gatk/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runGenomePlot | /{apiVersion}/analysis/variant/genomePlot/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runGwas | /{apiVersion}/analysis/variant/gwas/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runHrDetect | /{apiVersion}/analysis/variant/hrDetect/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runIndex | /{apiVersion}/analysis/variant/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | -#' | runIndividualQc | /{apiVersion}/analysis/variant/individual/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runInferredSex | /{apiVersion}/analysis/variant/inferredSex/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runFamilyQc | /{apiVersion}/analysis/variant/family/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | deleteFile | /{apiVersion}/analysis/variant/file/delete | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, study, file, resume | +#' | runGatk | /{apiVersion}/analysis/variant/gatk/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runGenomePlot | /{apiVersion}/analysis/variant/genomePlot/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runGwas | /{apiVersion}/analysis/variant/gwas/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runHrDetect | /{apiVersion}/analysis/variant/hrDetect/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runIndex | /{apiVersion}/analysis/variant/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runIndividualQc | /{apiVersion}/analysis/variant/individual/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runInferredSex | /{apiVersion}/analysis/variant/inferredSex/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | queryKnockoutGene | /{apiVersion}/analysis/variant/knockout/gene/query | limit, skip, study, job | #' | queryKnockoutIndividual | /{apiVersion}/analysis/variant/knockout/individual/query | limit, skip, study, job | -#' | runKnockout | /{apiVersion}/analysis/variant/knockout/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runMendelianError | /{apiVersion}/analysis/variant/mendelianError/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runKnockout | /{apiVersion}/analysis/variant/knockout/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runMendelianError | /{apiVersion}/analysis/variant/mendelianError/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | metadata | /{apiVersion}/analysis/variant/metadata | project, study, file, sample, includeStudy, includeFile, includeSample, include, exclude | #' | queryMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/query | study, sample, type, ct, biotype, fileData, filter, qual, region, gene, panel, panelModeOfInheritance, panelConfidence, panelFeatureType, panelRoleInCancer, panelIntersection, msId, msDescription | -#' | runMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runPlink | /{apiVersion}/analysis/variant/plink/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runPlink | /{apiVersion}/analysis/variant/plink/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | query | /{apiVersion}/analysis/variant/query | include, exclude, limit, skip, count, sort, summary, approximateCount, approximateCountSamplingSize, savedFilter, id, region, type, reference, alternate, project, study, file, filter, qual, fileData, sample, genotype, sampleData, sampleAnnotation, sampleMetadata, unknownGenotype, sampleLimit, sampleSkip, cohort, cohortStatsRef, cohortStatsAlt, cohortStatsMaf, cohortStatsMgf, cohortStatsPass, missingAlleles, missingGenotypes, score, family, familyDisorder, familySegregation, familyMembers, familyProband, includeStudy, includeFile, includeSample, includeSampleData, includeGenotype, includeSampleId, annotationExists, gene, ct, xref, biotype, proteinSubstitution, conservation, populationFrequencyAlt, populationFrequencyRef, populationFrequencyMaf, transcriptFlag, geneTraitId, go, expression, proteinKeyword, drug, functionalScore, clinical, clinicalSignificance, clinicalConfirmedStatus, customAnnotation, panel, panelModeOfInheritance, panelConfidence, panelRoleInCancer, panelFeatureType, panelIntersection, trait | -#' | runRelatedness | /{apiVersion}/analysis/variant/relatedness/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runRvtests | /{apiVersion}/analysis/variant/rvtests/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runRelatedness | /{apiVersion}/analysis/variant/relatedness/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runRvtests | /{apiVersion}/analysis/variant/rvtests/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | aggregationStatsSample | /{apiVersion}/analysis/variant/sample/aggregationStats | savedFilter, region, type, project, study, file, filter, sample, genotype, sampleAnnotation, family, familyDisorder, familySegregation, familyMembers, familyProband, ct, biotype, populationFrequencyAlt, clinical, clinicalSignificance, clinicalConfirmedStatus, field | -#' | runSampleEligibility | /{apiVersion}/analysis/variant/sample/eligibility/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runSampleQc | /{apiVersion}/analysis/variant/sample/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runSampleEligibility | /{apiVersion}/analysis/variant/sample/eligibility/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runSampleQc | /{apiVersion}/analysis/variant/sample/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | querySample | /{apiVersion}/analysis/variant/sample/query | limit, skip, variant, study, genotype | -#' | runSample | /{apiVersion}/analysis/variant/sample/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runSample | /{apiVersion}/analysis/variant/sample/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | querySampleStats | /{apiVersion}/analysis/variant/sample/stats/query | region, type, study, file, filter, sampleData, ct, biotype, transcriptFlag, populationFrequencyAlt, clinical, clinicalSignificance, clinicalConfirmedStatus, study, filterTranscript, sample[*] | -#' | runSampleStats | /{apiVersion}/analysis/variant/sample/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runStatsExport | /{apiVersion}/analysis/variant/stats/export/run | project, study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | -#' | runStats | /{apiVersion}/analysis/variant/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runSampleStats | /{apiVersion}/analysis/variant/sample/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runStatsExport | /{apiVersion}/analysis/variant/stats/export/run | project, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runStats | /{apiVersion}/analysis/variant/stats/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' #' @md #' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation @@ -87,7 +87,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param annotationExists Return only annotated variants. #' @param gene List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter. #' @param ct List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'. - #' @param xref List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ... + #' @param xref List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ... #' @param biotype List of biotypes, e.g. protein_coding. #' @param proteinSubstitution Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. #' @param conservation Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1. @@ -121,7 +121,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @section Endpoint /{apiVersion}/analysis/variant/annotation/query: #' Query variant annotations from any saved versions. - #' @param id List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + #' @param id List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param include Fields included in the response, whole JSON path must be provided. #' @param exclude Fields excluded in the response, whole JSON path must be provided. @@ -162,6 +162,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Cohort variant stats params. runCohortStats=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/cohort/stats", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -174,6 +177,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Exomiser parameters. runExomiser=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/exomiser", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -188,6 +194,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant export params. runExport=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/export", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -211,6 +220,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Family QC analysis params. Family ID. Relatedness method, by default 'PLINK/IBD'. Minor allele frequence (MAF) is used to filter variants before computing relatedness, e.g.: 1000G:CEU>0.35 or cohort:ALL>0.05. runFamilyQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/family/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -222,6 +234,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. #' @param file Files to remove. #' @param resume Resume a previously failed indexation. @@ -235,6 +250,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gatk parameters. Supported Gatk commands: HaplotypeCaller. runGatk=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/gatk", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -246,6 +264,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Genome plot analysis params to customize the plot. The configuration file includes the title, the plot density (i.e., the number of points to display), the general query and the list of tracks. Currently, the supported track types are: COPY-NUMBER, INDEL, REARRANGEMENT and SNV. In addition, each track can contain a specific query. runGenomePlot=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/genomePlot", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -258,6 +279,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gwas analysis params. runGwas=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/gwas", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -269,6 +293,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data HRDetect analysis parameters. runHrDetect=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/hrDetect", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -280,6 +307,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant index params. runIndex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/index", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -291,6 +321,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Individual QC analysis params. runIndividualQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/individual/qc", subcategoryId=NULL, action="run", params=params, @@ -303,6 +336,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Inferred sex analysis params. runInferredSex=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/inferredSex", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -335,6 +371,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Gene knockout analysis params. runKnockout=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/knockout", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -346,6 +385,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Mendelian error analysis params. runMendelianError=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/mendelianError", subcategoryId=NULL, action="run", params=params, @@ -396,6 +438,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Mutational signature analysis parameters to index the genome context for that sample, and to compute both catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. runMutationalSignature=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/mutationalSignature", subcategoryId=NULL, action="run", params=params, @@ -408,6 +453,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Plink params. runPlink=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/plink", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -424,7 +472,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param approximateCount Get an approximate count, instead of an exact total count. Reduces execution time. #' @param approximateCountSamplingSize Sampling size to get the approximate count. Larger values increase accuracy but also increase execution time. #' @param savedFilter Use a saved filter at User level. - #' @param id List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + #' @param id List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. #' @param region List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. #' @param type List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, DELETION, DUPLICATION, TANDEM_DUPLICATION, BREAKEND, e.g. SNV,INDEL. #' @param reference Reference allele. @@ -466,7 +514,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param annotationExists Return only annotated variants. #' @param gene List of genes, most gene IDs are accepted (HGNC, Ensembl gene, ...). This is an alias to 'xref' parameter. #' @param ct List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'. - #' @param xref List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ... + #' @param xref List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ... #' @param biotype List of biotypes, e.g. protein_coding. #' @param proteinSubstitution Protein substitution scores include SIFT and PolyPhen. You can query using the score {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. #' @param conservation Filter by conservation score: {conservation_score}[<|>|<=|>=]{number} e.g. phastCons>0.5,phylop<0.1,gerp>0.1. @@ -501,6 +549,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Relatedness analysis params. runRelatedness=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/relatedness", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -513,6 +564,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data RvTests parameters. Supported RvTests commands: rvtest, vcf2kinship. runRvtests=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/rvtests", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -552,6 +606,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data . runSampleEligibility=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/eligibility", subcategoryId=NULL, action="run", params=params, @@ -564,6 +621,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. In order to skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, signature-fitting, genome-plot. runSampleQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -586,6 +646,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Sample variant filter params. runSample=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -618,6 +681,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Sample variant stats params. Use index=true and indexId='' to store the result in catalog sample QC. indexId=ALL requires an empty query. Use sample=all to compute sample stats of all samples in the variant storage. runSampleStats=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/stats", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -631,6 +697,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats export params. runStatsExport=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/stats/export", subcategoryId=NULL, action="run", params=params, httpMethod="POST", @@ -643,6 +712,9 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. #' @param data Variant stats params. runStats=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/stats", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index ca1e1ecc744..238df94066d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -65,6 +65,10 @@ public AlignmentClient(String token, ClientConfiguration configuration) { * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -83,6 +87,10 @@ public RestResponse runBwa(BwaWrapperParams data, ObjectMap params) throws * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -101,6 +109,10 @@ public RestResponse runCoverageIndex(CoverageIndexParams data, ObjectMap pa * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -184,6 +196,10 @@ public RestResponse statsCoverage(String file, String gene, O * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -202,6 +218,10 @@ public RestResponse runDeeptools(DeeptoolsWrapperParams data, ObjectMap par * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -220,6 +240,10 @@ public RestResponse runFastqc(FastqcWrapperParams data, ObjectMap params) t * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -239,6 +263,10 @@ public RestResponse runIndex(AlignmentIndexParams data, ObjectMap params) t * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -258,6 +286,10 @@ public RestResponse runPicard(PicardWrapperParams data, ObjectMap params) t * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -309,6 +341,10 @@ public RestResponse query(String file, ObjectMap params) throws C * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index 976ea17a4f6..869c987e791 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -277,6 +277,10 @@ public RestResponse infoInterpretation(String interpretations, O * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -296,6 +300,10 @@ public RestResponse runInterpreterCancerTiering(CancerTieringInterpretation * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -314,6 +322,10 @@ public RestResponse runInterpreterExomiser(ExomiserInterpretationAnalysisPa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -332,6 +344,10 @@ public RestResponse runInterpreterTeam(TeamInterpretationAnalysisParams dat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -350,6 +366,10 @@ public RestResponse runInterpreterTiering(TieringInterpretationAnalysisPara * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -368,6 +388,10 @@ public RestResponse runInterpreterZetta(ZettaInterpretationAnalysisParams d * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -498,6 +522,10 @@ public RestResponse summaryRgaGene(ObjectMap params) thro * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * auxiliarIndex: Index auxiliar collection to improve performance assuming RGA is completely indexed. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -715,7 +743,7 @@ public RestResponse search(ObjectMap params) throws ClientExce * execution time. * savedFilter: Use a saved filter at User level. * includeInterpretation: Interpretation ID to include the fields related to this interpretation. - * id: List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + * id: List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. * region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: * 2,3:100000-200000. * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, @@ -763,7 +791,7 @@ public RestResponse search(ObjectMap params) throws ClientExce * ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' * and 'protein_altering'. * xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - * dbSNP, ClinVar, HPO, Cosmic, ... + * dbSNP, ClinVar, HPO, Cosmic, HGVS ... * biotype: List of biotypes, e.g. protein_coding. * proteinSubstitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index ead924800fa..15d71d98597 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -139,6 +139,10 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 09cc5ab8c25..dcf3ae18ef8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -179,6 +179,10 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -223,6 +227,10 @@ public RestResponse link(FileLinkParams data, ObjectMap params) throws Cli * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -241,6 +249,10 @@ public RestResponse runLink(FileLinkToolParams data, ObjectMap params) thro * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index 523523ab4b3..77c04be5053 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -125,6 +125,7 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -254,6 +255,19 @@ public RestResponse update(String jobs, JobUpdateParams data, ObjectMap par return execute("jobs", jobs, null, null, "update", params, POST, Job.class); } + /** + * Send a signal to kill a pending or running job. + * @param job Job ID or UUID. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse kill(String job, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("jobs", job, null, null, "kill", params, POST, Job.class); + } + /** * Show the first lines of a log file (up to a limit). * @param job Job ID or UUID. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java index aebe0ba61f5..67d55693b52 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/OrganizationClient.java @@ -24,8 +24,12 @@ import org.opencb.opencga.core.models.notes.NoteCreateParams; import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserStatusUpdateParams; import org.opencb.opencga.core.response.RestResponse; @@ -134,6 +138,61 @@ public RestResponse updateNotes(String id, NoteUpdateParams data, ObjectMa return execute("organizations", null, "notes", id, "update", params, POST, Note.class); } + /** + * Update the user status. + * @param user User ID. + * @param data JSON containing the User fields to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * organization: Organization id. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse userUpdateStatus(String user, UserStatusUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations/user", user, "status", null, "update", params, POST, User.class); + } + + /** + * Update the user information. + * @param user User ID. + * @param data JSON containing the User fields to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * organization: Organization id. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateUser(String user, OrganizationUserUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", null, "user", user, "update", params, POST, User.class); + } + + /** + * Update the Organization configuration attributes. + * @param organization Organization id. + * @param data JSON containing the params to be updated. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * includeResult: Flag indicating to include the created or updated document result in the response. + * authenticationOriginsAction: Action to be performed if the array of authenticationOrigins is being updated. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateConfiguration(String organization, OrganizationConfiguration data, ObjectMap + params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("organizations", organization, "configuration", null, "update", params, POST, OrganizationConfiguration.class); + } + /** * Return the organization information. * @param organization Organization id. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 71e89be53fb..e5b054ae283 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -341,6 +341,10 @@ public RestResponse updatePermissionRules(String study, String e * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index 93ae92a1d88..6a68ae8ea82 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -32,6 +32,7 @@ import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsExportParams; import org.opencb.opencga.core.models.variant.CircosAnalysisParams; import org.opencb.opencga.core.models.variant.CohortVariantStatsAnalysisParams; @@ -53,7 +54,6 @@ import org.opencb.opencga.core.models.variant.SampleVariantFilterParams; import org.opencb.opencga.core.models.variant.SampleVariantStatsAnalysisParams; import org.opencb.opencga.core.models.variant.VariantExportParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.models.variant.VariantStatsAnalysisParams; import org.opencb.opencga.core.response.RestResponse; @@ -103,7 +103,7 @@ public VariantClient(String token, ClientConfiguration configuration) { * ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' * and 'protein_altering'. * xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - * dbSNP, ClinVar, HPO, Cosmic, ... + * dbSNP, ClinVar, HPO, Cosmic, HGVS ... * biotype: List of biotypes, e.g. protein_coding. * proteinSubstitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. @@ -150,7 +150,7 @@ public RestResponse metadataAnnotation(ObjectMap params) throws Clien /** * Query variant annotations from any saved versions. * @param params Map containing any of the following optional parameters. - * id: List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + * id: List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. * region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: * 2,3:100000-200000. * include: Fields included in the response, whole JSON path must be provided. @@ -218,6 +218,10 @@ public RestResponse infoCohortStats(String cohort, ObjectMap pa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -236,6 +240,10 @@ public RestResponse runCohortStats(CohortVariantStatsAnalysisParams data, O * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -257,6 +265,10 @@ public RestResponse runExomiser(ExomiserWrapperParams data, ObjectMap param * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -294,6 +306,10 @@ public RestResponse genotypesFamily(String modeOfInheritance, ObjectM * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -310,6 +326,10 @@ public RestResponse runFamilyQc(FamilyQcAnalysisParams data, ObjectMap para * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * file: Files to remove. * resume: Resume a previously failed indexation. @@ -330,6 +350,10 @@ public RestResponse deleteFile(ObjectMap params) throws ClientException { * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -350,6 +374,10 @@ public RestResponse runGatk(GatkWrapperParams data, ObjectMap params) throw * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -368,6 +396,10 @@ public RestResponse runGenomePlot(GenomePlotAnalysisParams data, ObjectMap * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -386,6 +418,10 @@ public RestResponse runGwas(GwasAnalysisParams data, ObjectMap params) thro * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -404,6 +440,10 @@ public RestResponse runHrDetect(HRDetectAnalysisParams data, ObjectMap para * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobDescription: Job description. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -422,6 +462,10 @@ public RestResponse runIndex(VariantIndexParams data, ObjectMap params) thr * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -440,6 +484,10 @@ public RestResponse runIndividualQc(IndividualQcAnalysisParams data, Object * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -488,6 +536,10 @@ public RestResponse queryKnockoutIndividual(ObjectMap para * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -506,6 +558,10 @@ public RestResponse runKnockout(KnockoutAnalysisParams data, ObjectMap para * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -597,6 +653,10 @@ public RestResponse queryMutationalSignature(ObjectMap params) throws * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -615,6 +675,10 @@ public RestResponse runMutationalSignature(MutationalSignatureAnalysisParam * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -638,7 +702,7 @@ public RestResponse runPlink(PlinkWrapperParams data, ObjectMap params) thr * approximateCountSamplingSize: Sampling size to get the approximate count. Larger values increase accuracy but also increase * execution time. * savedFilter: Use a saved filter at User level. - * id: List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + * id: List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. * region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: * 2,3:100000-200000. * type: List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, COPY_NUMBER_GAIN, INSERTION, @@ -710,7 +774,7 @@ public RestResponse runPlink(PlinkWrapperParams data, ObjectMap params) thr * ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' * and 'protein_altering'. * xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - * dbSNP, ClinVar, HPO, Cosmic, ... + * dbSNP, ClinVar, HPO, Cosmic, HGVS ... * biotype: List of biotypes, e.g. protein_coding. * proteinSubstitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. @@ -759,6 +823,10 @@ public RestResponse query(ObjectMap params) throws ClientException { * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -777,6 +845,10 @@ public RestResponse runRelatedness(RelatednessAnalysisParams data, ObjectMa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -851,6 +923,10 @@ public RestResponse aggregationStatsSample(ObjectMap params) throws * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -872,6 +948,10 @@ public RestResponse runSampleEligibility(SampleEligibilityAnalysisParams da * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -906,6 +986,10 @@ public RestResponse querySample(ObjectMap params) throws ClientExceptio * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -960,6 +1044,10 @@ public RestResponse querySampleStats(String sample, ObjectMa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -979,6 +1067,10 @@ public RestResponse runSampleStats(SampleVariantStatsAnalysisParams data, O * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -997,6 +1089,10 @@ public RestResponse runStatsExport(VariantStatsExportParams data, ObjectMap * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index 9731eb2bcc0..e3f43735a7d 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -28,21 +28,21 @@ import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; import org.opencb.opencga.core.models.operations.variant.VariantAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantAnnotationSaveParams; +import org.opencb.opencga.core.models.operations.variant.VariantConfigureParams; import org.opencb.opencga.core.models.operations.variant.VariantFamilyIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantFileIndexJobLauncherParams; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; +import org.opencb.opencga.core.models.operations.variant.VariantPruneParams; +import org.opencb.opencga.core.models.operations.variant.VariantSampleDeleteParams; import org.opencb.opencga.core.models.operations.variant.VariantScoreIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondaryAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsDeleteParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataRepairToolParams; -import org.opencb.opencga.core.models.variant.VariantConfigureParams; -import org.opencb.opencga.core.models.variant.VariantFileDeleteParams; -import org.opencb.opencga.core.models.variant.VariantFileIndexJobLauncherParams; -import org.opencb.opencga.core.models.variant.VariantIndexParams; -import org.opencb.opencga.core.models.variant.VariantPruneParams; -import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; -import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; -import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantStorageMetadataSynchronizeParams; +import org.opencb.opencga.core.models.operations.variant.VariantStudyDeleteParams; import org.opencb.opencga.core.response.RestResponse; @@ -90,6 +90,10 @@ public RestResponse configureCellbase(CellBaseConfiguration data, ObjectMap * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -107,6 +111,10 @@ public RestResponse aggregateVariant(VariantAggregateParams data, ObjectMap * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: Project [organization@]project where project can be either the ID or the alias. * annotationId: Annotation identifier. * @return a RestResponse object. @@ -125,6 +133,10 @@ public RestResponse deleteVariantAnnotation(ObjectMap params) throws Client * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: Project [organization@]project where project can be either the ID or the alias. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. @@ -144,6 +156,10 @@ public RestResponse indexVariantAnnotation(VariantAnnotationIndexParams dat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: Project [organization@]project where project can be either the ID or the alias. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -177,6 +193,10 @@ public RestResponse configureVariant(VariantConfigureParams data, Obj * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -195,6 +215,10 @@ public RestResponse deleteVariant(VariantFileDeleteParams data, ObjectMap p * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -213,6 +237,10 @@ public RestResponse aggregateVariantFamily(VariantAggregateFamilyParams dat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -231,6 +259,10 @@ public RestResponse indexVariantFamily(VariantFamilyIndexParams data, Objec * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -249,6 +281,10 @@ public RestResponse indexVariant(VariantIndexParams data, ObjectMap params) * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -267,6 +303,10 @@ public RestResponse launcherVariantIndex(VariantFileIndexJobLauncherParams * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: project. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -285,6 +325,10 @@ public RestResponse runVariantJulie(JulieParams data, ObjectMap params) thr * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -302,6 +346,10 @@ public RestResponse repairVariantMetadata(VariantStorageMetadataRepairToolP * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -321,6 +369,10 @@ public RestResponse synchronizeVariantMetadata(VariantStorageMetadataSynchr * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -338,6 +390,10 @@ public RestResponse pruneVariant(VariantPruneParams data, ObjectMap params) * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -356,6 +412,10 @@ public RestResponse deleteVariantSample(VariantSampleDeleteParams data, Obj * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -388,6 +448,10 @@ public RestResponse variantSampleIndexConfigure(SampleIndexConfiguration da * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * name: Unique name of the score within the study. * resume: Resume a previously failed remove. @@ -415,6 +479,10 @@ public RestResponse deleteVariantScore(ObjectMap params) throws ClientExcep * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -433,6 +501,10 @@ public RestResponse indexVariantScore(VariantScoreIndexParams data, ObjectM * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: Project [organization@]project where project can be either the ID or the alias. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. @@ -453,6 +525,10 @@ public RestResponse variantSecondaryAnnotationIndex(VariantSecondaryAnnotat * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. @@ -486,6 +562,10 @@ public RestResponse configureVariantSecondarySampleIndex(SampleIndexConfigu * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * project: Project [organization@]project where project can be either the ID or the alias. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. @@ -504,6 +584,10 @@ public RestResponse secondaryIndexVariant(VariantSecondaryAnnotationIndexPa * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * samples: Samples to remove. Needs to provide all the samples in the secondary index. * @return a RestResponse object. @@ -523,6 +607,10 @@ public RestResponse deleteVariantSecondaryIndex(ObjectMap params) throws Cl * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -541,6 +629,10 @@ public RestResponse deleteVariantStats(VariantStatsDeleteParams data, Objec * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -558,6 +650,10 @@ public RestResponse indexVariantStats(VariantStatsIndexParams data, ObjectM * jobDescription: Job description. * jobDependsOn: Comma separated list of existing job IDs the job will depend on. * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index 0b374aab6a5..fc302177872 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -43,6 +43,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runBwa(data, params) { @@ -58,6 +62,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runCoverageIndex(data, params) { @@ -73,6 +81,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ coverageQcGeneCoverageStatsRun(data, params) { @@ -138,6 +150,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runDeeptools(data, params) { @@ -153,6 +169,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runFastqc(data, params) { @@ -168,6 +188,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runIndex(data, params) { @@ -184,6 +208,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runPicard(data, params) { @@ -200,6 +228,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runQc(data, params) { @@ -246,6 +278,10 @@ export default class Alignment extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSamtools(data, params) { diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 265e9e28a2b..f2e4031455c 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -209,6 +209,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterCancerTiering(data, params) { @@ -224,6 +228,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterExomiser(data, params) { @@ -239,6 +247,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterTeam(data, params) { @@ -254,6 +266,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterTiering(data, params) { @@ -269,6 +285,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInterpreterZetta(data, params) { @@ -284,6 +304,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ load(data, params) { @@ -401,6 +425,10 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {Boolean} [params.auxiliarIndex = "false"] - Index auxiliar collection to improve performance assuming RGA is completely * indexed. The default value is false. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -602,8 +630,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * but also increase execution time. * @param {String} [params.savedFilter] - Use a saved filter at User level. * @param {String} [params.includeInterpretation] - Interpretation ID to include the fields related to this interpretation. - * @param {String} [params.id] - List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. - * rs116600158,19:7177679:C:T. + * @param {String} [params.id] - List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. * @param {String} [params.region] - List of regions, these can be just a single chromosome name or regions in the format chr:start-end, * e.g.: 2,3:100000-200000. * @param {String} [params.type] - List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, @@ -655,7 +682,7 @@ export default class ClinicalAnalysis extends OpenCGAParentClass { * @param {String} [params.ct] - List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases * 'loss_of_function' and 'protein_altering'. * @param {String} [params.xref] - List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, - * Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ... + * Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ... * @param {String} [params.biotype] - List of biotypes, e.g. protein_coding. * @param {String} [params.proteinSubstitution] - Protein substitution scores include SIFT and PolyPhen. You can query using the score * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index b23920896d2..1aca2e4ef57 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -109,6 +109,10 @@ export default class DiseasePanel extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ importPanels(data, params) { diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index 80c275318ef..3e1abab6f85 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -131,6 +131,10 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -166,6 +170,10 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runLink(data, params) { @@ -181,6 +189,10 @@ export default class File extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runPostlink(data, params) { diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index e9014ed296f..f0ebb640147 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -95,6 +95,7 @@ export default class Job extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -205,6 +206,16 @@ export default class Job extends OpenCGAParentClass { return this._post("jobs", jobs, null, null, "update", data, params); } + /** Send a signal to kill a pending or running job + * @param {String} job - Job ID or UUID. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + kill(job, params) { + return this._post("jobs", job, null, null, "kill", params); + } + /** Show the first lines of a log file (up to a limit) * @param {String} job - Job ID or UUID. * @param {Object} [params] - The Object containing the following optional parameters: diff --git a/opencga-client/src/main/javascript/Organization.js b/opencga-client/src/main/javascript/Organization.js index 4385a545eb0..1580a4a7828 100644 --- a/opencga-client/src/main/javascript/Organization.js +++ b/opencga-client/src/main/javascript/Organization.js @@ -105,6 +105,52 @@ export default class Organization extends OpenCGAParentClass { return this._post("organizations", null, "notes", id, "update", data, params); } + /** Update the user status + * @param {String} user - User ID. + * @param {Object} data - JSON containing the User fields to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.organization] - Organization id. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + userUpdateStatus(user, data, params) { + return this._post("organizations/user", user, "status", null, "update", data, params); + } + + /** Update the user information + * @param {String} user - User ID. + * @param {Object} data - JSON containing the User fields to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.organization] - Organization id. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateUser(user, data, params) { + return this._post("organizations", null, "user", user, "update", data, params); + } + + /** Update the Organization configuration attributes + * @param {String} organization - Organization id. + * @param {Object} data - JSON containing the params to be updated. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @param {"ADD SET REMOVE REPLACE"} [params.authenticationOriginsAction = "ADD"] - Action to be performed if the array of + * authenticationOrigins is being updated. The default value is ADD. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateConfiguration(organization, data, params) { + return this._post("organizations", organization, "configuration", null, "update", data, params); + } + /** Return the organization information * @param {String} organization - Organization id. * @param {Object} [params] - The Object containing the following optional parameters: diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 1be4598d5bc..9e013fef0dc 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -262,6 +262,10 @@ export default class Study extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runTemplates(study, data, params) { diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index 177fdffdae3..7d8b01966da 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -59,7 +59,7 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.ct] - List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases * 'loss_of_function' and 'protein_altering'. * @param {String} [params.xref] - List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, - * Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ... + * Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ... * @param {String} [params.biotype] - List of biotypes, e.g. protein_coding. * @param {String} [params.proteinSubstitution] - Protein substitution scores include SIFT and PolyPhen. You can query using the score * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. @@ -105,8 +105,7 @@ export default class Variant extends OpenCGAParentClass { /** Query variant annotations from any saved versions * @param {Object} [params] - The Object containing the following optional parameters: - * @param {String} [params.id] - List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. - * rs116600158,19:7177679:C:T. + * @param {String} [params.id] - List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. * @param {String} [params.region] - List of regions, these can be just a single chromosome name or regions in the format chr:start-end, * e.g.: 2,3:100000-200000. * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. @@ -161,6 +160,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runCohortStats(data, params) { @@ -176,6 +179,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runExomiser(data, params) { @@ -194,6 +201,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runExport(data, params) { @@ -225,6 +236,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runFamilyQc(data, params) { @@ -238,6 +253,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.file] - Files to remove. * @param {Boolean} [params.resume] - Resume a previously failed indexation. @@ -256,6 +275,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runGatk(data, params) { @@ -273,6 +296,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runGenomePlot(data, params) { @@ -288,6 +315,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runGwas(data, params) { @@ -303,6 +334,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runHrDetect(data, params) { @@ -318,6 +353,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runIndex(data, params) { @@ -333,6 +372,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runIndividualQc(data, params) { @@ -348,6 +391,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runInferredSex(data, params) { @@ -387,6 +434,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runKnockout(data, params) { @@ -402,6 +453,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runMendelianError(data, params) { @@ -487,6 +542,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runMutationalSignature(data, params) { @@ -502,6 +561,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runPlink(data, params) { @@ -521,8 +584,7 @@ export default class Variant extends OpenCGAParentClass { * @param {Number} [params.approximateCountSamplingSize] - Sampling size to get the approximate count. Larger values increase accuracy * but also increase execution time. * @param {String} [params.savedFilter] - Use a saved filter at User level. - * @param {String} [params.id] - List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. - * rs116600158,19:7177679:C:T. + * @param {String} [params.id] - List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T. * @param {String} [params.region] - List of regions, these can be just a single chromosome name or regions in the format chr:start-end, * e.g.: 2,3:100000-200000. * @param {String} [params.type] - List of types, accepted values are SNV, MNV, INDEL, SV, COPY_NUMBER, COPY_NUMBER_LOSS, @@ -597,7 +659,7 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.ct] - List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases * 'loss_of_function' and 'protein_altering'. * @param {String} [params.xref] - List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, - * Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ... + * Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ... * @param {String} [params.biotype] - List of biotypes, e.g. protein_coding. * @param {String} [params.proteinSubstitution] - Protein substitution scores include SIFT and PolyPhen. You can query using the score * {protein_score}[<|>|<=|>=]{number} or the description {protein_score}[~=|=]{description} e.g. polyphen>0.1,sift=tolerant. @@ -650,6 +712,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runRelatedness(data, params) { @@ -665,6 +731,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runRvtests(data, params) { @@ -735,6 +805,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSampleEligibility(data, params) { @@ -753,6 +827,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSampleQc(data, params) { @@ -781,6 +859,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSample(data, params) { @@ -828,6 +910,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runSampleStats(data, params) { @@ -844,6 +930,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runStatsExport(data, params) { @@ -859,6 +949,10 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ runStats(data, params) { diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index 6f2ce1afafa..0d79024c496 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -54,6 +54,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -68,6 +72,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.annotationId] - Annotation identifier. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -84,6 +92,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -100,6 +112,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -126,6 +142,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -141,6 +161,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -156,6 +180,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -171,6 +199,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -186,6 +218,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -201,6 +237,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - project. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -216,6 +256,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ repairVariantMetadata(data, params) { @@ -230,6 +274,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -245,6 +293,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ pruneVariant(data, params) { @@ -259,6 +311,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -274,6 +330,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -299,6 +359,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.name] - Unique name of the score within the study. * @param {Boolean} [params.resume] - Resume a previously failed remove. @@ -324,6 +388,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -339,6 +407,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -355,6 +427,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ @@ -381,6 +457,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.project] - Project [organization@]project where project can be either the ID or the alias. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -396,6 +476,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.samples] - Samples to remove. Needs to provide all the samples in the secondary index. * @returns {Promise} Promise object in the form of RestResponse instance. @@ -413,6 +497,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ deleteVariantStats(data, params) { @@ -428,6 +516,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @returns {Promise} Promise object in the form of RestResponse instance. */ indexVariantStats(data, params) { @@ -442,6 +534,10 @@ export default class VariantOperation extends OpenCGAParentClass { * @param {String} [params.jobDescription] - Job description. * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @returns {Promise} Promise object in the form of RestResponse instance. */ diff --git a/opencga-client/src/main/python/README.rst b/opencga-client/src/main/python/README.rst index d8c3f51b64c..43b39e1c983 100644 --- a/opencga-client/src/main/python/README.rst +++ b/opencga-client/src/main/python/README.rst @@ -4,34 +4,28 @@ PyOpenCGA ========== This Python client package makes use of the comprehensive RESTful web services API implemented for the `OpenCGA`_ platform. -OpenCGA is an open-source project that implements a high-performance, scalable and secure platform for Genomic data analysis and visualisation +OpenCGA is an open-source project that implements a high-performance, scalable and secure platform for Genomic data analysis and visualisation. OpenCGA implements a secure and high performance platform for Big Data analysis and visualisation in current genomics. -OpenCGA uses the most modern and advanced technologies to scale to petabytes of data. OpenCGA is designed and implemented to work with -few million genomes. It is built on top of three main components: Catalog, Variant and Alignment Storage and Analysis. +OpenCGA uses the most modern and advanced technologies to scale to petabytes of data. OpenCGA is designed and implemented to work with few million genomes. It is built on top of three main components: Catalog, Variant and Alignment Storage and Analysis. -More info about this project in the `OpenCGA Docs`_ +More info about this project in `OpenCGA Docs`_ Installation ------------ -Cloning -``````` -PyOpenCGA can be cloned in your local machine by executing in your terminal:: +PyOpenCGA can be installed from the Pypi repository. Make sure you have pip available in your machine. You can check this by running:: - $ git clone https://github.com/opencb/opencga.git + $ python3 -m pip --version -Once you have downloaded the project you can install the library. We recommend to install it inside a `virtual environment`_:: - $ cd opencga/tree/develop/opencga-client/src/main/python/pyOpenCGA - $ python setup.py install +If you don't have Python or pip, please refer to https://packaging.python.org/en/latest/tutorials/installing-packages/ -Pip install -``````````` -Run the following command in the shell:: +To install PyOpencga, run the following command in the shell:: $ pip install pyopencga + Usage ----- @@ -48,14 +42,14 @@ The first step is to import the ClientConfiguration and OpenCGAClient from pyOpe Setting up server host configuration ```````````````````````````````````` -The second step is to generate a ClientConfiguration instance by passing a configuration dictionary containing the host to point to or a client-configuration.yml file: +The second step is to generate a ClientConfiguration instance by passing a configuration dictionary containing the opencga host OR a client-configuration.yml file with that information: .. code-block:: python >>> config = ClientConfiguration('/opt/opencga/conf/client-configuration.yml') >>> config = ClientConfiguration({ "rest": { - "host": "http://bioinfo.hpc.cam.ac.uk/opencga-demo" + "host": "https://demo.app.zettagenomics.com/opencga" } }) @@ -67,33 +61,22 @@ With this configuration you can initialize the OpenCGAClient, and log in: .. code-block:: python >>> oc = OpenCGAClient(config) - >>> oc.login('user') - -For scripting or using Jupyter Notebooks is preferable to load user credentials from an external JSON file. - -Once you are logged in, it is mandatory to use the token of the session to propagate the access of the clients to the host server: - -.. code-block:: python - - >>> token = oc.token - >>> print(token) - eyJhbGciOi... - - >>> oc = OpenCGAClient(configuration=config_dict, token=token) + >>> oc.login(user='user', password='pass', organization='organization') Examples ```````` -The next step is to get an instance of the clients we may want to use: +The first step is to get an instance of the clients we may want to use: .. code-block:: python - >>> projects = oc.projects # Project client - >>> studies = oc.studies # Study client - >>> samples = oc.samples # Sample client - >>> cohorts = oc.cohorts # Cohort client + >>> projects = oc.projects # Project client + >>> studies = oc.studies # Study client + >>> samples = oc.samples # Sample client + >>> individuals = oc.individuals # Individual client + >>> cohorts = oc.cohorts # Cohort client -Now you can start asking to the OpenCGA RESTful service with pyOpenCGA: +Now you can start querying with pyOpenCGA: .. code-block:: python @@ -103,41 +86,38 @@ Now you can start asking to the OpenCGA RESTful service with pyOpenCGA: project2 [...] -There are two different ways to access to the query response data: +There are two different ways to access query response data: .. code-block:: python - >>> foo_client.method().get_results() # Iterates over all the results of all the QueryResults - >>> foo_client.method().get_responses() # Iterates over all the responses + >>> foo_client.method().get_responses() # Iterates over all the responses + >>> foo_client.method().get_results() # Iterates over all the results of the first response -Data can be accessed specifying comma-separated IDs or a list of IDs: +Data can be accessed specifying comma-separated IDs or a list of IDs. -.. code-block:: python +e.g. Retrieving individual karyotypic sex for a list of individuals: - >>> samples = 'NA12877,NA12878,NA12879' - >>> samples_list = ['NA12877','NA12878','NA12879'] - >>> sc = oc.samples +.. code-block:: python - >>> for result in sc.info(query_id=samples, study='user@project1:study1').get_results(): - ... print(result['id'], result['attributes']['OPENCGA_INDIVIDUAL']['disorders']) - NA12877 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] - NA12878 [] - NA12879 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] + >>> for result in oc.samples.info(samples='NA12877,NA12878,NA12889', study='platinum').get_results(): + ... print(result['id'], result['karyotypicSex']) + NA12877 XY + NA12878 XX + NA12889 XY - >>> for result in sc.info(query_id=samples_list, study='user@project1:study1').get_results(): - ... print(result['id'], result['attributes']['OPENCGA_INDIVIDUAL']['disorders']) - NA12877 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] - NA12878 [] - NA12879 [{'id': 'OMIM6500', 'name': "Chron's Disease"}] + >>> for result in oc.samples.info(samples=['NA12877', 'NA12878', 'NA12889'], study='platinum').get_results(): + ... print(result['id'], result['karyotypicSex']) + NA12877 XY + NA12878 XX + NA12889 XY Optional filters and extra options can be added as key-value parameters (where the values can be a comma-separated string or a list). What can I ask for? ``````````````````` -The best way to know which data can be retrieved for each client check `OpenCGA web services`_ swagger. - +The best way to know which data can be retrieved for each client, log into `OpenCGA Demo`_ and check the **OpenCGA REST API** in the **About** section (at the top right corner of the screen). .. _OpenCGA: https://github.com/opencb/opencga .. _OpenCGA Docs: http://docs.opencb.org/display/opencga -.. _virtual environment: https://help.dreamhost.com/hc/en-us/articles/115000695551-Installing-and-using-virtualenv-with-Python-3 -.. _OpenCGA web services: http://bioinfodev.hpc.cam.ac.uk/opencga/webservices/ +.. _OpenCGA REST API: https://demo.app.zettagenomics.com/ +.. _OpenCGA Demo: https://demo.app.zettagenomics.com/ diff --git a/opencga-client/src/main/python/pyopencga/__init__.py b/opencga-client/src/main/python/pyopencga/__init__.py index d781d658402..e69de29bb2d 100644 --- a/opencga-client/src/main/python/pyopencga/__init__.py +++ b/opencga-client/src/main/python/pyopencga/__init__.py @@ -1,2 +0,0 @@ -"refactoring from from fork pyCGA 7f2e3e404 branch release-1.4.0" -__author__="dev_team@mgviz.org" \ No newline at end of file diff --git a/opencga-client/src/main/python/pyopencga/opencga_client.py b/opencga-client/src/main/python/pyopencga/opencga_client.py index 0d2937ca638..4c5baf4506e 100644 --- a/opencga-client/src/main/python/pyopencga/opencga_client.py +++ b/opencga-client/src/main/python/pyopencga/opencga_client.py @@ -2,11 +2,15 @@ import time import sys import re +if sys.version_info >= (3, 8): + from importlib.metadata import version +else: + from importlib_metadata import version from pyopencga.opencga_config import ClientConfiguration from pyopencga.rest_clients.admin_client import Admin from pyopencga.rest_clients.alignment_client import Alignment -from pyopencga.rest_clients.clinical_client import Clinical +from pyopencga.rest_clients.clinical_analysis_client import ClinicalAnalysis from pyopencga.rest_clients.cohort_client import Cohort from pyopencga.rest_clients.family_client import Family from pyopencga.rest_clients.file_client import File @@ -21,6 +25,7 @@ from pyopencga.rest_clients.variant_operation_client import VariantOperation from pyopencga.rest_clients.user_client import User from pyopencga.rest_clients.variant_client import Variant +from pyopencga.rest_clients.organization_client import Organization class OpencgaClient(object): @@ -50,8 +55,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.logout() def _check_versions(self): + # Getting client and server versions + client_version = version("pyopencga") server_version = self.meta.about().get_result(0)['Version'].split('-')[0] - client_version = re.findall(r'Client version: (.+)\n', str(self.meta.__doc__))[0] + ansi_reset = "\033[0m" ansi_red = "\033[31m" ansi_yellow = "\033[33m" @@ -60,10 +67,12 @@ def _check_versions(self): ' Some client features may not be implemented in the server.\n'.format(client_version, server_version) sys.stdout.write('{}{}{}'.format(ansi_red, msg, ansi_reset)) elif tuple(server_version.split('.')[:2]) > tuple(client_version.split('.')[:2]): - msg = '[INFO]: Client version ({}) is lower than server version ({}).\n'.format(client_version, server_version) + msg = '[INFO]: Client version ({}) is lower than server version ({}).' \ + ' Some client features may not work as intended.\n'.format(client_version, server_version) sys.stdout.write('{}{}{}'.format(ansi_yellow, msg, ansi_reset)) def _create_clients(self): + self.organizations = Organization(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.users = User(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.projects = Project(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.studies = Study(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) @@ -76,7 +85,7 @@ def _create_clients(self): self.disease_panels = DiseasePanel(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.alignments = Alignment(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.variants = Variant(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) - self.clinical = Clinical(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) + self.clinical = ClinicalAnalysis(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.operations = VariantOperation(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.variant_operations = self.operations # DEPRECATED: use 'self.operations' self.meta = Meta(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) @@ -84,7 +93,7 @@ def _create_clients(self): self.admin = Admin(self.configuration, self.token, self._login_handler, auto_refresh=self.auto_refresh) self.clients = [ - self.users, self.projects, self.studies, self.files, self.jobs, + self.organizations, self.users, self.projects, self.studies, self.files, self.jobs, self.samples, self.individuals, self.families, self.cohorts, self.disease_panels, self.alignments, self.variants, self.clinical, self.variant_operations, self.meta, self.ga4gh, self.admin @@ -93,7 +102,7 @@ def _create_clients(self): for client in self.clients: client.on_retry = self.on_retry - def _make_login_handler(self, user, password): + def _make_login_handler(self, user, password, organization): """ Returns a closure that performs the log-in. This will be called on retries if the current session ever expires. @@ -109,13 +118,15 @@ def login_handler(refresh=False): data = {'user': user, 'password': password} else: data = {'user': user, 'password': password} + if organization: + data.update({'organization': organization}) tokens = User(self.configuration).login(data=data).get_result(0) self.token = tokens['token'] self.refresh_token = tokens['refreshToken'] return self.token return login_handler - def login(self, user=None, password=None): + def login(self, user=None, password=None, organization=None): if user is not None: if password is None: password = getpass.getpass() @@ -125,7 +136,7 @@ def login(self, user=None, password=None): except AssertionError: raise ValueError("User and password required") - self._login_handler = self._make_login_handler(user, password) + self._login_handler = self._make_login_handler(user, password, organization) self._login_handler() for client in self.clients: client.token = self.token @@ -174,15 +185,14 @@ def _get_help_info(self, client_name=None, parameters=False): # Description and path class_docstring = client.__doc__ - cls_desc = re.findall('(.+)\n +Client version', class_docstring)[0] - cls_desc = cls_desc.strip().replace('This class contains methods', 'Client') + cls_desc = re.findall('(This class contains methods .+)\n', class_docstring)[0] cls_path = re.findall('PATH: (.+)\n', class_docstring)[0] # Methods methods = [] method_names = [method_name for method_name in dir(client) if callable(getattr(client, method_name)) - and not method_name.startswith('_')] + and not method_name.startswith('_') and method_name != 'login_handler'] for method_name in method_names: if client_name is None: continue @@ -251,6 +261,9 @@ def help(self, client_name=None, show_parameters=False): )] sys.stdout.write('\n'.join(help_txt) + '\n') + def get_organization_client(self): + return self.organizations + def get_user_client(self): return self.users diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index 4924a3e1233..0374b6d4ecc 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -33,6 +33,13 @@ def run_bwa(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/bwa', data=data, **options) @@ -50,6 +57,13 @@ def run_coverage_index(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/coverage/index', data=data, **options) @@ -69,6 +83,13 @@ def coverage_qc_gene_coverage_stats_run(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/coverage/qc/geneCoverageStats', data=data, **options) @@ -164,6 +185,13 @@ def run_deeptools(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/deeptools', data=data, **options) @@ -181,6 +209,13 @@ def run_fastqc(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/fastqc', data=data, **options) @@ -198,6 +233,13 @@ def run_index(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/index', data=data, **options) @@ -219,6 +261,13 @@ def run_picard(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/picard', data=data, **options) @@ -241,6 +290,13 @@ def run_qc(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/qc', data=data, **options) @@ -302,6 +358,13 @@ def run_samtools(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='alignment/samtools', data=data, **options) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index e66873eba68..6ccab3a988a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -264,6 +264,13 @@ def run_interpreter_cancer_tiering(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/cancerTiering', data=data, **options) @@ -282,6 +289,13 @@ def run_interpreter_exomiser(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/exomiser', data=data, **options) @@ -300,6 +314,13 @@ def run_interpreter_team(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/team', data=data, **options) @@ -318,6 +339,13 @@ def run_interpreter_tiering(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/tiering', data=data, **options) @@ -336,6 +364,13 @@ def run_interpreter_zetta(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='clinical/interpreter/zetta', data=data, **options) @@ -355,6 +390,13 @@ def load(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='load', subcategory='clinical', data=data, **options) @@ -489,6 +531,13 @@ def run_rga_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param bool auxiliar_index: Index auxiliar collection to improve performance assuming RGA is completely indexed. """ @@ -730,8 +779,8 @@ def query_variant(self, **options): :param str saved_filter: Use a saved filter at User level. :param str include_interpretation: Interpretation ID to include the fields related to this interpretation. - :param str id: List of IDs, these can be rs IDs (dbSNP) or variants in - the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + :param str id: List of variant IDs in the format chrom:start:ref:alt, + e.g. 19:7177679:C:T. :param str region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. @@ -817,7 +866,7 @@ def query_variant(self, **options): aliases 'loss_of_function' and 'protein_altering'. :param str xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - dbSNP, ClinVar, HPO, Cosmic, ... + dbSNP, ClinVar, HPO, Cosmic, HGVS ... :param str biotype: List of biotypes, e.g. protein_coding. :param str protein_substitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index dff08d5a0f8..8634c983e2e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -133,6 +133,13 @@ def import_panels(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param dict data: Panel parameters. """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index f922f15b910..505d21464b2 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -162,6 +162,13 @@ def fetch(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. """ @@ -204,6 +211,13 @@ def run_link(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='files', resource='run', subcategory='link', data=data, **options) @@ -223,6 +237,13 @@ def run_postlink(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='files', resource='run', subcategory='postlink', data=data, **options) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 459007a179f..c8425b16e6a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -106,6 +106,8 @@ def retry(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. """ @@ -249,6 +251,18 @@ def update(self, jobs, data=None, **options): return self._post(category='jobs', resource='update', query_id=jobs, data=data, **options) + def kill(self, job, **options): + """ + Send a signal to kill a pending or running job. + PATH: /{apiVersion}/jobs/{job}/kill + + :param str job: Job ID or UUID. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + return self._post(category='jobs', resource='kill', query_id=job, **options) + def head_log(self, job, **options): """ Show the first lines of a log file (up to a limit). diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py index a3124bfcf45..0c695ca0a0e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/organization_client.py @@ -110,6 +110,64 @@ def update_notes(self, id, data=None, **options): return self._post(category='organizations', resource='update', subcategory='notes', second_query_id=id, data=data, **options) + def user_update_status(self, user, data=None, **options): + """ + Update the user status. + PATH: /{apiVersion}/organizations/user/{user}/status/update + + :param dict data: JSON containing the User fields to be updated. + (REQUIRED) + :param str user: User ID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str organization: Organization id. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations/user', resource='update', query_id=user, subcategory='status', data=data, **options) + + def update_user(self, user, data=None, **options): + """ + Update the user information. + PATH: /{apiVersion}/organizations/user/{user}/update + + :param dict data: JSON containing the User fields to be updated. + (REQUIRED) + :param str user: User ID. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str organization: Organization id. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='organizations', resource='update', subcategory='user', second_query_id=user, data=data, **options) + + def update_configuration(self, organization, data=None, **options): + """ + Update the Organization configuration attributes. + PATH: /{apiVersion}/organizations/{organization}/configuration/update + + :param dict data: JSON containing the params to be updated. (REQUIRED) + :param str organization: Organization id. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param str authentication_origins_action: Action to be performed if + the array of authenticationOrigins is being updated. Allowed + values: ['ADD SET REMOVE REPLACE'] + """ + + return self._post(category='organizations', resource='update', query_id=organization, subcategory='configuration', data=data, **options) + def info(self, organization, **options): """ Return the organization information. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 57826554192..93e70823d52 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -330,6 +330,13 @@ def run_templates(self, study, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='studies', resource='run', query_id=study, subcategory='templates', data=data, **options) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index e1d63675309..3993f48ba22 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -63,7 +63,7 @@ def aggregation_stats(self, **options): aliases 'loss_of_function' and 'protein_altering'. :param str xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - dbSNP, ClinVar, HPO, Cosmic, ... + dbSNP, ClinVar, HPO, Cosmic, HGVS ... :param str biotype: List of biotypes, e.g. protein_coding. :param str protein_substitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score @@ -124,8 +124,8 @@ def query_annotation(self, **options): Query variant annotations from any saved versions. PATH: /{apiVersion}/analysis/variant/annotation/query - :param str id: List of IDs, these can be rs IDs (dbSNP) or variants in - the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + :param str id: List of variant IDs in the format chrom:start:ref:alt, + e.g. 19:7177679:C:T. :param str region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. @@ -194,6 +194,13 @@ def run_cohort_stats(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/cohort/stats', data=data, **options) @@ -212,6 +219,13 @@ def run_exomiser(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/exomiser', data=data, **options) @@ -236,6 +250,13 @@ def run_export(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/export', data=data, **options) @@ -279,6 +300,13 @@ def run_family_qc(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/family/qc', data=data, **options) @@ -294,6 +322,13 @@ def delete_file(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str file: Files to remove. @@ -318,6 +353,13 @@ def run_gatk(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/gatk', data=data, **options) @@ -341,6 +383,13 @@ def run_genome_plot(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/genomePlot', data=data, **options) @@ -359,6 +408,13 @@ def run_gwas(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/gwas', data=data, **options) @@ -377,6 +433,13 @@ def run_hr_detect(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/hrDetect', data=data, **options) @@ -395,6 +458,13 @@ def run_index(self, data=None, **options): the job will depend on. :param str job_description: Job description. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/index', data=data, **options) @@ -414,6 +484,13 @@ def run_individual_qc(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/individual/qc', data=data, **options) @@ -432,6 +509,13 @@ def run_inferred_sex(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/inferredSex', data=data, **options) @@ -475,6 +559,13 @@ def run_knockout(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/knockout', data=data, **options) @@ -493,6 +584,13 @@ def run_mendelian_error(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/mendelianError', data=data, **options) @@ -615,6 +713,13 @@ def run_mutational_signature(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/mutationalSignature', data=data, **options) @@ -633,6 +738,13 @@ def run_plink(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/plink', data=data, **options) @@ -659,8 +771,8 @@ def query(self, **options): approximate count. Larger values increase accuracy but also increase execution time. :param str saved_filter: Use a saved filter at User level. - :param str id: List of IDs, these can be rs IDs (dbSNP) or variants in - the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T. + :param str id: List of variant IDs in the format chrom:start:ref:alt, + e.g. 19:7177679:C:T. :param str region: List of regions, these can be just a single chromosome name or regions in the format chr:start-end, e.g.: 2,3:100000-200000. @@ -785,7 +897,7 @@ def query(self, **options): aliases 'loss_of_function' and 'protein_altering'. :param str xref: List of any external reference, these can be genes, proteins or variants. Accepted IDs include HGNC, Ensembl genes, - dbSNP, ClinVar, HPO, Cosmic, ... + dbSNP, ClinVar, HPO, Cosmic, HGVS ... :param str biotype: List of biotypes, e.g. protein_coding. :param str protein_substitution: Protein substitution scores include SIFT and PolyPhen. You can query using the score @@ -857,6 +969,13 @@ def run_relatedness(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/relatedness', data=data, **options) @@ -876,6 +995,13 @@ def run_rvtests(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/rvtests', data=data, **options) @@ -979,6 +1105,13 @@ def run_sample_eligibility(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample/eligibility', data=data, **options) @@ -1003,6 +1136,13 @@ def run_sample_qc(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample/qc', data=data, **options) @@ -1036,6 +1176,13 @@ def run_sample(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample', data=data, **options) @@ -1102,6 +1249,13 @@ def run_sample_stats(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/sample/stats', data=data, **options) @@ -1122,6 +1276,13 @@ def run_stats_export(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/stats/export', data=data, **options) @@ -1140,6 +1301,13 @@ def run_stats(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='analysis', resource='run', subcategory='variant/stats', data=data, **options) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index e726e9dd2db..98dfc7d4c25 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -47,6 +47,13 @@ def aggregate_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant aggregate params. @@ -65,6 +72,13 @@ def delete_variant_annotation(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: Project [organization@]project where project can be either the ID or the alias. :param str annotation_id: Annotation identifier. @@ -83,6 +97,13 @@ def index_variant_annotation(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: Project [organization@]project where project can be either the ID or the alias. :param str study: Study [[organization@]project:]study where study and @@ -103,6 +124,13 @@ def save_variant_annotation(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: Project [organization@]project where project can be either the ID or the alias. :param dict data: Variant annotation save params. @@ -136,6 +164,13 @@ def delete_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant delete file params. @@ -155,6 +190,13 @@ def aggregate_variant_family(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant aggregate family params. @@ -173,6 +215,13 @@ def index_variant_family(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant family index params. @@ -191,6 +240,13 @@ def index_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant index params. @@ -210,6 +266,13 @@ def launcher_variant_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: . @@ -231,6 +294,13 @@ def run_variant_julie(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: project. """ @@ -247,6 +317,13 @@ def repair_variant_metadata(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param dict data: Variant storage metadata repair params. """ @@ -263,6 +340,13 @@ def synchronize_variant_metadata(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant storage metadata synchronize params. @@ -281,6 +365,13 @@ def prune_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param dict data: Variant prune params. Use dry-run to just generate a report with the orphan variants. """ @@ -298,6 +389,13 @@ def delete_variant_sample(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant delete sample params. @@ -316,6 +414,13 @@ def index_variant_sample(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant sample index params. @@ -347,6 +452,13 @@ def delete_variant_score(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str name: Unique name of the score within the study. @@ -367,6 +479,13 @@ def index_variant_score(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant score index params. scoreName: Unique name @@ -401,6 +520,13 @@ def variant_secondary_annotation_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: Project [organization@]project where project can be either the ID or the alias. :param str study: Study [[organization@]project:]study where study and @@ -421,6 +547,13 @@ def variant_secondary_sample_index(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant sample index params. @@ -452,6 +585,13 @@ def secondary_index_variant(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str project: Project [organization@]project where project can be either the ID or the alias. :param str study: Study [[organization@]project:]study where study and @@ -473,6 +613,13 @@ def delete_variant_secondary_index(self, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str samples: Samples to remove. Needs to provide all the @@ -495,6 +642,13 @@ def delete_variant_stats(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='operation', resource='delete', subcategory='variant/stats', data=data, **options) @@ -514,6 +668,13 @@ def index_variant_stats(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. """ return self._post(category='operation', resource='index', subcategory='variant/stats', data=data, **options) @@ -529,6 +690,13 @@ def delete_variant_study(self, data=None, **options): :param str job_depends_on: Comma separated list of existing job IDs the job will depend on. :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param dict data: Variant delete study params. diff --git a/opencga-client/src/main/python/setup.py b/opencga-client/src/main/python/setup.py index 0764eeda3db..6be7c84c577 100644 --- a/opencga-client/src/main/python/setup.py +++ b/opencga-client/src/main/python/setup.py @@ -21,14 +21,13 @@ url='https://github.com/opencb/opencga/tree/develop/opencga-client/src/main/python/pyopencga', packages=['pyopencga', 'pyopencga.rest_clients'], license='Apache Software License', - author='David Gomez-Peregrina, Pablo Marin-Garcia, Daniel Perez-Gil', - author_email='david.gomez@mgviz.org, pmarin@kanteron.com, dp529@cam.ac.uk', + author='Pablo Marin-Garcia, Daniel Perez-Gil', + author_email='pablo.marin@zettagenomics.com, daniel.perez@zettagenomics.com', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Topic :: Scientific/Engineering :: Bio-Informatics', 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', ], diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 16aff526b3e..02a59d91cf6 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 73047a516e9..c59da1f754d 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index c2d7cfe4961..835f89a61a6 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -68,9 +68,7 @@ public class FieldConstants { public static final String NOTES_VISIBILITY_PARAM = "visibility"; //Sample - public static final String SAMPLE_ID_DESCRIPTION = "Sample data model hosts information about any biological material, normally " - + "extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most " - + "basic and important information."; + public static final String SAMPLE_ID_DESCRIPTION = "Sample ID."; public static final String SAMPLE_PROCESSING_DESCRIPTION = "Describes how the sample was processed in the lab."; public static final String SAMPLE_SAMPLE_COLLECTION_DESCRIPTION = "Describes how the sample was collected."; public static final String SAMPLE_QUALITY_CONTROL_DESCRIPTION = "Contains different metrics to evaluate the quality of the sample."; @@ -341,16 +339,25 @@ public class FieldConstants { + "ERROR, UNKNOWN, REGISTERING, UNREGISTERED, ABORTED, DELETED."; public static final String JOB_INTERNAL_EVENTS_DESCRIPTION = "Events of the internal job."; public static final String JOB_INTERNAL_WEBHOOK_DESCRIPTION = "Job internal Webhook."; + public static final String JOB_INTERNAL_KILL_JOB_REQUESTED_DESCRIPTION = "Flag used to track whether a user has requested to" + + " terminate the execution of a job before its natural completion. When set to true, it indicates that a request has been" + + " made to prematurely terminate the ongoing job execution. Its value is typically checked periodically during the" + + " execution of the job to determine if termination is necessary."; //JobInternalWebhook public static final String JOB_INTERNAL_WEBHOOK_URL_DESCRIPTION = "Webhook URL."; public static final String JOB_INTERNAL_WEBHOOK_STATUS_DESCRIPTION = "Webhook status map can have the values SUCCESS or ERROR."; + public static final String JOB_PARENT_ID = "jobParentId"; + public static final String JOB_SCHEDULED_START_TIME = "jobScheduledStartTime"; + public static final String JOB_OUT_DIR_DESCRIPTION = "Output dir for the job."; public static final String JOB_INPUT_DESCRIPTION = "List of input files."; public static final String JOB_OUTPUT_DESCRIPTION = "List of output files."; public static final String JOB_TAGS_DESCRIPTION = "List of tags for the job."; public static final String JOB_DEPENDS_ON_DESCRIPTION = "List of jobs the current job depends on."; + public static final String JOB_PARENT_ID_DESCRIPTION = "Id of the job that generated this job (if any)"; + public static final String JOB_SCHEDULED_START_TIME_DESCRIPTION = "Time when the job is scheduled to start."; public static final String JOB_EXECUTION_DESCRIPTION = "Result of the execution."; //ExecutorInfo @@ -516,6 +523,16 @@ public class FieldConstants { public static final String VARIANT_STATS_DESCRIPTION_DESCRIPTION = "Variant stats description."; public static final String VARIANT_STATS_QUERY_DESCRIPTION = "Variant stats query in JSON format."; + // Alignment index parameter descriptions + public static final String ALIGNMENT_INDEX_FILE_ID_DESCRIPTION = "File ID, (i.e., BAM/CRAM file ID)."; + public static final String ALIGNMENT_INDEX_OVERWRITE_DESCRIPTION = "Flag to force indexing."; + + // Coverage index parameter descriptions + public static final String COVERAGE_INDEX_BAM_FILE_ID_DESCRIPTION = "BAM file ID."; + public static final String COVERAGE_INDEX_BAI_FILE_ID_DESCRIPTION = "BAI file ID."; + public static final String COVERAGE_INDEX_OVERWRITE_DESCRIPTION = "Window size (i.e., the size of the bins, in bases, for the output" + + " of the BIGWIG file)."; + // Alignment QC analysis (asample-qc-run) public static final String ALIGNMENT_QC_BAM_FILE_DESCRIPTION = "ID for the BAM file to process."; public static final String ALIGNMENT_QC_SKIP_DESCRIPTION = "To skip any alignment QC metrics use the following keywords (separated by" @@ -523,4 +540,6 @@ public class FieldConstants { + AlignmentQcParams.FASTQC_METRICS_SKIP_VALUE; public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; + // Exomiser + public static final String EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION = "Clinical analysis type: SINGLE or FAMILY."; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 6bf964e760f..0c5c0d27c4b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -80,8 +80,8 @@ public class ParamConstants { private static final String UP_TO_100 = " up to a maximum of 100"; public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase"; - public static final String CELLBASE_VERSION = "v5.2"; - public static final String CELLBASE_DATA_RELEASE = "3"; + public static final String CELLBASE_VERSION = "v5.8"; + public static final String CELLBASE_DATA_RELEASE_GRCH38 = "7"; public static final String CELLBASE_APIKEY = ""; public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3"; @@ -91,6 +91,8 @@ public class ParamConstants { public static final String OPENCGA_TOKEN_CLI_PARAM = "--opencga-token"; + public static final String RESUME_DESCRIPTION = "Resume a previously failed index operation"; + public static final String CATEGORY = "category"; public static final String CATEGORY_DESCRIPTION = "Category corresponding to the id's provided."; @@ -558,6 +560,7 @@ public class ParamConstants { public static final String JOB_USER_PARAM = "userId"; public static final String JOB_USER_DESCRIPTION = "User that created the job"; public static final String JOB_PRIORITY_PARAM = "priority"; + public static final String SUBMIT_JOB_PRIORITY_PARAM = "jobPriority"; public static final String JOB_PRIORITY_DESCRIPTION = "Priority of the job"; public static final String JOB_INTERNAL_STATUS_PARAM = INTERNAL_STATUS_PARAM; public static final String JOB_INTERNAL_STATUS_DESCRIPTION = INTERNAL_STATUS_DESCRIPTION; @@ -568,6 +571,12 @@ public class ParamConstants { public static final String JOB_TAGS = "jobTags"; public static final String JOB_TAGS_PARAM = "tags"; public static final String JOB_TAGS_DESCRIPTION = "Job tags"; + public static final String JOB_SCHEDULED_START_TIME_DESCRIPTION = FieldConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION; + public static final String JOB_SCHEDULED_START_TIME = FieldConstants.JOB_SCHEDULED_START_TIME; + public static final String JOB_DRY_RUN = "jobDryRun"; + public static final String JOB_DRY_RUN_DESCRIPTION = "Flag indicating that the job will be executed in dry-run mode. In this mode," + + " OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will" + + " not actually run."; // --------------------------------------------- public static final String JOB_INPUT_FILES_PARAM = "input"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index 5ed331143db..4dd5543e9b8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -171,7 +171,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException logger.warn("DataRelease not supported on version '" + serverVersion + ".x'"); } else { String dataRelease = getDataRelease(); - if (dataRelease == null) { + if (StringUtils.isEmpty(dataRelease)) { if (autoComplete) { cellBaseConfiguration.setDataRelease(getDefaultDataRelease()); } else { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java index 19338531443..0a5a1ad4f9f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/MailUtils.java @@ -33,19 +33,18 @@ public class MailUtils { private static final Logger logger = LoggerFactory.getLogger(MailUtils.class); public static void sendResetPasswordMail(String to, String newPassword, final String mailUser, final String mailPassword, - String mailHost, String mailPort) throws Exception { - sendResetPasswordMail(to, newPassword, mailUser, mailPassword, - mailHost, mailPort, "true"); - } - - public static void sendResetPasswordMail(String to, String newPassword, final String mailUser, final String mailPassword, - String mailHost, String mailPort, String ssl) throws Exception { + String mailHost, String mailPort, String userId) throws Exception { Properties props = new Properties(); - props.put("mail.smtp.auth", "true"); - props.put("mail.smtp.starttls.enable", ssl); props.put("mail.smtp.host", mailHost); props.put("mail.smtp.port", mailPort); + props.put("mail.smtp.auth", "true"); + props.put("mail.smtp.ssl.enable", "true"); + props.put("mail.smtp.starttls.enable", "true"); + props.put("mail.smtp.starttls.required", "true"); + props.put("mail.smtp.ssl.protocols", "TLSv1.2"); + props.put("mail.smtp.ssl.checkserveridentity", "true"); + props.put("mail.smtp.socketFactory.class", "javax.net.ssl.SSLSocketFactory"); Session session = Session.getInstance(props, new javax.mail.Authenticator() { @@ -59,20 +58,31 @@ protected PasswordAuthentication getPasswordAuthentication() { message.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to)); - message.setSubject("Your password has been reset"); - message.setText("Hello, \n" + - "You can now login using this new password:" + - "\n\n" + - newPassword + - "\n\n\n" + - "Please change it when you first login" + - "\n\n" + - "Best regards,\n\n" + - "Systems Genomics Laboratory" + - "\n"); + message.setSubject("XetaBase: Password Reset"); + message.setText(getEmailContent(userId,newPassword)); Transport.send(message); } + public static String getEmailContent(String userId, String temporaryPassword) { + StringBuilder sb = new StringBuilder(); + + sb.append("Hi ").append(userId).append(",\n\n"); + sb.append("We confirm that your password has been successfully reset.\n\n"); + sb.append("Please find your new login credentials below:\n\n"); + sb.append("User ID: ").append(userId).append("\n"); + sb.append("Temporary Password: ").append(temporaryPassword).append("\n\n"); + sb.append("For your security, we strongly recommend that you log in using the temporary password provided "); + sb.append("and promptly create a new password that is unique and known only to you. "); + sb.append("You can change your password by accessing \"Your Profile > Change Password\" in your User Profile.\n\n"); + sb.append("If you did not request a password reset, please contact our support team immediately at support@zettagenomics.com.\n\n"); + sb.append("Best regards,\n\n"); + sb.append("ZettaGenomics Support Team \n\n"); + + + + return sb.toString(); + } + public static void sendMail(String smtpServer, String to, String from, String subject, String body) throws Exception { Properties props = System.getProperties(); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java index 65c2cfda80b..da6aa39f1f6 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java @@ -127,6 +127,14 @@ public static Date add1MonthtoDate(Date date) { return new Date(cal.getTimeInMillis()); } + public static Date add1YeartoDate(Date date) { + Calendar cal = new GregorianCalendar(); + cal.setTime(date); + cal.setTimeInMillis(date.getTime()); + cal.add(Calendar.YEAR, 1); + return new Date(cal.getTimeInMillis()); + } + public static Date toDate(String dateStr) { Date date = null; try { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AbstractServerConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AbstractServerConfiguration.java index 060c72ceacc..cffc7fe68b8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/AbstractServerConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AbstractServerConfiguration.java @@ -16,13 +16,13 @@ package org.opencb.opencga.core.config; + /** * Created by imedina on 25/04/16. */ public abstract class AbstractServerConfiguration { protected int port; - protected String logFile; public AbstractServerConfiguration() { } @@ -35,7 +35,6 @@ public AbstractServerConfiguration(int port) { public String toString() { final StringBuilder sb = new StringBuilder("ServerConfiguration{"); sb.append("port=").append(port); - sb.append(", logFile='").append(logFile).append('\''); sb.append('}'); return sb.toString(); } @@ -49,13 +48,9 @@ public AbstractServerConfiguration setPort(int port) { return this; } - public String getLogFile() { - return logFile; - } - - public AbstractServerConfiguration setLogFile(String logFile) { - this.logFile = logFile; - return this; + @Deprecated + protected void setLogFile(Object o) { + Configuration.reportUnusedField("configuration.yml#server.[rest|grpc].logFile", o); } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java index 2b8bfab8624..2b4962b024a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/AuthenticationOrigin.java @@ -31,7 +31,8 @@ public class AuthenticationOrigin { public enum AuthenticationType { OPENCGA, LDAP, - AzureAD + AzureAD, + SSO } // Possible keys of the options map diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java index 65fe5b7a4b8..0df3425dfba 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Catalog.java @@ -16,27 +16,33 @@ package org.opencb.opencga.core.config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Created by pfurio on 01/02/17. */ public class Catalog { private DatabaseCredentials database; - private DatabaseCredentials searchEngine; + + private static final Logger logger; + + static { + logger = LoggerFactory.getLogger(Catalog.class); + } public Catalog() { } - public Catalog(DatabaseCredentials database, DatabaseCredentials searchEngine) { + public Catalog(DatabaseCredentials database) { this.database = database; - this.searchEngine = searchEngine; } @Override public String toString() { final StringBuilder sb = new StringBuilder("Catalog{"); sb.append("database=").append(database); - sb.append(", searchEngine=").append(searchEngine); sb.append('}'); return sb.toString(); } @@ -50,12 +56,15 @@ public Catalog setDatabase(DatabaseCredentials database) { return this; } + @Deprecated public DatabaseCredentials getSearchEngine() { - return searchEngine; + return null; } + @Deprecated public Catalog setSearchEngine(DatabaseCredentials searchEngine) { - this.searchEngine = searchEngine; + logger.warn("Ignored configuration option 'configuration.yml#catalog.searchEngine' with value '{}'." + + " The option was deprecated and removed.", searchEngine); return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index bb5057a76e2..600def109ff 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -40,14 +40,12 @@ public class Configuration { */ private String logDir; - @Deprecated - private boolean openRegister; - private String databasePrefix; private String workspace; private String jobDir; - private Admin admin; + private int maxLoginAttempts; + private Monitor monitor; private HealthCheck healthCheck; private Audit audit; @@ -59,12 +57,11 @@ public class Configuration { private Analysis analysis; private Panel panel; - private Optimizations optimizations; - private ServerConfiguration server; - private Authentication authentication; - private static Logger logger; + private static final Set reportedFields = new HashSet<>(); + + private static final Logger logger; private static final String DEFAULT_CONFIGURATION_FORMAT = "yaml"; @@ -73,7 +70,6 @@ public class Configuration { } public Configuration() { - admin = new Admin(); monitor = new Monitor(); healthCheck = new HealthCheck(); audit = new Audit(); @@ -82,9 +78,7 @@ public Configuration() { catalog = new Catalog(); analysis = new Analysis(); panel = new Panel(); - optimizations = new Optimizations(); server = new ServerConfiguration(); - authentication = new Authentication(); } public void serialize(OutputStream configurationOututStream) throws IOException { @@ -120,11 +114,19 @@ public static Configuration load(InputStream configurationInputStream, String fo throw new IOException("Configuration file could not be parsed: " + e.getMessage(), e); } + addDefaultValueIfMissing(configuration); + // We must always overwrite configuration with environment parameters overwriteWithEnvironmentVariables(configuration); return configuration; } + private static void addDefaultValueIfMissing(Configuration configuration) { + if (configuration.getMaxLoginAttempts() <= 0) { + configuration.setMaxLoginAttempts(5); + } + } + private static void overwriteWithEnvironmentVariables(Configuration configuration) { Map envVariables = System.getenv(); for (String variable : envVariables.keySet()) { @@ -144,6 +146,9 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio case "OPENCGA_MONITOR_PORT": configuration.getMonitor().setPort(Integer.parseInt(value)); break; + case "OPENCGA.MAX_LOGIN_ATTEMPTS": + configuration.setMaxLoginAttempts(Integer.parseInt(value)); + break; case "OPENCGA_EXECUTION_MODE": case "OPENCGA_EXECUTION_ID": configuration.getAnalysis().getExecution().setId(value); @@ -175,18 +180,6 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio case "OPENCGA_CATALOG_DB_CONNECTIONS_PER_HOST": configuration.getCatalog().getDatabase().getOptions().put("connectionsPerHost", value); break; - case "OPENCGA_CATALOG_SEARCH_HOST": - configuration.getCatalog().getSearchEngine().setHosts(Collections.singletonList(value)); - break; - case "OPENCGA_CATALOG_SEARCH_TIMEOUT": - configuration.getCatalog().getSearchEngine().getOptions().put("timeout", value); - break; - case "OPENCGA_CATALOG_SEARCH_BATCH": - configuration.getCatalog().getSearchEngine().getOptions().put("insertBatchSize", value); - break; - case "OPENCGA_OPTIMIZATIONS_SIMPLIFY_PERMISSIONS": - configuration.getOptimizations().setSimplifyPermissions(Boolean.parseBoolean(value)); - break; case "OPENCGA_SERVER_REST_PORT": configuration.getServer().getRest().setPort(Integer.parseInt(value)); break; @@ -200,6 +193,16 @@ private static void overwriteWithEnvironmentVariables(Configuration configuratio } } + public static void reportUnusedField(String field, Object value) { + // Report only if the value is not null and not an empty string + if (value != null && !(value instanceof String && ((String) value).isEmpty())) { + if (reportedFields.add(field)) { + // Only log the first time a field is found + logger.warn("Ignored configuration option '{}' with value '{}'. The option was deprecated and removed.", field, value); + } + } + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("Configuration{"); @@ -207,16 +210,17 @@ public String toString() { sb.append(", logDir='").append(logDir).append('\''); sb.append(", databasePrefix='").append(databasePrefix).append('\''); sb.append(", workspace='").append(workspace).append('\''); - sb.append(", admin=").append(admin); + sb.append(", jobDir='").append(jobDir).append('\''); + sb.append(", maxLoginAttempts=").append(maxLoginAttempts); sb.append(", monitor=").append(monitor); + sb.append(", healthCheck=").append(healthCheck); sb.append(", audit=").append(audit); sb.append(", hooks=").append(hooks); sb.append(", email=").append(email); sb.append(", catalog=").append(catalog); + sb.append(", analysis=").append(analysis); sb.append(", panel=").append(panel); - sb.append(", optimizations=").append(optimizations); sb.append(", server=").append(server); - sb.append(", authentication=").append(authentication); sb.append('}'); return sb.toString(); } @@ -246,20 +250,18 @@ public String getLogFile() { @Deprecated public Configuration setLogFile(String logFile) { - if (logFile != null) { - logger.warn("Deprecated option 'configuration.yml#logFile'"); - } + reportUnusedField("configuration.yml#logFile", logFile); return this; } @Deprecated - public boolean isOpenRegister() { - return openRegister; + public Boolean isOpenRegister() { + return null; } @Deprecated public Configuration setOpenRegister(boolean openRegister) { - this.openRegister = openRegister; + reportUnusedField("configuration.yml#openRegister", openRegister); return this; } @@ -290,12 +292,23 @@ public Configuration setJobDir(String jobDir) { return this; } + public int getMaxLoginAttempts() { + return maxLoginAttempts; + } + + public Configuration setMaxLoginAttempts(int maxLoginAttempts) { + this.maxLoginAttempts = maxLoginAttempts; + return this; + } + + @Deprecated public Admin getAdmin() { - return admin; + return null; } + @Deprecated public Configuration setAdmin(Admin admin) { - this.admin = admin; + reportUnusedField("configuration.yml#admin", admin); return this; } @@ -373,12 +386,14 @@ public Configuration setPanel(Panel panel) { return this; } + @Deprecated public Optimizations getOptimizations() { - return optimizations; + return null; } + @Deprecated public Configuration setOptimizations(Optimizations optimizations) { - this.optimizations = optimizations; + reportUnusedField("configuration.yml#optimizations", optimizations); return this; } @@ -391,12 +406,15 @@ public Configuration setServer(ServerConfiguration server) { return this; } + @Deprecated public Authentication getAuthentication() { - return authentication; + return null; } - public void setAuthentication(Authentication authentication) { - this.authentication = authentication; + @Deprecated + public Configuration setAuthentication(Authentication authentication) { + reportUnusedField("configuration.yml#authentication", authentication); + return this; } public HealthCheck getHealthCheck() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java index 2bc200fe8fb..1384d0a7df1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java @@ -18,6 +18,7 @@ import org.opencb.commons.datastore.core.ObjectMap; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,16 @@ */ public class Execution { + public static final String JOBS_REUSE_ENABLED = "jobs.reuse.enabled"; + public static final boolean JOBS_REUSE_ENABLED_DEFAULT = true; + public static final String JOBS_REUSE_TOOLS = "jobs.reuse.tools"; + public static final List JOBS_REUSE_TOOLS_DEFAULT = Arrays.asList( + "variant-index", + "variant-stats-index", + "variant-annotation-index", + "variant-secondary-annotation-index", + "variant-secondary-sample-index" + ); private String id; private String defaultQueue; private String availableQueues; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/RestServerConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/RestServerConfiguration.java index 99ea3ada10c..f87075e9a02 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/RestServerConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/RestServerConfiguration.java @@ -16,52 +16,132 @@ package org.opencb.opencga.core.config; +import java.util.Objects; + /** * Created by imedina on 22/05/16. */ public class RestServerConfiguration extends AbstractServerConfiguration { - private int defaultLimit; - private int maxLimit; - - + public HttpConfiguration httpConfiguration = new HttpConfiguration(); public RestServerConfiguration() { } public RestServerConfiguration(int port) { - this(port, 2000, 5000); - } - - public RestServerConfiguration(int port, int defaultLimit, int maxLimit) { super(port); - this.defaultLimit = defaultLimit; - this.maxLimit = maxLimit; } @Override public String toString() { final StringBuilder sb = new StringBuilder("RestServerConfiguration{"); - sb.append("defaultLimit=").append(defaultLimit); - sb.append(", maxLimit=").append(maxLimit); + sb.append("port=").append(port); + sb.append(", httpConfiguration=").append(httpConfiguration); sb.append('}'); return sb.toString(); } - public int getDefaultLimit() { - return defaultLimit; + @Deprecated + protected void setDefaultLimit(Object o) { + Configuration.reportUnusedField("configuration.yml#server.rest.defaultLimit", o); } - public RestServerConfiguration setDefaultLimit(int defaultLimit) { - this.defaultLimit = defaultLimit; - return this; + @Deprecated + protected void setMaxLimit(Object o) { + Configuration.reportUnusedField("configuration.yml#server.rest.maxLimit", o); } - public int getMaxLimit() { - return maxLimit; + public HttpConfiguration getHttpConfiguration() { + return httpConfiguration; } - public RestServerConfiguration setMaxLimit(int maxLimit) { - this.maxLimit = maxLimit; + public RestServerConfiguration setHttpConfiguration(HttpConfiguration httpConfiguration) { + this.httpConfiguration = httpConfiguration; return this; } + + public static class HttpConfiguration { + private int outputBufferSize = -1; + private int outputAggregationSize = -1; + private int requestHeaderSize = -1; + private int responseHeaderSize = -1; + private int headerCacheSize = -1; + + public int getOutputBufferSize() { + return outputBufferSize; + } + + public HttpConfiguration setOutputBufferSize(int outputBufferSize) { + this.outputBufferSize = outputBufferSize; + return this; + } + + public int getOutputAggregationSize() { + return outputAggregationSize; + } + + public HttpConfiguration setOutputAggregationSize(int outputAggregationSize) { + this.outputAggregationSize = outputAggregationSize; + return this; + } + + public int getRequestHeaderSize() { + return requestHeaderSize; + } + + public HttpConfiguration setRequestHeaderSize(int requestHeaderSize) { + this.requestHeaderSize = requestHeaderSize; + return this; + } + + public int getResponseHeaderSize() { + return responseHeaderSize; + } + + public HttpConfiguration setResponseHeaderSize(int responseHeaderSize) { + this.responseHeaderSize = responseHeaderSize; + return this; + } + + public int getHeaderCacheSize() { + return headerCacheSize; + } + + public HttpConfiguration setHeaderCacheSize(int headerCacheSize) { + this.headerCacheSize = headerCacheSize; + return this; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HttpConfiguration{"); + sb.append("outputBufferSize=").append(outputBufferSize); + sb.append(", outputAggregationSize=").append(outputAggregationSize); + sb.append(", requestHeaderSize=").append(requestHeaderSize); + sb.append(", responseHeaderSize=").append(responseHeaderSize); + sb.append(", headerCacheSize=").append(headerCacheSize); + sb.append('}'); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + HttpConfiguration that = (HttpConfiguration) o; + return outputBufferSize == that.outputBufferSize && + outputAggregationSize == that.outputAggregationSize && + requestHeaderSize == that.requestHeaderSize && + responseHeaderSize == that.responseHeaderSize && + headerCacheSize == that.headerCacheSize; + } + + @Override + public int hashCode() { + return Objects.hash(outputBufferSize, outputAggregationSize, requestHeaderSize, responseHeaderSize, headerCacheSize); + } + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java index 2ef6a77cca4..be0b4c366a8 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java @@ -46,7 +46,8 @@ public class CellBaseConfiguration { private String apiKey; public CellBaseConfiguration() { - this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION); + this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE_GRCH38, + ParamConstants.CELLBASE_APIKEY); } public CellBaseConfiguration(String url, String version) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java index 06949efb937..4d8df0f6f0a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/JwtPayload.java @@ -4,6 +4,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.AuthenticationOrigin; import java.util.Base64; import java.util.Date; @@ -12,15 +13,20 @@ public class JwtPayload { private final String userId; private final String organization; + private final AuthenticationOrigin.AuthenticationType authOrigin; private final String issuer; // Issuer of the JWT token. private final Date issuedAt; // Time when the JWT was issued. private final Date expirationTime; // Expiration time of the JWT. private final String token; - public JwtPayload(String userId, String organization, String issuer, Date issuedAt, Date expirationTime, String token) { + public static final String AUTH_ORIGIN = "authOrigin"; + + public JwtPayload(String userId, String organization, AuthenticationOrigin.AuthenticationType authOrigin, String issuer, Date issuedAt, + Date expirationTime, String token) { this.token = token; this.userId = userId; this.organization = organization; + this.authOrigin = authOrigin; this.issuer = issuer; this.issuedAt = issuedAt; this.expirationTime = expirationTime; @@ -56,6 +62,12 @@ public JwtPayload(String token) { this.organization = claimsMap.getString("aud"); this.issuer = claimsMap.getString("iss"); + if (claimsMap.containsKey(AUTH_ORIGIN)) { + this.authOrigin = AuthenticationOrigin.AuthenticationType.valueOf(claimsMap.getString(AUTH_ORIGIN)); + } else { + this.authOrigin = null; + } + if (claimsMap.containsKey("iat")) { long iat = 1000L * claimsMap.getLong("iat"); this.issuedAt = new Date(iat); @@ -77,6 +89,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("JwtPayload{"); sb.append("userId='").append(userId).append('\''); sb.append(", organization='").append(organization).append('\''); + sb.append(", authOrigin=").append(authOrigin); sb.append(", issuer='").append(issuer).append('\''); sb.append(", issuedAt=").append(issuedAt); sb.append(", expirationTime=").append(expirationTime); @@ -101,6 +114,10 @@ public String getOrganization() { return organization; } + public AuthenticationOrigin.AuthenticationType getAuthOrigin() { + return authOrigin; + } + public String getIssuer() { return issuer; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java index b4e4dc87607..43bc73f7931 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentIndexParams.java @@ -1,36 +1,41 @@ package org.opencb.opencga.core.models.alignment; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; public class AlignmentIndexParams extends ToolParams { public static final String DESCRIPTION = "Alignment index params"; - private String file; + @DataField(id = "fileId", description = FieldConstants.ALIGNMENT_INDEX_FILE_ID_DESCRIPTION, required = true) + private String fileId; + + @DataField(id = "overwrite", description = FieldConstants.ALIGNMENT_INDEX_OVERWRITE_DESCRIPTION) private boolean overwrite; public AlignmentIndexParams() { } - public AlignmentIndexParams(String file, boolean overwrite) { - this.file = file; + public AlignmentIndexParams(String fileId, boolean overwrite) { + this.fileId = fileId; this.overwrite = overwrite; } @Override public String toString() { final StringBuilder sb = new StringBuilder("AlignmentIndexParams{"); - sb.append("file='").append(file).append('\''); + sb.append("fileId='").append(fileId).append('\''); sb.append(", overwrite=").append(overwrite); sb.append('}'); return sb.toString(); } - public String getFile() { - return file; + public String getFileId() { + return fileId; } - public AlignmentIndexParams setFile(String file) { - this.file = file; + public AlignmentIndexParams setFileId(String fileId) { + this.fileId = fileId; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java index f655829625d..9ce861660f2 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/CoverageIndexParams.java @@ -2,15 +2,22 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.tools.ToolParams; public class CoverageIndexParams extends ToolParams { public static final String DESCRIPTION = "Coverage computation parameters"; + @DataField(id = "bamFileId", description = FieldConstants.COVERAGE_INDEX_BAM_FILE_ID_DESCRIPTION, required = true) private String bamFileId; + + @DataField(id = "baiFileId", description = FieldConstants.COVERAGE_INDEX_BAI_FILE_ID_DESCRIPTION) private String baiFileId; - @JsonProperty(defaultValue = "1") + @DataField(id = "windowSize", description = FieldConstants.COVERAGE_INDEX_OVERWRITE_DESCRIPTION, + defaultValue = ParamConstants.COVERAGE_WINDOW_SIZE_DEFAULT) private int windowSize; public CoverageIndexParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java index 312e1f32e87..f8ba4b924bc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/audit/AuditRecord.java @@ -275,6 +275,11 @@ public Status(Result name, Error error) { this.error = error; } + public Status(Result name, Throwable exception) { + this.name = name; + this.error = new Error(0, exception.getClass().getSimpleName(), exception.getMessage()); + } + public enum Result { SUCCESS, ERROR diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java index 1fa40dfbaf7..3b836063439 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ExomiserWrapperParams.java @@ -1,5 +1,7 @@ package org.opencb.opencga.core.models.clinical; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; import java.util.Map; @@ -7,14 +9,22 @@ public class ExomiserWrapperParams extends ToolParams { public static final String DESCRIPTION = "Exomiser parameters"; + @DataField(id = "sample", description = FieldConstants.SAMPLE_ID_DESCRIPTION) private String sample; + + @DataField(id = "clinicalAnalysisType", description = FieldConstants.EXOMISER_CLINICAL_ANALYSIS_TYPE_DESCRIPTION, + defaultValue = "SINGLE") + private String clinicalAnalysisType; + + @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) private String outdir; public ExomiserWrapperParams() { } - public ExomiserWrapperParams(String sample, String outdir) { + public ExomiserWrapperParams(String sample, String clinicalAnalysisType, String outdir) { this.sample = sample; + this.clinicalAnalysisType = clinicalAnalysisType; this.outdir = outdir; } @@ -22,6 +32,7 @@ public ExomiserWrapperParams(String sample, String outdir) { public String toString() { final StringBuilder sb = new StringBuilder("ExomiserWrapperParams{"); sb.append("sample='").append(sample).append('\''); + sb.append(", clinicalAnalysisType=").append(clinicalAnalysisType); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -36,6 +47,15 @@ public ExomiserWrapperParams setSample(String sample) { return this; } + public String getClinicalAnalysisType() { + return clinicalAnalysisType; + } + + public ExomiserWrapperParams setClinicalAnalysisType(String clinicalAnalysisType) { + this.clinicalAnalysisType = clinicalAnalysisType; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java index 10b270b92c0..2777222715b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java @@ -250,6 +250,7 @@ public enum Action { MOVE_AND_REGISTER, VISIT, + KILL_JOB, IMPORT, diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java index 5a1518440ab..83836d0f7d4 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/IndexStatus.java @@ -13,8 +13,9 @@ public class IndexStatus extends InternalStatus { */ public static final String NONE = "NONE"; public static final String INDEXING = "INDEXING"; + public static final String INVALID = "INVALID"; - public static final List STATUS_LIST = Arrays.asList(READY, DELETED, NONE, INDEXING); + public static final List STATUS_LIST = Arrays.asList(READY, DELETED, NONE, INDEXING, INVALID); public IndexStatus(String status, String message) { if (isValid(status)) { @@ -50,6 +51,7 @@ public static boolean isValid(String status) { return status != null && (status.equals(READY) || status.equals(DELETED) + || status.equals(INVALID) || status.equals(NONE) || status.equals(INDEXING)); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java index 3dbe5f21f7d..a6592c369bf 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/VariantIndexStatus.java @@ -19,7 +19,8 @@ public class VariantIndexStatus extends IndexStatus { public static final String TRANSFORMED = "TRANSFORMED"; public static final String LOADING = "LOADING"; - public static final List STATUS_LIST = Arrays.asList(READY, DELETED, NONE, TRANSFORMED, TRANSFORMING, LOADING, INDEXING); + public static final List STATUS_LIST = Arrays.asList(READY, DELETED, NONE, TRANSFORMED, TRANSFORMING, LOADING, INDEXING, + INVALID); public VariantIndexStatus(String status, String message) { if (isValid(status)) { @@ -56,6 +57,7 @@ public static boolean isValid(String status) { && (status.equals(READY) || status.equals(DELETED) || status.equals(NONE) + || status.equals(INVALID) || status.equals(TRANSFORMED) || status.equals(TRANSFORMING) || status.equals(LOADING) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java index be3952b1e28..802f5967f2f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/Job.java @@ -19,6 +19,7 @@ import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.PrivateStudyUid; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; @@ -73,7 +74,6 @@ public class Job extends PrivateStudyUid { description = FieldConstants.JOB_COMMAND_LINE) private String commandLine; - @DataField(id = "params", indexed = true, description = FieldConstants.JOB_PARAMS) private Map params; @@ -129,6 +129,17 @@ public class Job extends PrivateStudyUid { description = FieldConstants.JOB_DEPENDS_ON_DESCRIPTION) private List dependsOn; + @DataField(id = FieldConstants.JOB_PARENT_ID, indexed = false, since = "3.2.0", + description = FieldConstants.JOB_PARENT_ID_DESCRIPTION) + private String parentId; + + @DataField(id = FieldConstants.JOB_SCHEDULED_START_TIME, indexed = false, since = "3.2.0", + description = FieldConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) + private String scheduledStartTime; + + @DataField(id = ParamConstants.JOB_DRY_RUN, since = "3.2.0", description = ParamConstants.JOB_DRY_RUN_DESCRIPTION) + private boolean dryRun; + @DataField(id = "execution", indexed = true, description = FieldConstants.JOB_EXECUTION_DESCRIPTION) private ExecutionResult execution; @@ -168,8 +179,9 @@ public Job() { public Job(String id, String uuid, String description, ToolInfo tool, String userId, String commandLine, Map params, String creationDate, String modificationDate, Enums.Priority priority, JobInternal internal, File outDir, - List input, List output, List dependsOn, List tags, ExecutionResult execution, boolean visited, - File stdout, File stderr, int release, JobStudyParam study, Map attributes) { + List input, List output, List dependsOn, String parentId, String scheduledStartTime, boolean dryRun, + List tags, ExecutionResult execution, boolean visited, File stdout, File stderr, int release, JobStudyParam study, + Map attributes) { this.id = id; this.uuid = uuid; this.tool = tool; @@ -185,6 +197,9 @@ public Job(String id, String uuid, String description, ToolInfo tool, String use this.input = input; this.output = output; this.dependsOn = dependsOn; + this.parentId = parentId; + this.scheduledStartTime = scheduledStartTime; + this.dryRun = dryRun; this.tags = tags; this.execution = execution; this.visited = visited; @@ -214,6 +229,9 @@ public String toString() { sb.append(", output=").append(output); sb.append(", tags=").append(tags); sb.append(", dependsOn=").append(dependsOn); + sb.append(", parentId='").append(parentId).append('\''); + sb.append(", scheduledStartTime='").append(scheduledStartTime).append('\''); + sb.append(", dryRun=").append(dryRun); sb.append(", execution=").append(execution); sb.append(", stdout=").append(stdout); sb.append(", stderr=").append(stderr); @@ -375,6 +393,33 @@ public Job setDependsOn(List dependsOn) { return this; } + public String getParentId() { + return parentId; + } + + public Job setParentId(String parentId) { + this.parentId = parentId; + return this; + } + + public String getScheduledStartTime() { + return scheduledStartTime; + } + + public Job setScheduledStartTime(String scheduledStartTime) { + this.scheduledStartTime = scheduledStartTime; + return this; + } + + public boolean isDryRun() { + return dryRun; + } + + public Job setDryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + public List getTags() { return tags; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java index 278acfed94c..68382ca25c9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobCreateParams.java @@ -271,9 +271,11 @@ public JobCreateParams setAttributes(Map attributes) { public Job toJob() { return new Job(id, null, description, tool, null, commandLine, params, creationDate, null, priority, internal != null ? new org.opencb.opencga.core.models.job.JobInternal(internal.getStatus()) : null, - outDir != null ? outDir.toFile() : null, getInput().stream().map(TinyFile::toFile).collect(Collectors.toList()), - getOutput().stream().map(TinyFile::toFile).collect(Collectors.toList()), Collections.emptyList(), - tags, result, false, stdout != null ? stdout.toFile() : null, stderr != null ? stderr.toFile() : null, 1, null, attributes); + outDir != null ? outDir.toFile() : null, + getInput() != null ? getInput().stream().map(TinyFile::toFile).collect(Collectors.toList()) : Collections.emptyList(), + getOutput() != null ? getOutput().stream().map(TinyFile::toFile).collect(Collectors.toList()) : Collections.emptyList(), + Collections.emptyList(), null, null, false, tags, result, false, stdout != null ? stdout.toFile() : null, + stderr != null ? stderr.toFile() : null, 1, null, attributes); } public static class JobInternal { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java index 6bb8d08cdea..123dc8bf70e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/JobInternal.java @@ -42,23 +42,34 @@ public class JobInternal extends Internal implements Cloneable { description = FieldConstants.JOB_INTERNAL_EVENTS_DESCRIPTION) private List events; + @DataField(id = "killJobRequested", indexed = false, since = "3.2.0", + description = FieldConstants.JOB_INTERNAL_KILL_JOB_REQUESTED_DESCRIPTION) + private boolean killJobRequested; + public JobInternal() { } public JobInternal(Enums.ExecutionStatus status) { - this(null, null, status, null, null); + this(null, null, status, null, null, false); } + @Deprecated public JobInternal(String registrationDate, String modificationDate, Enums.ExecutionStatus status, JobInternalWebhook webhook, List events) { + this(registrationDate, modificationDate, status, webhook, events, false); + } + + public JobInternal(String registrationDate, String modificationDate, Enums.ExecutionStatus status, JobInternalWebhook webhook, + List events, boolean killJobRequested) { super(null, registrationDate, modificationDate); this.status = status; this.webhook = webhook; this.events = events; + this.killJobRequested = killJobRequested; } public static JobInternal init() { - return new JobInternal(TimeUtils.getTime(), TimeUtils.getTime(), new Enums.ExecutionStatus(), null, new ArrayList<>()); + return new JobInternal(TimeUtils.getTime(), TimeUtils.getTime(), new Enums.ExecutionStatus(), null, new ArrayList<>(), false); } @Override @@ -69,13 +80,14 @@ public String toString() { sb.append(", status=").append(status); sb.append(", webhook=").append(webhook); sb.append(", events=").append(events); + sb.append(", killJobRequested=").append(killJobRequested); sb.append('}'); return sb.toString(); } @Override public JobInternal clone() throws CloneNotSupportedException { - return new JobInternal(registrationDate, lastModified, status, webhook.clone(), new LinkedList<>(events)); + return new JobInternal(registrationDate, lastModified, status, webhook.clone(), new LinkedList<>(events), killJobRequested); } public Enums.ExecutionStatus getStatus() { @@ -122,4 +134,13 @@ public JobInternal setLastModified(String lastModified) { this.lastModified = lastModified; return this; } + + public boolean isKillJobRequested() { + return killJobRequested; + } + + public JobInternal setKillJobRequested(boolean killJobRequested) { + this.killJobRequested = killJobRequested; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateFamilyParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateFamilyParams.java index 42b9073b45d..5f73c45e964 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateFamilyParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateFamilyParams.java @@ -16,14 +16,20 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.tools.ToolParams; import java.util.List; public class VariantAggregateFamilyParams extends ToolParams { public static final String DESCRIPTION = "Variant aggregate family params."; + + @DataField(description = "Samples within the same study to aggregate") private List samples; + @DataField(description = "Genotype to be used in gaps. Either 0/0, ./. or ?/?") private String gapsGenotype; + @DataField(description = ParamConstants.RESUME_DESCRIPTION) private boolean resume; public VariantAggregateFamilyParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateParams.java index 87880318218..434643acc33 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAggregateParams.java @@ -16,13 +16,17 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.tools.ToolParams; public class VariantAggregateParams extends ToolParams { public static final String DESCRIPTION = "Variant aggregate params."; - // private String region; + // private String region + @DataField(description = "Overwrite aggregation for all files and variants. Repeat operation for already processed variants.") private boolean overwrite; + @DataField(description = ParamConstants.RESUME_DESCRIPTION) private boolean resume; public VariantAggregateParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationDeleteParams.java index bb1072e5f98..797ee2cbdf5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationDeleteParams.java @@ -16,10 +16,12 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; public class VariantAnnotationDeleteParams extends ToolParams { + @DataField(description = "Variant Annotation identifier") private String annotationId; public VariantAnnotationDeleteParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationSaveParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationSaveParams.java index 373261ea395..948f3a453c0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationSaveParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantAnnotationSaveParams.java @@ -16,11 +16,13 @@ package org.opencb.opencga.core.models.operations.variant; +import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.tools.ToolParams; public class VariantAnnotationSaveParams extends ToolParams { public static final String DESCRIPTION = "Variant annotation save params"; + @DataField(description = "New Variant Annotation identifier") private String annotationId; public VariantAnnotationSaveParams() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantConfigureParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantConfigureParams.java similarity index 85% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantConfigureParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantConfigureParams.java index 3365908a43f..51a22f36337 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantConfigureParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantConfigureParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.commons.datastore.core.ObjectMap; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFamilyIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFamilyIndexParams.java index 0a2c8d3b8b5..23f89c9bbd0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFamilyIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFamilyIndexParams.java @@ -20,6 +20,7 @@ import java.util.List; +@Deprecated public class VariantFamilyIndexParams extends ToolParams { public static final String DESCRIPTION = "Variant family index params."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantFileDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java similarity index 84% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantFileDeleteParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java index 82a1daa3670..85ba64ce78c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantFileDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileDeleteParams.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.opencga.core.tools.ToolParams; @@ -34,6 +34,7 @@ public VariantFileDeleteParams(List file, boolean resume) { private List file; private boolean resume; + private boolean force; public List getFile() { return file; @@ -52,4 +53,13 @@ public VariantFileDeleteParams setResume(boolean resume) { this.resume = resume; return this; } + + public boolean isForce() { + return force; + } + + public VariantFileDeleteParams setForce(boolean force) { + this.force = force; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantFileIndexJobLauncherParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileIndexJobLauncherParams.java similarity index 97% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantFileIndexJobLauncherParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileIndexJobLauncherParams.java index 75404921751..28f3dbc8ff1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantFileIndexJobLauncherParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantFileIndexJobLauncherParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.opencga.core.tools.ToolParams; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantIndexParams.java similarity index 98% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantIndexParams.java index 1e0cd7dac49..505327de602 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantIndexParams.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.annotations.DataField; @@ -81,7 +81,7 @@ public VariantIndexParams(String file, @DataField(description = "List of files to be indexed.") private String file; - @DataField(description = "Resume a previously failed index operation") + @DataField(description = ParamConstants.RESUME_DESCRIPTION) private boolean resume; @DataField(description = "Output directory") private String outdir; @@ -101,7 +101,7 @@ public VariantIndexParams(String file, private String failOnMalformedLines; @DataField(description = "Indicate that the files to be loaded are part of a family. " - + "This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards") + + "This will set 'load-hom-ref' to YES if it was in AUTO") private boolean family; @DataField(description = "Indicate that the files to be loaded contain somatic samples. " + "This will set 'load-hom-ref' to YES if it was in AUTO.") diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantPruneParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantPruneParams.java similarity index 93% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantPruneParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantPruneParams.java index ed0449871e6..d59ead10217 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantPruneParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantPruneParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.opencga.core.tools.ToolParams; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSampleDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantSampleDeleteParams.java similarity index 96% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSampleDeleteParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantSampleDeleteParams.java index 4857a35a650..3ae43e6b61f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantSampleDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantSampleDeleteParams.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.opencga.core.tools.ToolParams; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantStorageMetadataSynchronizeParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantStorageMetadataSynchronizeParams.java similarity index 92% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantStorageMetadataSynchronizeParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantStorageMetadataSynchronizeParams.java index f5a2b25f11d..002bff2f937 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantStorageMetadataSynchronizeParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantStorageMetadataSynchronizeParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.opencga.core.tools.ToolParams; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantStudyDeleteParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantStudyDeleteParams.java similarity index 95% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantStudyDeleteParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantStudyDeleteParams.java index 20f533e711a..1922fe78bf0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantStudyDeleteParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/operations/variant/VariantStudyDeleteParams.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.opencb.opencga.core.models.variant; +package org.opencb.opencga.core.models.operations.variant; import org.opencb.opencga.core.tools.ToolParams; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java index c8f3e849c89..cda477b2acc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/organizations/OrganizationConfiguration.java @@ -8,15 +8,17 @@ public class OrganizationConfiguration { private List authenticationOrigins; + private String defaultUserExpirationDate; private Optimizations optimizations; private TokenConfiguration token; public OrganizationConfiguration() { } - public OrganizationConfiguration(List authenticationOrigins, Optimizations optimizations, - TokenConfiguration token) { + public OrganizationConfiguration(List authenticationOrigins, String defaultUserExpirationDate, + Optimizations optimizations, TokenConfiguration token) { this.authenticationOrigins = authenticationOrigins; + this.defaultUserExpirationDate = defaultUserExpirationDate; this.optimizations = optimizations; this.token = token; } @@ -25,6 +27,7 @@ public OrganizationConfiguration(List authenticationOrigin public String toString() { final StringBuilder sb = new StringBuilder("OrganizationConfiguration{"); sb.append("authenticationOrigins=").append(authenticationOrigins); + sb.append(", defaultUserExpirationDate='").append(defaultUserExpirationDate).append('\''); sb.append(", optimizations=").append(optimizations); sb.append(", token=").append(token); sb.append('}'); @@ -40,6 +43,15 @@ public OrganizationConfiguration setAuthenticationOrigins(List attributes; @@ -38,13 +35,12 @@ public OrganizationUpdateParams() { } public OrganizationUpdateParams(String name, String owner, List admins, String creationDate, String modificationDate, - OrganizationConfiguration configuration, Map attributes) { + Map attributes) { this.name = name; this.owner = owner; this.admins = admins; this.creationDate = creationDate; this.modificationDate = modificationDate; - this.configuration = configuration; this.attributes = attributes; } @@ -61,7 +57,6 @@ public String toString() { sb.append(", admins=").append(admins); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); - sb.append(", configuration=").append(configuration); sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); @@ -112,15 +107,6 @@ public OrganizationUpdateParams setModificationDate(String modificationDate) { return this; } - public OrganizationConfiguration getConfiguration() { - return configuration; - } - - public OrganizationUpdateParams setConfiguration(OrganizationConfiguration configuration) { - this.configuration = configuration; - return this; - } - public Map getAttributes() { return attributes; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java new file mode 100644 index 00000000000..ce30fd63c41 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/OrganizationUserUpdateParams.java @@ -0,0 +1,109 @@ +package org.opencb.opencga.core.models.user; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.opencb.commons.datastore.core.ObjectMap; + +import java.util.Map; + +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; + +public class OrganizationUserUpdateParams extends UserUpdateParams { + + private UserQuota quota; + private Account account; + private Map attributes; + + public OrganizationUserUpdateParams() { + } + + public OrganizationUserUpdateParams(String name, String email, UserQuota quota, Account account, Map attributes) { + super(name, email); + this.quota = quota; + this.account = account; + this.attributes = attributes; + } + + @JsonIgnore + public ObjectMap getUpdateMap() throws JsonProcessingException { + return new ObjectMap(getUpdateObjectMapper().writeValueAsString(this)); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OrganizationUserUpdateParams{"); + sb.append("quota=").append(quota); + sb.append(", account=").append(account); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public UserQuota getQuota() { + return quota; + } + + public OrganizationUserUpdateParams setQuota(UserQuota quota) { + this.quota = quota; + return this; + } + + public Account getAccount() { + return account; + } + + public OrganizationUserUpdateParams setAccount(Account account) { + this.account = account; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public OrganizationUserUpdateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + @Override + public OrganizationUserUpdateParams setName(String name) { + super.setName(name); + return this; + } + + @Override + public OrganizationUserUpdateParams setEmail(String email) { + super.setEmail(email); + return this; + } + + public static class Account { + private String expirationDate; + + public Account() { + } + + public Account(String expirationDate) { + this.expirationDate = expirationDate; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Account{"); + sb.append("expirationDate='").append(expirationDate).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getExpirationDate() { + return expirationDate; + } + + public Account setExpirationDate(String expirationDate) { + this.expirationDate = expirationDate; + return this; + } + } + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java index 9a4b5fc9e23..329db613b5b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserInternal.java @@ -16,21 +16,34 @@ package org.opencb.opencga.core.models.user; -public class UserInternal { +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.common.Internal; + +public class UserInternal extends Internal { private UserStatus status; + private int failedAttempts; public UserInternal() { } public UserInternal(UserStatus status) { + this(TimeUtils.getTime(), TimeUtils.getTime(), status); + } + + public UserInternal(String registrationDate, String lastModified, UserStatus status) { + super(null, registrationDate, lastModified); this.status = status; + this.failedAttempts = 0; } @Override public String toString() { final StringBuilder sb = new StringBuilder("UserInternal{"); sb.append("status=").append(status); + sb.append(", failedAttempts=").append(failedAttempts); + sb.append(", registrationDate='").append(registrationDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); return sb.toString(); } @@ -43,4 +56,13 @@ public UserInternal setStatus(UserStatus status) { this.status = status; return this; } + + public int getFailedAttempts() { + return failedAttempts; + } + + public UserInternal setFailedAttempts(int failedAttempts) { + this.failedAttempts = failedAttempts; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java index bf497bd00f0..b9220d8fdce 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatus.java @@ -24,8 +24,9 @@ public class UserStatus extends InternalStatus { public static final String BANNED = "BANNED"; + public static final String SUSPENDED = "SUSPENDED"; - public static final List STATUS_LIST = Arrays.asList(READY, DELETED, BANNED); + public static final List STATUS_LIST = Arrays.asList(READY, DELETED, BANNED, SUSPENDED); public UserStatus(String status, String message) { if (isValid(status)) { @@ -47,7 +48,7 @@ public static boolean isValid(String status) { if (InternalStatus.isValid(status)) { return true; } - if (status != null && (status.equals(BANNED))) { + if (status != null && (status.equals(BANNED) || status.equals(SUSPENDED))) { return true; } return false; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatusUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatusUpdateParams.java new file mode 100644 index 00000000000..5c0356ded8e --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserStatusUpdateParams.java @@ -0,0 +1,30 @@ +package org.opencb.opencga.core.models.user; + +public class UserStatusUpdateParams { + + private String status; + + public UserStatusUpdateParams() { + } + + public UserStatusUpdateParams(String status) { + this.status = status; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("UserStatusUpdateParams{"); + sb.append("status='").append(status).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getStatus() { + return status; + } + + public UserStatusUpdateParams setStatus(String status) { + this.status = status; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java index d9c44a665c8..eda30607fbb 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/user/UserUpdateParams.java @@ -16,21 +16,17 @@ package org.opencb.opencga.core.models.user; -import java.util.Map; - public class UserUpdateParams { private String name; private String email; - private Map attributes; public UserUpdateParams() { } - public UserUpdateParams(String name, String email, Map attributes) { + public UserUpdateParams(String name, String email) { this.name = name; this.email = email; - this.attributes = attributes; } @Override @@ -38,7 +34,6 @@ public String toString() { final StringBuilder sb = new StringBuilder("UserUpdateParams{"); sb.append("name='").append(name).append('\''); sb.append(", email='").append(email).append('\''); - sb.append(", attributes=").append(attributes); sb.append('}'); return sb.toString(); } @@ -61,12 +56,4 @@ public UserUpdateParams setEmail(String email) { return this; } - public Map getAttributes() { - return attributes; - } - - public UserUpdateParams setAttributes(Map attributes) { - this.attributes = attributes; - return this; - } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java index ddbfa70636a..3bf38d51053 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/annotations/Tool.java @@ -52,6 +52,11 @@ */ String description() default ""; + /** + * @return Tool priority. + */ + Enums.Priority priority() default Enums.Priority.MEDIUM; + enum Type { OPERATION, ANALYSIS; diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index ac4acf283d8..e3661d0167a 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -6,6 +6,9 @@ databasePrefix: ${OPENCGA.DB.PREFIX} workspace: ${OPENCGA.USER.WORKSPACE} jobDir: ${OPENCGA.USER.WORKSPACE}/jobs +# Maximum number of login attempts before banning a user account +maxLoginAttempts: ${OPENCGA.MAX_LOGIN_ATTEMPTS} + panel: host: "http://resources.opencb.org/opencb/opencga/disease-panels" @@ -19,69 +22,24 @@ catalog: options: authenticationDatabase: ${OPENCGA.CATALOG.DB.AUTHENTICATION_DATABASE} connectionsPerHost: ${OPENCGA.CATALOG.DB.CONNECTIONS_PER_HOST} - ## Solr Search engine configuration, by default is the same than storage - searchEngine: - # List of hosts pointing either to the Solr nodes directly using a complete URL or to the zookeper nodes with HOST:PORT - # Example for Solr connection: http://opencga-solr-01.zone:8983/solr - # Example for Zookeeper connection: opencga-zookeeper-01:2181 <-- Recommended for replicated installations - hosts: - - ${OPENCGA.CATALOG.SEARCH.HOST} - user: "" - password: "" - options: - mode: "cloud" - timeout: ${OPENCGA.CATALOG.SEARCH.TIMEOUT} - insertBatchSize: ${OPENCGA.CATALOG.SEARCH.BATCH} - -## We support multiple Authentication providers, if none is provided then we use an internal authentication implementation -authentication: - # Session expiration time in seconds - expiration: 3600 - authenticationOrigins: -# Custom LDAP | LDAPS configuration example -# - id: ldaps # Any id -# type: LDAP -# host: ldaps://localhost:636 # or ldap://localhost:123 -# options: -# authUserId: "" # Auth user id and password if querying the LDAP system requires authentication. By default, empty. -# authPassword: "" # Auth user id and password if querying the LDAP system requires authentication. By default, empty. -# usersSearch: dc=ge,dc=co,dc=uk # Mandatory field. Base search to look for users. By default, empty. -# groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Mandatory field. Base search to look for groups. By default, empty. -# fullNameKey: displayname # Key to get the user's full name when importing users. By default, 'displayname'. -# memberKey: member # Key within groups to extract the user id. By default, 'member'. -# dnKey: dn # Key to get the user's DN or RDN unique identifier. By default, 'dn'. -# dnFormat: "%s" # Formatter to extract the user's DN if it is contained within a larger string. By default, '%s' to take the whole string. -# uidKey: uid # Key to get the user id to be used in OpenCGA. By default, 'uid'. -# uidFormat: "%s" # Formatter to extract the user id if the uid is contained within a larger string. By default, '%s' to take the whole string. -# sslInvalidCertificatesAllowed: false # If using LDAPs with a custom certificate, set to true to accept any certificate. By default, false. -# connectionTimeout: 500 # Connection timeout value. Default: 500 ms -# readTimeout: 1000 # Read timeout value. Default: 1000 ms - -# Azure AD configuration example -# - id: aad # Any id -# type: AzureAD -# host: -# options: -# tenantId: xxxx # Mandatory. Tenant id -# authClientId: xxxx # Mandatory. Client id of the client with permissions to authenticate users. -# syncClientId: xxxx # Mandatory. Client id of the client with permissions to inspect active directory. -# syncSecretKey: xxxx # Mandatory: Secret key of the client with permissions to inspect active directory. -# filters: tokenField1=aa,bb,cc;tokenField2=aa,bb,cc # Optional. Filters to be applied. OpenCGA will check if tokenField1 = aa or bb -# # or cc and tokenField2 = aa or bb or cc. If any of the filters don't succeed, even if the user is properly authenticated -# # in AAD, the user will not be able to generate a token and login in OpenCGA. server: rest: port: ${OPENCGA.SERVER.REST.PORT} - logFile: null - defaultLimit: 2000 - maxLimit: 5000 + httpConfiguration: + # The size in bytes of the output buffer used to aggregate HTTP output + outputBufferSize: 32768 + # The maximum size in bytes for HTTP output to be aggregated + outputAggregationSize: 8192 + # The maximum allowed size in bytes for a HTTP request header + requestHeaderSize: 8192 + # The maximum allowed size in bytes for a HTTP response header + responseHeaderSize: 8192 + # The maximum allowed size in bytes for a HTTP header field cache + headerCacheSize: 4096 + grpc: port: ${OPENCGA.SERVER.GRPC.PORT} - logFile: null - -optimizations: - simplifyPermissions: ${OPENCGA_OPTIMIZATIONS_SIMPLIFY_PERMISSIONS} audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. @@ -140,7 +98,8 @@ analysis: maxConcurrentJobs: variant-index: 20 variant-annotation-index: 5 - variant-secondary-index: 2 + variant-secondary-annotation-index: 2 + variant-secondary-sample-index: 2 options: ## Job reuse policy. Do not create a new job if an equivalent PENDING or QUEUED job exists. jobs.reuse.enabled: true @@ -150,6 +109,7 @@ analysis: - "variant-stats-index" - "variant-annotation-index" - "variant-secondary-annotation-index" + - "variant-secondary-sample-index" ## Local executor configuration local.maxConcurrentJobs: 2 # Max number of concurrent jobs to be executed locally in the master ## Azure Batch Service configuration example @@ -163,6 +123,8 @@ analysis: ## Kubernetes executor configuration example # k8s.masterUrl: "https://192.168.99.100:8443/" k8s.clientTimeout: 30000 # ms + k8s.terminationGracePeriodSeconds: 300 # s + k8s.logToStdout: true k8s.imageName: "opencb/opencga-base:${project.parent.version}-hdp3.1" k8s.imagePullPolicy: "IfNotPresent" # k8s.imagePullSecrets: diff --git a/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java b/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java index 948bb037dd8..618797dc5c0 100644 --- a/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java +++ b/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java @@ -20,8 +20,11 @@ import org.junit.experimental.categories.Category; import org.opencb.opencga.core.testclassification.duration.ShortTests; -import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; import java.util.*; /** @@ -31,18 +34,12 @@ public class ConfigurationTest { @Test - public void testDefault() { + public void testDefault() throws IOException { Configuration configuration = new Configuration(); configuration.setLogLevel("INFO"); configuration.setWorkspace("/opt/opencga/sessions"); - - configuration.setAdmin(new Admin()); - - Authentication authentication = new Authentication(); - configuration.setAuthentication(authentication); - configuration.setMonitor(new Monitor()); configuration.getAnalysis().setExecution(new Execution()); @@ -51,14 +48,6 @@ public void testDefault() { new HookConfiguration("name", "~*SV*", HookConfiguration.Stage.CREATE, HookConfiguration.Action.ADD, "tags", "SV") )))); - List authenticationOriginList = new ArrayList<>(); - authenticationOriginList.add(new AuthenticationOrigin()); - Map myMap = new HashMap<>(); - myMap.put("ou", "People"); - authenticationOriginList.add(new AuthenticationOrigin("opencga", AuthenticationOrigin.AuthenticationType.LDAP, - "ldap://10.10.0.20:389", myMap)); - configuration.getAuthentication().setAuthenticationOrigins(authenticationOriginList); - Email emailServer = new Email("localhost", "", "", "", "", false); configuration.setEmail(emailServer); @@ -71,7 +60,7 @@ public void testDefault() { configuration.setAudit(audit); ServerConfiguration serverConfiguration = new ServerConfiguration(); - RestServerConfiguration rest = new RestServerConfiguration(1000, 100, 1000); + RestServerConfiguration rest = new RestServerConfiguration(1000); GrpcServerConfiguration grpc = new GrpcServerConfiguration(1001); serverConfiguration.setGrpc(grpc); serverConfiguration.setRest(rest); @@ -90,11 +79,10 @@ public void testDefault() { // catalogConfiguration.getStorageEngines().add(storageEngineConfiguration1); // catalogConfiguration.getStorageEngines().add(storageEngineConfiguration2); - try { - configuration.serialize(new FileOutputStream("/tmp/configuration-test.yml")); - } catch (IOException e) { - e.printStackTrace(); - } + Path outdir = Paths.get("target/test-data", "junit-opencga-" + + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss.SSS").format(new Date())); + Files.createDirectories(outdir); + configuration.serialize(Files.newOutputStream(outdir.resolve("configuration-test.yml").toFile().toPath())); } @Test diff --git a/opencga-core/src/test/resources/configuration-test.yml b/opencga-core/src/test/resources/configuration-test.yml index 4d8d3c10f25..2c3736938b7 100644 --- a/opencga-core/src/test/resources/configuration-test.yml +++ b/opencga-core/src/test/resources/configuration-test.yml @@ -4,6 +4,7 @@ logDir: null databasePrefix: "opencga_test" workspace: "/tmp/opencga/sessions" +maxLoginAttempts: 5 audit: manager: "" # Java manager of the audit implementation to be used to audit. If empty, catalog database will be used. @@ -32,21 +33,11 @@ catalog: password: "" options: authenticationDatabase: "" -authentication: - expiration: 1000 -#LDAP configuration example - authenticationOrigins: - - id: ldap # Any id - type: LDAP # At the moment, we only support LDAP - host: ldap://localhost:9000 - options: - usersSearch: dc=ge,dc=co,dc=uk # Base search to look for the users - groupsSearch: ou=general,ou=groups,dc=ge,dc=co,dc=uk # Base search to look for the groups server: rest: port: 8080 - logFile: null + logFile: "some_file_but_this_field_is_deprecated" defaultLimit: 2000 maxLimit: 5000 diff --git a/opencga-core/src/test/resources/log4j2-test.xml b/opencga-core/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000000..dce741b72cb --- /dev/null +++ b/opencga-core/src/test/resources/log4j2-test.xml @@ -0,0 +1,20 @@ + + + + ${sys:opencga.log.level:-info} + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 2348ef6f6f7..73192c29e58 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml @@ -62,6 +62,11 @@ org.opencb.opencga opencga-catalog + + org.opencb.opencga + opencga-storage-core + test + org.eclipse.jetty @@ -71,6 +76,11 @@ org.eclipse.jetty jetty-servlet + + org.mortbay.jetty + servlet-api + test + com.microsoft.azure @@ -133,6 +143,18 @@ io.fabric8 kubernetes-model + + io.fabric8 + kubernetes-model-core + + + io.fabric8 + kubernetes-client-api + + + io.fabric8 + kubernetes-model-batch + commons-codec commons-codec diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java index f97c0b02957..c4c2ab43ae5 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/AuthorizationDaemon.java @@ -21,7 +21,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.managers.CatalogManager; -public class AuthorizationDaemon extends MonitorParentDaemon { +public abstract class AuthorizationDaemon extends MonitorParentDaemon { private final String INTERNAL_DELIMITER = "__"; diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index cadc1ef4ca7..5956eb68e65 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -83,6 +83,7 @@ import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.managers.JobManager; import org.opencb.opencga.catalog.managers.StudyManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -105,6 +106,7 @@ import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.core.tools.result.Status; import org.opencb.opencga.master.monitor.models.PrivateJobUpdateParams; +import org.opencb.opencga.master.monitor.schedulers.JobScheduler; import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; @@ -112,6 +114,7 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.Closeable; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -134,22 +137,20 @@ /** * Created by imedina on 16/06/16. */ -public class ExecutionDaemon extends MonitorParentDaemon { +public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { public static final String OUTDIR_PARAM = "outdir"; public static final int EXECUTION_RESULT_FILE_EXPIRATION_SECONDS = (int) TimeUnit.MINUTES.toSeconds(10); public static final String REDACTED_TOKEN = "xxxxxxxxxxxxxxxxxxxxx"; private final StorageConfiguration storageConfiguration; - private String internalCli; - private JobManager jobManager; - private FileManager fileManager; + private final String internalCli; + private final JobManager jobManager; + private final FileManager fileManager; private final Map jobsCountByType = new HashMap<>(); private final Map retainedLogsTime = new HashMap<>(); private List packages; - private Path defaultJobDir; - private static final Map TOOL_CLI_MAP; // Maximum number of jobs of each type (Pending, queued, running) that will be handled on each iteration. @@ -157,13 +158,14 @@ public class ExecutionDaemon extends MonitorParentDaemon { // On first iteration, it will queue 50 out of the 100 pending jobs. It will check up to 50 queue-running changes out of the 65 // (15 + 50 from pending), and it will check up to 50 finished jobs from the running ones. // On second iteration, it will queue the remaining 50 pending jobs, and so on... - private static final int NUM_JOBS_HANDLED = 50; + private static final int MAX_NUM_JOBS = 50; private final Query pendingJobsQuery; private final Query queuedJobsQuery; private final Query runningJobsQuery; private final QueryOptions queryOptions; private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final JobScheduler jobScheduler; static { TOOL_CLI_MAP = new HashMap() {{ @@ -257,16 +259,17 @@ public ExecutionDaemon(int interval, String token, CatalogManager catalogManager this.storageConfiguration = storageConfiguration; this.internalCli = appHome + "/bin/opencga-internal.sh"; - this.defaultJobDir = Paths.get(catalogManager.getConfiguration().getJobDir()); - pendingJobsQuery = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.PENDING); queuedJobsQuery = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.QUEUED); runningJobsQuery = new Query(JobDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), Enums.ExecutionStatus.RUNNING); // Sort jobs by priority and creation date queryOptions = new QueryOptions() - .append(QueryOptions.SORT, Arrays.asList(JobDBAdaptor.QueryParams.PRIORITY.key(), - JobDBAdaptor.QueryParams.CREATION_DATE.key())) - .append(QueryOptions.ORDER, QueryOptions.ASCENDING); + .append(QueryOptions.SORT, + Arrays.asList(JobDBAdaptor.QueryParams.PRIORITY.key(), JobDBAdaptor.QueryParams.CREATION_DATE.key())) + .append(QueryOptions.ORDER, QueryOptions.ASCENDING) + .append(QueryOptions.LIMIT, MAX_NUM_JOBS); + + this.jobScheduler = new JobScheduler(catalogManager, token); if (CollectionUtils.isEmpty(packages)) { this.packages = Collections.singletonList(ToolFactory.DEFAULT_PACKAGE); @@ -277,22 +280,13 @@ public ExecutionDaemon(int interval, String token, CatalogManager catalogManager } @Override - public void run() { - while (!exit) { - try { - Thread.sleep(interval); - } catch (InterruptedException e) { - if (!exit) { - e.printStackTrace(); - } - } + public void apply() throws Exception { + checkJobs(); + } - try { - checkJobs(); - } catch (Exception e) { - logger.error("Catch exception " + e.getMessage(), e); - } - } + @Override + public void close() throws IOException { + batchExecutor.close(); try { logger.info("Attempt to shutdown webhook executor"); @@ -311,6 +305,20 @@ public void run() { protected void checkJobs() throws CatalogException { List organizationIds = catalogManager.getAdminManager().getOrganizationIds(token); + + /* + RUNNING JOBS + */ + checkRunningJobs(organizationIds); + + /* + QUEUED JOBS + */ + checkQueuedJobs(organizationIds); + + long totalPendingJobs = 0; + long totalQueuedJobs = 0; + long totalRunningJobs = 0; for (String organizationId : organizationIds) { long pendingJobs = -1; long queuedJobs = -1; @@ -324,29 +332,22 @@ protected void checkJobs() throws CatalogException { } logger.info("----- EXECUTION DAEMON ----- Organization={} --> pending={}, queued={}, running={}", organizationId, pendingJobs, queuedJobs, runningJobs); + totalPendingJobs += pendingJobs; + totalQueuedJobs += queuedJobs; + totalRunningJobs += runningJobs; } - /* - PENDING JOBS - */ - checkPendingJobs(organizationIds); - - /* - QUEUED JOBS - */ - checkQueuedJobs(organizationIds); - - /* - RUNNING JOBS - */ - checkRunningJobs(organizationIds); + if (totalQueuedJobs == 0) { + // Check PENDING jobs + checkPendingJobs(organizationIds); + } } protected void checkRunningJobs(List organizationIds) { for (String organizationId : organizationIds) { int handledRunningJobs = 0; try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, runningJobsQuery, queryOptions, token)) { - while (handledRunningJobs < NUM_JOBS_HANDLED && iterator.hasNext()) { + while (handledRunningJobs < MAX_NUM_JOBS && iterator.hasNext()) { try { Job job = iterator.next(); handledRunningJobs += checkRunningJob(job); @@ -363,6 +364,23 @@ protected void checkRunningJobs(List organizationIds) { protected int checkRunningJob(Job job) { Enums.ExecutionStatus jobStatus = getCurrentStatus(job); + if (killSignalSent(job)) { + logger.info("[{}] - Kill signal request received for job with status='{}'. Attempting to abort execution.", job.getId(), + job.getInternal().getStatus().getId()); + try { + if (batchExecutor.kill(job.getId())) { + return abortKillJob(job, "Job was already in execution. Job killed by the user."); + } else { + logger.info("[{}] - Kill signal send. Waiting for job to finish.", job.getId()); + return 0; + } + } catch (Exception e) { + // Skip this job. Will be retried next loop iteration + logger.error("[{}] - Error trying to kill the job: {}", job.getId(), e.getMessage(), e); + return 0; + } + } + switch (jobStatus.getId()) { case Enums.ExecutionStatus.RUNNING: ExecutionResult result = readExecutionResult(job); @@ -405,7 +423,7 @@ protected void checkQueuedJobs(List organizationIds) { for (String organizationId : organizationIds) { int handledQueuedJobs = 0; try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, queuedJobsQuery, queryOptions, token)) { - while (handledQueuedJobs < NUM_JOBS_HANDLED && iterator.hasNext()) { + while (handledQueuedJobs < MAX_NUM_JOBS && iterator.hasNext()) { try { Job job = iterator.next(); handledQueuedJobs += checkQueuedJob(job); @@ -428,6 +446,24 @@ protected void checkQueuedJobs(List organizationIds) { protected int checkQueuedJob(Job job) { Enums.ExecutionStatus status = getCurrentStatus(job); + // If the job is already running, let the running-jobs step check it + if (killSignalSent(job) && !status.getId().equals(Enums.ExecutionStatus.RUNNING)) { + logger.info("[{}] - Kill signal request received for job with status='{}'. Attempting to avoid execution.", job.getId(), + job.getInternal().getStatus().getId()); + try { + if (batchExecutor.kill(job.getId())) { + return abortKillJob(job, "Job was already queued. Job killed by the user."); + } else { + logger.info("[{}] - Kill signal send. Waiting for job to finish.", job.getId()); + return 0; + } + } catch (Exception e) { + logger.error("[{}] - Error trying to kill the job: {}", job.getId(), e.getMessage(), e); + // Skip this job. Will be retried next loop iteration + return 0; + } + } + switch (status.getId()) { case Enums.ExecutionStatus.QUEUED: // Job is still queued @@ -459,17 +495,36 @@ protected void checkPendingJobs(List organizationIds) { // Clear job counts each cycle jobsCountByType.clear(); + // If there are no queued jobs, we can queue new jobs + List pendingJobs = new LinkedList<>(); + List runningJobs = new LinkedList<>(); + for (String organizationId : organizationIds) { - int handledPendingJobs = 0; try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, pendingJobsQuery, queryOptions, token)) { - while (handledPendingJobs < NUM_JOBS_HANDLED && iterator.hasNext()) { - try { - Job job = iterator.next(); - handledPendingJobs += checkPendingJob(organizationId, job); - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); - } + while (iterator.hasNext()) { + pendingJobs.add(iterator.next()); + } + } catch (Exception e) { + logger.error("Error listing pending jobs from organization {}", organizationId, e); + return; + } + + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, runningJobsQuery, queryOptions, token)) { + while (iterator.hasNext()) { + runningJobs.add(iterator.next()); } + } catch (Exception e) { + logger.error("Error listing running jobs from organization {}", organizationId, e); + return; + } + } + + Iterator iterator = jobScheduler.schedule(pendingJobs, Collections.emptyList(), runningJobs); + boolean success = false; + while (iterator.hasNext() && !success) { + Job job = iterator.next(); + try { + success = checkPendingJob(job) > 0; } catch (Exception e) { logger.error("{}", e.getMessage(), e); } @@ -479,19 +534,26 @@ protected void checkPendingJobs(List organizationIds) { /** * Check everything is correct and queues the job. * - * @param organizationId Organization id. * @param job Job object. * @return 1 if the job has changed the status, 0 otherwise. */ - protected int checkPendingJob(String organizationId, Job job) { + protected int checkPendingJob(Job job) { if (StringUtils.isEmpty(job.getStudy().getId())) { - return abortJob(job, "Missing mandatory 'studyUuid' field"); + return abortJob(job, "Missing mandatory 'study' field"); } + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudyFqn(job.getStudy().getId()); + String organizationId = catalogFqn.getOrganizationId(); if (StringUtils.isEmpty(job.getTool().getId())) { return abortJob(job, "Tool id '" + job.getTool().getId() + "' not found."); } + if (killSignalSent(job)) { + logger.info("[{}] - Kill signal request received for job with status='{}'. Job did not start the execution.", job.getId(), + job.getInternal().getStatus().getId()); + return abortJob(job, "Job killed by the user."); + } + if (!canBeQueued(organizationId, job)) { return 0; } @@ -606,6 +668,10 @@ protected int checkPendingJob(String organizationId, Job job) { return 1; } + private boolean killSignalSent(Job job) { + return job.getInternal().isKillJobRequested(); + } + protected void checkToolExecutionPermission(String organizationId, Job job) throws Exception { Tool tool = new ToolFactory().getTool(job.getTool().getId(), packages); @@ -755,12 +821,6 @@ private File getValidDefaultOutDir(String organizationId, Job job) throws Catalo try { catalogManager.getIoManagerFactory().get(folder.getUri()).createDirectory(folder.getUri(), true); } catch (CatalogIOException | IOException e) { - // Submit job to delete job folder - ObjectMap params = new ObjectMap() - .append("files", folder.getUuid()) - .append("study", job.getStudy().getId()) - .append(Constants.SKIP_TRASH, true); - jobManager.submit(job.getStudy().getId(), FileDeleteTask.ID, Enums.Priority.LOW, params, token); throw new CatalogException("Cannot create job directory '" + folder.getUri() + "' for path '" + folder.getPath() + "'"); } @@ -787,13 +847,14 @@ private File getValidDefaultOutDir(String organizationId, Job job) throws Catalo public static String buildCli(String internalCli, Job job) { String toolId = job.getTool().getId(); String internalCommand = TOOL_CLI_MAP.get(toolId); - if (StringUtils.isEmpty(internalCommand)) { + if (StringUtils.isEmpty(internalCommand) || job.isDryRun()) { ObjectMap params = new ObjectMap() .append(JOB_PARAM, job.getId()) .append(STUDY_PARAM, job.getStudy().getId()); return buildCli(internalCli, "tools execute-job", params); } else { - return buildCli(internalCli, internalCommand, job.getParams()); + ObjectMap params = new ObjectMap(job.getParams()); + return buildCli(internalCli, internalCommand, params); } } @@ -874,6 +935,14 @@ private boolean canBeQueued(String organizationId, Job job) { return false; } + if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { + Date date = TimeUtils.toDate(job.getScheduledStartTime()); + if (date.after(new Date())) { + logger.debug("Job '{}' can't be queued yet. It is scheduled to start at '{}'.", job.getId(), job.getScheduledStartTime()); + return false; + } + } + Integer maxJobs = catalogManager.getConfiguration().getAnalysis().getExecution().getMaxConcurrentJobs().get(job.getTool().getId()); if (maxJobs == null) { // No limit for this tool @@ -926,10 +995,15 @@ private int abortJob(Job job, String message, Exception e) { } private int abortJob(Job job, String description) { - logger.info("Aborting job: {} - Reason: '{}'", job.getId(), description); + logger.info("[{}] - Aborting job - Reason: '{}'", job.getId(), description); return setStatus(job, new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED, description)); } + private int abortKillJob(Job job, String description) { + logger.info("[{}] -Aborting job - Reason: '{}'", job.getId(), description); + return processFinishedJob(job, new Enums.ExecutionStatus(Enums.ExecutionStatus.ABORTED, description)); + } + private int setStatus(Job job, Enums.ExecutionStatus status) { PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams().setInternal(new JobInternal(status)); @@ -989,7 +1063,9 @@ private int processFinishedJob(Job job, Enums.ExecutionStatus status) { logger.info("[{}] - Registering job results from '{}'", job.getId(), Paths.get(job.getOutDir().getUri())); ExecutionResult execution = readExecutionResult(job); - if (execution != null) { + if (execution == null) { + logger.warn("[{}] - Execution result not found", job.getId()); + } else { if (execution.getEnd() == null) { // This could happen if the job finished abruptly logger.info("[{}] Missing end date at ExecutionResult", job.getId()); @@ -1058,29 +1134,35 @@ private int processFinishedJob(Job job, Enums.ExecutionStatus status) { updateParams.setInternal(new JobInternal()); // Check status of analysis result or if there are files that could not be moved to outdir to decide the final result - if (execution == null) { - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, - "Job could not finish successfully. Missing execution result")); - } else if (execution.getStatus().getName().equals(Status.Type.ERROR)) { - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, - "Job could not finish successfully")); - } else { - switch (status.getId()) { - case Enums.ExecutionStatus.DONE: - case Enums.ExecutionStatus.READY: - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE)); - break; - case Enums.ExecutionStatus.ABORTED: - updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, "Job aborted!")); - break; - case Enums.ExecutionStatus.ERROR: - default: + switch (status.getId()) { + case Enums.ExecutionStatus.DONE: + case Enums.ExecutionStatus.READY: + if (execution == null) { + // Regardless of the status, without execution result, we will set the status to ERROR updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR, - "Job could not finish successfully")); - break; - } + "Job could not finish successfully. Missing execution result")); + } else if (execution.getStatus().getName() == Status.Type.ERROR) { + // Discrepancy between the status in the execution result and the status of the job + status.setDescription("Job could not finish successfully." + + " Execution result status: " + execution.getStatus().getName()); + updateParams.getInternal().setStatus(status); + } else { + updateParams.getInternal().setStatus(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE)); + } + break; + case Enums.ExecutionStatus.ABORTED: + updateParams.getInternal().setStatus(status); + break; + case Enums.ExecutionStatus.ERROR: + default: + if (StringUtils.isEmpty(status.getDescription())) { + status.setDescription("Job could not finish successfully"); + } + updateParams.getInternal().setStatus(status); + break; } + logger.info("[{}] - Updating job information", job.getId()); // We update the job information try { @@ -1135,7 +1217,7 @@ private void notifyStatusChange(Job job) { } private void sendWebhookNotification(Job job, URL url) throws URISyntaxException, CatalogException, CloneNotSupportedException { - JobInternal jobInternal = new JobInternal(null, null, null, job.getInternal().getWebhook().clone(), null); + JobInternal jobInternal = new JobInternal(null, null, null, job.getInternal().getWebhook().clone(), null, false); PrivateJobUpdateParams updateParams = new PrivateJobUpdateParams() .setInternal(jobInternal); diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java index 62f98afa91f..393cb2afdd6 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java @@ -16,41 +16,34 @@ package org.opencb.opencga.master.monitor.daemons; -import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.master.monitor.executors.BatchExecutor; import org.opencb.opencga.master.monitor.executors.ExecutorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.file.Path; +import java.io.Closeable; +import java.io.IOException; /** * Created by imedina on 16/06/16. */ -public abstract class MonitorParentDaemon implements Runnable { +public abstract class MonitorParentDaemon implements Runnable, Closeable { - protected int interval; - protected CatalogManager catalogManager; - // FIXME: This should not be used directly! All the queries MUST go through the CatalogManager - @Deprecated - protected DBAdaptorFactory dbAdaptorFactory; + protected final int interval; + protected final CatalogManager catalogManager; protected BatchExecutor batchExecutor; - protected boolean exit = false; + private volatile boolean exit = false; - protected String token; + protected final String token; + protected final Logger logger; - protected Logger logger; - - public MonitorParentDaemon(int interval, String token, CatalogManager catalogManager) throws CatalogDBException { + public MonitorParentDaemon(int interval, String token, CatalogManager catalogManager) { this.interval = interval; this.catalogManager = catalogManager; this.token = token; logger = LoggerFactory.getLogger(this.getClass()); - dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory()); ExecutorFactory executorFactory = new ExecutorFactory(catalogManager.getConfiguration()); this.batchExecutor = executorFactory.getExecutor(); } @@ -63,16 +56,42 @@ public void setExit(boolean exit) { this.exit = exit; } - static Path getJobTemporaryFolder(long jobId, Path tempJobFolder) { - return tempJobFolder.resolve(getJobTemporaryFolderName(jobId)); - } + public void run() { + try { + init(); + } catch (Exception e) { + logger.error("Error initializing daemon", e); + throw new RuntimeException(e); + } + + while (!exit) { + try { + Thread.sleep(interval); + } catch (InterruptedException e) { + if (!exit) { + logger.warn("Interrupted while sleeping", e); + } + // If interrupted, stop the daemon + break; + } - static String getJobTemporaryFolderName(long jobId) { - return "J_" + jobId; + try { + apply(); + } catch (Exception e) { + logger.error("Catch exception " + e.getMessage(), e); + } + } + + try { + close(); + } catch (IOException e) { + logger.error("Error closing daemon", e); + } } - public MonitorParentDaemon setBatchExecutor(BatchExecutor batchExecutor) { - this.batchExecutor = batchExecutor; - return this; + public void init() throws Exception { + } + + public abstract void apply() throws Exception; } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java index 05c6a10be19..28900481a81 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/AzureBatchExecutor.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; /** @@ -58,7 +59,11 @@ public AzureBatchExecutor(Execution execution) { this.poolInformation = new PoolInformation().withPoolId(batchPoolId); } -// public void submitAzureTask(Job job, String token) throws IOException { + @Override + public void close() throws IOException { + } + + // public void submitAzureTask(Job job, String token) throws IOException { // String jobId = getOrCreateAzureJob(job.getType()); // TaskAddParameter taskToAdd = new TaskAddParameter(); // taskToAdd.withId(job.getId()).withCommandLine(job.getCommandLine()).withContainerSettings( diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java index 58a811f1538..05c38e18545 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/BatchExecutor.java @@ -16,12 +16,13 @@ package org.opencb.opencga.master.monitor.executors; +import java.io.Closeable; import java.nio.file.Path; /** * Created by pfurio on 22/08/16. */ -public interface BatchExecutor { +public interface BatchExecutor extends Closeable { String TIMEOUT = "timeout"; String STDOUT = "stdout"; @@ -29,9 +30,7 @@ public interface BatchExecutor { String OUTDIR = "outdir"; String NUM_THREADS = "num_threads"; String MAX_MEM = "max_mem"; - @Deprecated - String JOB_STATUS_FILE = "status.json"; - String OUT_LOG_EXTENSION = ".out"; + String OUT_LOG_EXTENSION = ".log"; String ERR_LOG_EXTENSION = ".err"; void execute(String jobId, String queue, String commandLine, Path stdout, Path stderr) throws Exception; diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java index 4b55a6acd69..b96dcf6f7e4 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java @@ -18,9 +18,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.api.model.batch.Job; -import io.fabric8.kubernetes.api.model.batch.JobBuilder; -import io.fabric8.kubernetes.api.model.batch.JobSpecBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.Job; +import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.JobSpecBuilder; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.*; @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -51,6 +52,8 @@ public class K8SExecutor implements BatchExecutor { public static final String K8S_IMAGE_PULL_POLICY = "k8s.imagePullPolicy"; public static final String K8S_IMAGE_PULL_SECRETS = "k8s.imagePullSecrets"; public static final String K8S_TTL_SECONDS_AFTER_FINISHED = "k8s.ttlSecondsAfterFinished"; + public static final String K8S_TERMINATION_GRACE_PERIOD_SECONDS = "k8s.terminationGracePeriodSeconds"; + public static final String K8S_LOG_TO_STDOUT = "k8s.logToStdout"; public static final String K8S_DIND_ROOTLESS = "k8s.dind.rootless"; public static final String K8S_DIND_IMAGE_NAME = "k8s.dind.imageName"; public static final String K8S_REQUESTS = "k8s.requests"; @@ -87,6 +90,7 @@ public class K8SExecutor implements BatchExecutor { .withMountPath("/usr/share/pod") .build(); private static final String DIND_DONE_FILE = "/usr/share/pod/done"; + public static final String JOB_NAME = "job-name"; private final String namespace; private final String imageName; @@ -109,6 +113,8 @@ public class K8SExecutor implements BatchExecutor { private final String imagePullPolicy; private final List imagePullSecrets; private final int ttlSecondsAfterFinished; + private final boolean logToStdout; + private long terminationGracePeriodSeconds; public K8SExecutor(Configuration configuration) { Execution execution = configuration.getAnalysis().getExecution(); @@ -129,6 +135,8 @@ public K8SExecutor(Configuration configuration) { this.imagePullPolicy = execution.getOptions().getString(K8S_IMAGE_PULL_POLICY, "IfNotPresent"); this.imagePullSecrets = buildLocalObjectReference(execution.getOptions().get(K8S_IMAGE_PULL_SECRETS)); this.ttlSecondsAfterFinished = execution.getOptions().getInt(K8S_TTL_SECONDS_AFTER_FINISHED, 3600); + this.terminationGracePeriodSeconds = execution.getOptions().getInt(K8S_TERMINATION_GRACE_PERIOD_SECONDS, 5 * 60); + this.logToStdout = execution.getOptions().getBoolean(K8S_LOG_TO_STDOUT, true); nodeSelector = getMap(execution.getOptions(), K8S_NODE_SELECTOR); if (execution.getOptions().containsKey(K8S_SECURITY_CONTEXT)) { securityContext = buildObject(execution.getOptions().get(K8S_SECURITY_CONTEXT), SecurityContext.class); @@ -216,7 +224,12 @@ public K8SExecutor(Configuration configuration) { .withEnv(new EnvVar("DOCKER_TLS_CERTDIR", "", null)) // .withResources(resources) // TODO: Should we add resources here? .withCommand("/bin/sh", "-c") - .addToArgs("dockerd-entrypoint.sh & while ! test -f " + DIND_DONE_FILE + "; do sleep 5; done; exit 0") + .addToArgs("dockerd-entrypoint.sh & " + // Add trap to capture TERM signal and finish main process + + "trap '" + + "echo \"Container terminated! ;\n" + + "touch " + DIND_DONE_FILE + " ' TERM ;" + + "while ! test -f " + DIND_DONE_FILE + "; do sleep 5; done; exit 0") .addToVolumeMounts(DOCKER_GRAPH_VOLUMEMOUNT) .addToVolumeMounts(TMP_POD_VOLUMEMOUNT) .addAllToVolumeMounts(volumeMounts) @@ -236,10 +249,9 @@ public void eventReceived(Action action, Job k8Job) { } @Override - public void onClose(KubernetesClientException e) { + public void onClose(WatcherException e) { if (e != null) { logger.error("Catch exception at jobs watcher", e); - throw e; } } }); @@ -261,19 +273,25 @@ public void eventReceived(Action action, Pod pod) { } @Override - public void onClose(KubernetesClientException e) { + public void onClose(WatcherException e) { if (e != null) { logger.error("Catch exception at pods watcher", e); - throw e; } } }); } + @Override + public void close() throws IOException { + podsWatcher.close(); + jobsWatcher.close(); + kubernetesClient.close(); + } + @Override public void execute(String jobId, String queue, String commandLine, Path stdout, Path stderr) throws Exception { String jobName = buildJobName(jobId); - final io.fabric8.kubernetes.api.model.batch.Job k8sJob = new JobBuilder() + final io.fabric8.kubernetes.api.model.batch.v1.Job k8sJob = new JobBuilder() .withApiVersion("batch/v1") .withKind("Job") .withMetadata(new ObjectMetaBuilder() @@ -290,6 +308,7 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, .addToAnnotations("cluster-autoscaler.kubernetes.io/safe-to-evict", "false") .build()) .withSpec(new PodSpecBuilder() + .withTerminationGracePeriodSeconds(terminationGracePeriodSeconds) .withImagePullSecrets(imagePullSecrets) .addToContainers(new ContainerBuilder() .withName("opencga") @@ -297,9 +316,8 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, .withImagePullPolicy(imagePullPolicy) .withResources(resources) .addAllToEnv(envVars) - .withCommand("/bin/sh", "-c") - .withArgs("trap 'touch " + DIND_DONE_FILE + "' EXIT ; " - + getCommandLine(commandLine, stdout, stderr)) + .withCommand("/bin/bash", "-c") + .withArgs(getCommandLine(commandLine, stdout, stderr)) .withVolumeMounts(volumeMounts) .addToVolumeMounts(TMP_POD_VOLUMEMOUNT) .withSecurityContext(securityContext) @@ -330,19 +348,20 @@ private boolean shouldAddDockerDaemon(String queue) { /** * Build a valid K8S job name. - * + *

* DNS-1123 label must consist of lower case alphanumeric characters or '-', and must start and * end with an alphanumeric character (e.g. 'my-name', or '123-abc', regex used for validation * is '[a-z0-9]([-a-z0-9]*[a-z0-9])?' - * + *

* Max length = 63 - * + *

* DNS-1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must * start and end with an alphanumeric character (e.g. 'example.com', regex used for validation * is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + * * @param jobIdInput job Is - * @link https://github.com/kubernetes/kubernetes/blob/c560907/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L135 * @return valid name + * @link https://github.com/kubernetes/kubernetes/blob/c560907/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L135 */ protected static String buildJobName(String jobIdInput) { String jobId = jobIdInput.replace("_", "-"); @@ -404,7 +423,53 @@ public boolean resume(String jobId) throws Exception { @Override public boolean kill(String jobId) throws Exception { - return false; + String k8sJobName = buildJobName(jobId); + + switch (getStatus(jobId)) { + case Enums.ExecutionStatus.DONE: + case Enums.ExecutionStatus.ERROR: + case Enums.ExecutionStatus.ABORTED: { + return jobPodExists(k8sJobName); + } + case Enums.ExecutionStatus.UNKNOWN: { + logger.warn("K8s Job '" + k8sJobName + "' not found!"); + return jobPodExists(k8sJobName); + } + case Enums.ExecutionStatus.QUEUED: + case Enums.ExecutionStatus.PENDING: + case Enums.ExecutionStatus.RUNNING: { + deleteJobIfAny(k8sJobName); + return jobPodExists(k8sJobName); + } + default: + return false; + } + } + + private void deleteJobIfAny(String k8sJobName) { + Job k8Job = getKubernetesClient() + .batch() + .v1() + .jobs() + .inNamespace(namespace) + .withName(k8sJobName) + .get(); + + if (k8Job != null) { + logger.info("Deleting kubernetes job '" + k8Job.getMetadata().getName() + "'"); + getKubernetesClient() + .batch() + .v1() + .jobs() + .inNamespace(namespace) + .withName(k8sJobName) + .withGracePeriod(terminationGracePeriodSeconds) + .delete(); + } + } + + private boolean jobPodExists(String k8sJobName) { + return getJobPod(k8sJobName) == null; } @Override @@ -412,6 +477,58 @@ public boolean isExecutorAlive() { return false; } + /** + * We do it this way to avoid writing the session id in the command line (avoid display/monitor/logs) attribute of Job. + * + * @param commandLine Basic command line + * @param stdout File where the standard output will be redirected + * @param stderr File where the standard error will be redirected + * @return The complete command line + */ + @Override + public String getCommandLine(String commandLine, Path stdout, Path stderr) { + // Do "exec" the main command to keep the PID 1 on the main process to allow grace kills. + commandLine = "exec " + commandLine; + // https://stackoverflow.com/questions/692000/how-do-i-write-standard-error-to-a-file-while-using-tee-with-a-pipe + if (stderr != null) { + if (logToStdout) { + commandLine = commandLine + " 2> >( tee -a \"" + stderr.toAbsolutePath() + "\" >&2 )"; + } else { + commandLine = commandLine + " 2>> " + stderr.toAbsolutePath(); + } + } + if (stdout != null) { + if (logToStdout) { + commandLine = commandLine + " > >( tee -a \"" + stdout.toAbsolutePath() + "\")"; + } else { + commandLine = commandLine + " >> " + stdout.toAbsolutePath(); + } + } + + // Add trap to capture TERM signal and kill the main process + String trapTerm = "trap '" + + "echo \"Job terminated! Run time : ${SECONDS}s\" ;\n" + + "touch INTERRUPTED ;\n" + + "if [ -s PID ] && ps -p $(cat PID) > /dev/null; then\n" + + " kill -15 $(cat PID) ;\n" + + "fi' TERM ;"; + + // Launch the main process in background. + String mainProcess = commandLine + " &"; + + String wait = "PID=$! ; \n" + + "echo $PID > PID ; \n" + + "while ps -p \"$PID\" >/dev/null; do \n" + + " sleep 1 ; \n" + + "done \n" + + "touch '" + DIND_DONE_FILE + "' \n" + + "if [ -f INTERRUPTED ]; then\n" + + " exit 1\n" + + "fi"; + + return trapTerm + " " + mainProcess + " " + wait; + } + private String getStatusForce(String k8sJobName) { Job k8Job = getKubernetesClient() .batch() @@ -421,7 +538,7 @@ private String getStatusForce(String k8sJobName) { .get(); if (k8Job == null) { - logger.warn("Job " + k8sJobName + " not found!"); + logger.warn("Job '" + k8sJobName + "' not found!"); // Job has been deleted. manually? return Enums.ExecutionStatus.ABORTED; } @@ -430,15 +547,23 @@ private String getStatusForce(String k8sJobName) { if (statusFromK8sJob.equalsIgnoreCase(Enums.ExecutionStatus.UNKNOWN) || statusFromK8sJob.equalsIgnoreCase(Enums.ExecutionStatus.QUEUED)) { logger.warn("Job status " + statusFromK8sJob + " . Fetch POD info"); - List pods = getKubernetesClient().pods().withLabel("job-name", k8sJobName).list(1, null).getItems(); - if (!pods.isEmpty()) { - Pod pod = pods.get(0); + Pod pod = getJobPod(k8sJobName); + if (pod != null) { return getStatusFromPod(pod); } } return statusFromK8sJob; } + private Pod getJobPod(String k8sJobName) { + List pods = getKubernetesClient().pods().withLabel(JOB_NAME, k8sJobName).list(1, null).getItems(); + if (pods.isEmpty()) { + return null; + } else { + return pods.get(0); + } + } + private String getJobName(Pod pod) { if (pod.getMetadata() == null || pod.getMetadata().getLabels() == null @@ -446,7 +571,7 @@ private String getJobName(Pod pod) { || pod.getStatus().getPhase() == null) { return null; } - return pod.getMetadata().getLabels().get("job-name"); + return pod.getMetadata().getLabels().get(JOB_NAME); } private String getStatusFromPod(Pod pod) { @@ -486,7 +611,7 @@ private String getStatusFromK8sJob(Job k8Job, String k8sJobName) { } else if (k8Job.getStatus().getActive() != null && k8Job.getStatus().getActive() > 0) { // status = Enums.ExecutionStatus.RUNNING; status = Enums.ExecutionStatus.QUEUED; - } else { + } else { status = Enums.ExecutionStatus.UNKNOWN; } logger.debug("k8Job '{}}' status = '{}'", k8sJobName, status); @@ -514,6 +639,7 @@ private List buildObjects(List list, Class clazz) { } return ts; } + private T buildObject(Object o, Class clazz) { if (o == null) { return null; diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java index 5fd46ba866e..901dd7cb3c1 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/LocalExecutor.java @@ -44,6 +44,7 @@ public class LocalExecutor implements BatchExecutor { private static Logger logger; private final ExecutorService threadPool; private final Map jobStatus; + private final Map runningJobs; private final int maxConcurrentJobs; public LocalExecutor(Execution execution) { @@ -56,6 +57,17 @@ protected boolean removeEldestEntry(Map.Entry eldest) { return size() > 1000; } }); + runningJobs = Collections.synchronizedMap(new LinkedHashMap(1000) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 1000; + } + }); + } + + @Override + public void close() throws IOException { + threadPool.shutdown(); } @Override @@ -65,8 +77,9 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, try { Thread.currentThread().setName("LocalExecutor-" + nextThreadNum()); logger.info("Ready to run - {}", commandLine); - jobStatus.put(jobId, Enums.ExecutionStatus.RUNNING); Command com = new Command(commandLine); + runningJobs.put(jobId, com); + jobStatus.put(jobId, Enums.ExecutionStatus.RUNNING); DataOutputStream dataOutputStream = new DataOutputStream(new FileOutputStream(stdout.toFile())); com.setOutputOutputStream(dataOutputStream); @@ -78,6 +91,7 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, logger.info("Running ShutdownHook. Job {id: " + jobId + "} has being aborted."); com.setStatus(RunnableProcess.Status.KILLED); com.setExitValue(-2); + runningJobs.remove(jobId); closeOutputStreams(com); jobStatus.put(jobId, Enums.ExecutionStatus.ERROR); }); @@ -90,9 +104,11 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, try { Runtime.getRuntime().addShutdownHook(hook); + runningJobs.put(jobId, com); com.run(); } finally { Runtime.getRuntime().removeShutdownHook(hook); + runningJobs.remove(jobId); closeOutputStreams(com); } @@ -135,7 +151,13 @@ public boolean resume(String jobId) throws Exception { @Override public boolean kill(String jobId) throws Exception { - return false; + Command command = runningJobs.get(jobId); + if (command != null) { + command.destroy(); + return true; + } else { + return false; + } } @Override diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java index 1d46dd5b0a0..0d0734deaca 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/SGEExecutor.java @@ -21,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; /** @@ -36,6 +37,10 @@ public SGEExecutor(Execution execution) { sgeManager = new SGEManager(execution); } + @Override + public void close() throws IOException { + } + @Override public void execute(String jobId, String queue, String commandLine, Path stdout, Path stderr) throws Exception { sgeManager.queueJob(jobId, "", -1, getCommandLine(commandLine, stdout, stderr), null); diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java new file mode 100644 index 00000000000..e17fc25e0ab --- /dev/null +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java @@ -0,0 +1,232 @@ +package org.opencb.opencga.master.monitor.schedulers; + +import org.apache.commons.lang3.time.StopWatch; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.api.UserDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.managers.OrganizationManager; +import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.study.Group; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class JobScheduler { + + private final CatalogManager catalogManager; + private final String token; + + private Map userRoles; + + private static final float PRIORITY_WEIGHT = 0.6F; + private static final float IDLE_TIME_WEIGHT = 0.4F; +// private static final float OPERATION_WEIGHT = 0.2F; +// private static final float USER_PRIORITY_WEIGHT = 0.2F; + + private final Logger logger = LoggerFactory.getLogger(JobScheduler.class); + + + public JobScheduler(CatalogManager catalogManager, String token) { + this.catalogManager = catalogManager; + this.token = token; + } + + private void getUserRoles() throws CatalogException { + StopWatch stopWatch = StopWatch.createStarted(); + this.userRoles = new HashMap<>(); + + List organizationIds = catalogManager.getOrganizationManager().getOrganizationIds(token); + for (String organizationId : organizationIds) { + if (ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.ID.key()); + catalogManager.getUserManager().search(organizationId, new Query(), options, token).getResults() + .forEach(user -> getUserRole(organizationId, user.getId()).setSuperAdmin(true)); + } else { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, + OrganizationManager.INCLUDE_ORGANIZATION_ADMINS, token).first(); + getUserRole(organizationId, organization.getOwner()).addOrganizationOwner(organizationId); + organization.getAdmins().forEach(user -> getUserRole(organizationId, user).addOrganizationAdmin(organizationId)); + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.FQN.key(), + StudyDBAdaptor.QueryParams.GROUPS.key())); + catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), options, token).getResults() + .forEach(study -> { + for (Group group : study.getGroups()) { + if (ParamConstants.ADMINS_GROUP.equals(group.getId())) { + group.getUserIds().forEach(user -> getUserRole(organizationId, user).addStudyAdmin(study.getFqn())); + } + } + }); + } + } + + logger.debug("Time spent fetching user roles: {}", TimeUtils.durationToString(stopWatch)); + } + + private UserRole getUserRole(String organizationId, String userId) { + String id = organizationId + "@" + userId; + if (!this.userRoles.containsKey(id)) { + this.userRoles.put(id, new UserRole()); + } + return this.userRoles.get(id); + } + + private float getPriorityWeight(Job job) { + float appPriority; + switch (job.getPriority()) { + case URGENT: + appPriority = 1f; + break; + case HIGH: + appPriority = 0.8f; + break; + case MEDIUM: + appPriority = 0.5f; + break; + case LOW: + case UNKNOWN: + default: + appPriority = 0.2f; + break; + } + + // Increase priority depending on the time the job has been waiting in PENDING status + Calendar todayCalendar = Calendar.getInstance(); + Date date = TimeUtils.toDate(job.getInternal().getRegistrationDate()); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + while (appPriority < 1f) { + calendar.add(Calendar.HOUR, 24); + if (calendar.before(todayCalendar)) { + appPriority = Math.max(1f, appPriority + 0.05f); + } else { + break; + } + } + + // User's priority + float usersPriority; + switch (job.getPriority()) { + case URGENT: + usersPriority = 1f; + break; + case HIGH: + usersPriority = 0.8f; + break; + case MEDIUM: + usersPriority = 0.5f; + break; + case LOW: + case UNKNOWN: + default: + usersPriority = 0.2f; + break; + } + + // Adjust user's priority depending on the user's role + String userId = job.getUserId(); + String organizationId = CatalogFqn.extractFqnFromStudyFqn(job.getStudy().getId()).getOrganizationId(); + + UserRole userRole = getUserRole(organizationId, userId); + if (userRole.isSuperAdmin) { + usersPriority = usersPriority * 1; + } else if (userRole.isOrganizationOwner(organizationId)) { + usersPriority = usersPriority * 0.8f; + } else if (userRole.isOrganizationAdmin(organizationId)) { + usersPriority = usersPriority * 0.75f; + } else if (userRole.isStudyAdmin(job.getStudy().getId())) { + usersPriority = usersPriority * 0.6f; + } else { + usersPriority = usersPriority * 0.4f; + } + + return appPriority * 0.6f + usersPriority * 0.4f; + } + + public Iterator schedule(List pendingJobs, List queuedJobs, List runningJobs) { + TreeMap> jobTreeMap = new TreeMap<>(); + + try { + getUserRoles(); + } catch (CatalogException e) { + throw new RuntimeException("Scheduler exception: " + e.getMessage(), e); + } + + StopWatch stopWatch = StopWatch.createStarted(); + for (Job job : pendingJobs) { + float priority = getPriorityWeight(job); + if (!jobTreeMap.containsKey(priority)) { + jobTreeMap.put(priority, new ArrayList<>()); + } + jobTreeMap.get(priority).add(job); + } + logger.debug("Time spent scheduling jobs: {}", TimeUtils.durationToString(stopWatch)); + + stopWatch.reset(); + stopWatch.start(); + // Obtain iterator + List allJobs = new ArrayList<>(); + for (Float priority : jobTreeMap.descendingKeySet()) { + allJobs.addAll(jobTreeMap.get(priority)); + } + logger.debug("Time spent creating iterator: {}", TimeUtils.durationToString(stopWatch)); + + return allJobs.iterator(); + } + + private static class UserRole { + + private boolean isSuperAdmin; + private Set organizationOwners; + private Set organizationAdmins; + private Set studyAdmins; + + UserRole() { + this.organizationOwners = new HashSet<>(); + this.organizationAdmins = new HashSet<>(); + this.studyAdmins = new HashSet<>(); + } + + public boolean isSuperAdmin() { + return isSuperAdmin; + } + + public UserRole setSuperAdmin(boolean superAdmin) { + isSuperAdmin = superAdmin; + return this; + } + + public void addOrganizationOwner(String userId) { + organizationOwners.add(userId); + } + + public void addOrganizationAdmin(String userId) { + organizationAdmins.add(userId); + } + + public void addStudyAdmin(String userId) { + studyAdmins.add(userId); + } + + public boolean isOrganizationOwner(String organizationId) { + return organizationOwners.contains(organizationId); + } + + public boolean isOrganizationAdmin(String organizationId) { + return organizationAdmins.contains(organizationId); + } + + public boolean isStudyAdmin(String studyFqn) { + return studyAdmins.contains(studyFqn); + } + } + +} diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java index 89f316e1c21..9d2de1bdb42 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java @@ -24,6 +24,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; +import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.operations.VariantAnnotationIndexOperationTool; import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; @@ -33,6 +34,7 @@ import org.opencb.opencga.catalog.managers.FileManager; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.AclParams; @@ -52,6 +54,7 @@ import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.master.monitor.executors.BatchExecutor; import org.opencb.opencga.master.monitor.models.PrivateJobUpdateParams; +import org.opencb.opencga.storage.core.StorageEngineFactory; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -214,12 +217,31 @@ public void testNotEmptyOutDir() throws Exception { } @Test - public void testProjectScopeTask() throws Exception { + public void dryRunExecutionTest() throws Exception { + ObjectMap params = new ObjectMap(); + String jobId1 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, + params, "job1", "", null, null, null, null, true, ownerToken).first().getId(); + daemon.checkJobs(); + Job job = catalogManager.getJobManager().get(studyFqn, jobId1, null, ownerToken).first(); + + StorageConfiguration storageConfiguration = new StorageConfiguration(); + storageConfiguration.getVariant().setDefaultEngine("mongodb"); + ToolRunner toolRunner = new ToolRunner(catalogManagerResource.getOpencgaHome().toString(), catalogManager, + StorageEngineFactory.get(storageConfiguration)); + toolRunner.execute(job, ownerToken); + daemon.checkJobs(); + + job = catalogManager.getJobManager().get(studyFqn, jobId1, null, ownerToken).first(); + assertEquals(Enums.ExecutionStatus.DONE, job.getInternal().getStatus().getId()); + assertEquals(1, job.getExecution().getSteps().size()); + assertEquals("check", job.getExecution().getSteps().get(0).getId()); + } + + @Test + public void testProjectScopeTaskAndScheduler() throws Exception { // User 2 to admins group in study1 but not in study2 catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, new GroupUpdateParams(Collections.singletonList(normalUserId2)), ownerToken); -// catalogManager.getStudyManager().updateAcl(studyFqn2, normalUserId2, new StudyAclParams().setTemplate(AuthorizationManager.ROLE_VIEW_ONLY), -// ParamUtils.AclAction.SET, ownerToken); // User 3 to admins group in both study1 and study2 catalogManager.getStudyManager().updateGroup(studyFqn, ParamConstants.ADMINS_GROUP, ParamUtils.BasicUpdateAction.ADD, @@ -228,29 +250,97 @@ public void testProjectScopeTask() throws Exception { new GroupUpdateParams(Collections.singletonList(normalUserId3)), ownerToken); HashMap params = new HashMap<>(); - String jobId = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, - params, "job1", "", null, null, ownerToken).first().getId(); + String jobId1 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, + params, "job1", "", null, null, null, null, false, normalToken2).first().getId(); String jobId2 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, - params, "job2", "", null, null, normalToken2).first().getId(); + params, "job2", "", null, null, null, null, false, orgAdminToken1).first().getId(); String jobId3 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, - params, "job3", "", null, null, normalToken3).first().getId(); + params, "job3", "", null, null, null, null, false, ownerToken).first().getId(); + String jobId4 = catalogManager.getJobManager().submit(studyFqn, VariantAnnotationIndexOperationTool.ID, Enums.Priority.MEDIUM, + params, "job4", "", null, null, null, null, false, normalToken3).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkJobs(); // Job sent by the owner - OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), ownerToken); + OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId3, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); - // Job sent by user2 (admin from study1 but not from study2) + // Job sent by normal user + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + daemon.checkJobs(); + + // Because there's already a QUEUED job, it should not process any more jobs + // Job sent by the owner + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId3, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); + + // Job sent normal user + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Set to DONE jobId3 so it can process more jobs + catalogManager.getJobManager().update(studyFqn, jobId3, new PrivateJobUpdateParams() + .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), QueryOptions.empty(), ownerToken); + + daemon.checkJobs(); + // Job sent by normal user should still be PENDING + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + // Job sent by org admin + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId2, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); + + // Set to DONE jobId2 so it can process more jobs + catalogManager.getJobManager().update(studyFqn, jobId2, new PrivateJobUpdateParams() + .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.DONE))), QueryOptions.empty(), ownerToken); + + daemon.checkJobs(); + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId1, QueryOptions.empty(), ownerToken); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.ABORTED); assertTrue(jobOpenCGAResult.first().getInternal().getStatus().getDescription() .contains("can only be executed by the project owners or members of " + ParamConstants.ADMINS_GROUP)); - // Job sent by user3 (admin from study1 and study2) - jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId3, QueryOptions.empty(), ownerToken); + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); + assertEquals(1, jobOpenCGAResult.getNumResults()); + checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.PENDING); + + daemon.checkJobs(); // to process jobId4 + + // Job sent by study administrator + jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId4, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.QUEUED); @@ -259,7 +349,7 @@ public void testProjectScopeTask() throws Exception { jobOpenCGAResult = catalogManager.getJobManager().search(studyFqn2, new Query(), new QueryOptions(ParamConstants.OTHER_STUDIES_FLAG, true), ownerToken); - assertEquals(2, jobOpenCGAResult.getNumResults()); + assertEquals(3, jobOpenCGAResult.getNumResults()); } @Test @@ -267,7 +357,7 @@ public void testDependsOnJobs() throws Exception { HashMap params = new HashMap<>(); String job1 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); String job2 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, null, null, - Collections.singletonList(job1), null, ownerToken).first().getId(); + Collections.singletonList(job1), null, null, null, false, ownerToken).first().getId(); daemon.checkPendingJobs(organizationIds); @@ -296,7 +386,7 @@ public void testDependsOnJobs() throws Exception { // And create a new job to simulate a normal successfully dependency String job3 = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, null, null, - Collections.singletonList(job1), null, ownerToken).first().getId(); + Collections.singletonList(job1), null, null, null, false, ownerToken).first().getId(); daemon.checkPendingJobs(organizationIds); jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job3, QueryOptions.empty(), ownerToken); @@ -309,7 +399,7 @@ public void testDependsOnMultiStudy() throws Exception { HashMap params = new HashMap<>(); Job firstJob = catalogManager.getJobManager().submit(studyFqn, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first(); Job job = catalogManager.getJobManager().submit(studyFqn2, "files-delete", Enums.Priority.MEDIUM, params, null, null, - Collections.singletonList(firstJob.getUuid()), null, ownerToken).first(); + Collections.singletonList(firstJob.getUuid()), null, null, null, false, ownerToken).first(); assertEquals(1, job.getDependsOn().size()); assertEquals(firstJob.getId(), job.getDependsOn().get(0).getId()); assertEquals(firstJob.getUuid(), job.getDependsOn().get(0).getUuid()); @@ -417,6 +507,7 @@ public void testCheckLogsNoPermissions() throws Exception { Paths.get(job.getOutDir().getUri()).resolve(job.getId() + ".log").toUri()); thrown.expect(CatalogAuthorizationException.class); + thrown.expectMessage("view log file"); catalogManager.getJobManager().log(studyFqn, jobId, 0, 1, "stdout", true, normalToken2); } @@ -500,6 +591,29 @@ public void testRegisterFilesSuccessfully() throws Exception { assertEquals(16, files.getNumMatches()); } + @Test + public void scheduledJobTest() throws CatalogException, InterruptedException { + HashMap params = new HashMap<>(); + params.put(ExecutionDaemon.OUTDIR_PARAM, "outputDir/"); + Date date = new Date(); + // Create a date object with the current time + 2 seconds + date.setTime(date.getTime() + 2000); + String scheduledTime = TimeUtils.getTime(date); + System.out.println("Scheduled time: " + scheduledTime); + Job job = catalogManager.getJobManager().submit(studyFqn, "variant-index", Enums.Priority.MEDIUM, params, null, null, null, + null, null, scheduledTime, null, ownerToken).first(); + + daemon.checkJobs(); + checkStatus(getJob(job.getId()), Enums.ExecutionStatus.PENDING); + daemon.checkJobs(); + checkStatus(getJob(job.getId()), Enums.ExecutionStatus.PENDING); + + // Sleep for 2 seconds and check again + Thread.sleep(2000); + daemon.checkJobs(); + checkStatus(getJob(job.getId()), Enums.ExecutionStatus.QUEUED); + } + @Test public void testRunJobFail() throws Exception { HashMap params = new HashMap<>(); @@ -604,6 +718,10 @@ public void execute(String jobId, String queue, String commandLine, Path stdout, jobStatus.put(jobId, Enums.ExecutionStatus.QUEUED); } + @Override + public void close() throws IOException { + } + @Override public String getStatus(String jobId) { return jobStatus.getOrDefault(jobId, Enums.ExecutionStatus.UNKNOWN); diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java index 5a237749412..a350fdc3c77 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java @@ -31,14 +31,18 @@ import java.util.Collections; import java.util.Date; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + @Category(ShortTests.class) public class LocalExecutorTest { + public static final int MAX_CONCURRENT_JOBS = 5; private LocalExecutor localExecutor; private Path rootDir; @Before public void setUp() throws Exception { - localExecutor = new LocalExecutor(new Execution().setOptions(new ObjectMap(LocalExecutor.MAX_CONCURRENT_JOBS, 5))); + localExecutor = new LocalExecutor(new Execution().setOptions(new ObjectMap(LocalExecutor.MAX_CONCURRENT_JOBS, MAX_CONCURRENT_JOBS))); rootDir = Paths.get("target/test-data", "junit-opencga-" + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss.SSS").format(new Date())); Files.createDirectories(rootDir); @@ -58,11 +62,41 @@ public void test() throws Exception { Thread.sleep(1000); } Assert.assertTrue(Files.exists(rootDir.resolve("out_" + i + ".txt"))); - Assert.assertEquals(Collections.singletonList("Hello World"), Files.readAllLines(rootDir.resolve("out_" + i + ".txt"))); + assertEquals(Collections.singletonList("Hello World"), Files.readAllLines(rootDir.resolve("out_" + i + ".txt"))); Assert.assertTrue(Files.exists(rootDir.resolve("err_" + i + ".txt"))); - Assert.assertEquals(Collections.emptyList(), Files.readAllLines(rootDir.resolve("err_" + i + ".txt"))); + assertEquals(Collections.emptyList(), Files.readAllLines(rootDir.resolve("err_" + i + ".txt"))); + } + } + + @Test(timeout = 10000) + public void testKill() throws Exception { + System.out.println(rootDir.toAbsolutePath()); + for (int i = 0; i < 10; i++) { +// System.out.println("Submitting job " + i); + localExecutor.execute("jobId-" + i, "default", "sleep 20", rootDir.resolve("out_" + i + ".txt"), rootDir.resolve("err_" + i + ".txt")); } + // Allow some time for the jobs to start + Thread.sleep(50); + for (int i = 0; i < 10; i++) { + String jobId = "jobId-" + i; +// System.out.println("Checking status of job " + jobId); + if (i < MAX_CONCURRENT_JOBS) { + assertEquals(jobId, "RUNNING", localExecutor.getStatus(jobId)); + } else { + assertEquals(jobId, "QUEUED", localExecutor.getStatus(jobId)); + } + } + for (int i = 0; i < 10; i++) { + String jobId = "jobId-" + i; +// System.out.println("Checking status of job " + jobId); + do { + Thread.sleep(100); + } while (!localExecutor.getStatus(jobId).equals("RUNNING")); + assertEquals("RUNNING", localExecutor.getStatus(jobId)); + assertTrue(localExecutor.kill(jobId)); + } } + } \ No newline at end of file diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index 6863e08197c..eabe8a65d7c 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/AbstractStorageServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/AbstractStorageServer.java index e1e880f2d47..d4bcb75d575 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/AbstractStorageServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/AbstractStorageServer.java @@ -16,58 +16,29 @@ package org.opencb.opencga.server; -import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; /** * Created by imedina on 02/01/16. */ public abstract class AbstractStorageServer { - @Deprecated - protected Path configDir; protected Path opencgaHome; protected int port; protected Configuration configuration; protected StorageConfiguration storageConfiguration; - /** - * This is the default StorageEngine to use when it is not provided by the client. - */ - @Deprecated - protected String defaultStorageEngine; - protected Logger logger; - - @Deprecated - public AbstractStorageServer() { - initDefaultConfigurationFiles(); - } - - @Deprecated - public AbstractStorageServer(int port, String defaultStorageEngine) { - initDefaultConfigurationFiles(); - - this.port = port; - if (StringUtils.isNotEmpty(defaultStorageEngine)) { - this.defaultStorageEngine = defaultStorageEngine; - } else { - this.defaultStorageEngine = storageConfiguration.getVariant().getDefaultEngine(); - } - } - public AbstractStorageServer(Path opencgaHome) { this(opencgaHome, 0); } @@ -93,33 +64,15 @@ private void initConfigurationFiles(Path opencgaHome) { try { if (opencgaHome != null && Files.exists(opencgaHome) && Files.isDirectory(opencgaHome) && Files.exists(opencgaHome.resolve("conf"))) { - logger.info("Loading configuration files from '{}'", opencgaHome.toString()); -// generalConfiguration = GeneralConfiguration.load(GeneralConfiguration.class.getClassLoader().getResourceAsStream("configuration.yml")); - configuration = Configuration - .load(new FileInputStream(new File(opencgaHome.resolve("conf").toFile().getAbsolutePath() + "/configuration.yml"))); - storageConfiguration = StorageConfiguration - .load(new FileInputStream(new File(opencgaHome.resolve("conf").toFile().getAbsolutePath() + "/storage-configuration.yml"))); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Deprecated - private void initDefaultConfigurationFiles() { - try { - if (System.getenv("OPENCGA_HOME") != null) { - initConfigurationFiles(Paths.get(System.getenv("OPENCGA_HOME") + "/conf")); - } else { - logger.info("Loading configuration files from inside JAR file"); + logger.info("Loading configuration files from '{}'", opencgaHome); // generalConfiguration = GeneralConfiguration.load(GeneralConfiguration.class.getClassLoader().getResourceAsStream("configuration.yml")); configuration = Configuration - .load(Configuration.class.getClassLoader().getResourceAsStream("configuration.yml")); + .load(Files.newInputStream(opencgaHome.resolve("conf").resolve("configuration.yml").toFile().toPath())); storageConfiguration = StorageConfiguration - .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml")); + .load(Files.newInputStream(opencgaHome.resolve("conf").resolve("storage-configuration.yml").toFile().toPath())); } } catch (IOException e) { - e.printStackTrace(); + throw new UncheckedIOException(e); } } @@ -133,7 +86,6 @@ private void initDefaultConfigurationFiles() { public String toString() { final StringBuilder sb = new StringBuilder("StorageServer{"); sb.append("port=").append(port); - sb.append(", defaultStorageEngine='").append(defaultStorageEngine).append('\''); sb.append('}'); return sb.toString(); } @@ -146,12 +98,4 @@ public void setPort(int port) { this.port = port; } - public String getDefaultStorageEngine() { - return defaultStorageEngine; - } - - public void setDefaultStorageEngine(String defaultStorageEngine) { - this.defaultStorageEngine = defaultStorageEngine; - } - } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java index 28bdffc8016..c3f87c68239 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/RestServer.java @@ -16,8 +16,12 @@ package org.opencb.opencga.server; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.webapp.WebAppContext; +import org.opencb.opencga.core.config.RestServerConfiguration; import java.io.IOException; import java.nio.file.Files; @@ -44,9 +48,34 @@ public RestServer(Path opencgaHome, int port) { @Override public void start() throws Exception { - server = new Server(port); + initServer(); - WebAppContext webapp = new WebAppContext(); + Path war = getOpencgaWar(); + + initWebApp(war); + + server.start(); + logger.info("REST server started, listening on {}", server.getURI()); + + initHooks(); + +// // AdminWSServer server needs a reference to this class to cll to .stop() +// AdminRestWebService.setServer(this); + } + + protected Server initServer() { + server = new Server(); + + HttpConfiguration httpConfig = getHttpConfiguration(); + + ServerConnector httpConnector = new ServerConnector(server, new HttpConnectionFactory(httpConfig)); + httpConnector.setPort(port); + + server.addConnector(httpConnector); + return server; + } + + protected Path getOpencgaWar() throws Exception { Optional warPath; try (Stream stream = Files.list(opencgaHome)) { warPath = stream @@ -56,35 +85,36 @@ public void start() throws Exception { throw new Exception("Error accessing OpenCGA Home: " + opencgaHome.toString(), e); } // Check is a war file has been found in opencgaHome - if (warPath == null || !warPath.isPresent()) { + if (!warPath.isPresent()) { throw new Exception("No war file found at " + opencgaHome.toString()); } + return warPath.get(); + } - String opencgaVersion = warPath.get().toFile().getName().replace(".war", ""); + protected WebAppContext initWebApp(Path war) throws Exception { + String opencgaVersion = war.toFile().getName().replace(".war", ""); + WebAppContext webapp = new WebAppContext(); webapp.setContextPath("/" + opencgaVersion); - webapp.setWar(warPath.get().toString()); + webapp.setWar(war.toString()); webapp.setClassLoader(this.getClass().getClassLoader()); webapp.setInitParameter("OPENCGA_HOME", opencgaHome.toFile().toString()); webapp.getServletContext().setAttribute("OPENCGA_HOME", opencgaHome.toFile().toString()); // webapp.setInitParameter("log4jConfiguration", opencgaHome.resolve("conf/log4j2.server.xml").toString()); server.setHandler(webapp); + return webapp; + } - server.start(); - logger.info("REST server started, listening on {}", port); - + protected void initHooks() { // A hook is added in case the JVM is shutting down - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - try { - if (server.isRunning()) { - stopJettyServer(); - } - } catch (Exception e) { - e.printStackTrace(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + if (server.isRunning()) { + stopJettyServer(); } + } catch (Exception e) { + logger.error("Error stopping Jetty server", e); } - }); + })); // A separated thread is launched to shut down the server new Thread(() -> { @@ -97,12 +127,30 @@ public void run() { Thread.sleep(500); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error stopping Jetty server", e); } }).start(); + } -// // AdminWSServer server needs a reference to this class to cll to .stop() -// AdminRestWebService.setServer(this); + protected HttpConfiguration getHttpConfiguration() { + HttpConfiguration httpConfig = new HttpConfiguration(); + RestServerConfiguration.HttpConfiguration restHttpConf = configuration.getServer().getRest().getHttpConfiguration(); + if (restHttpConf.getOutputBufferSize() > 0) { + httpConfig.setOutputBufferSize(restHttpConf.getOutputBufferSize()); + } + if (restHttpConf.getOutputAggregationSize() > 0) { + httpConfig.setOutputAggregationSize(restHttpConf.getOutputAggregationSize()); + } + if (restHttpConf.getRequestHeaderSize() > 0) { + httpConfig.setRequestHeaderSize(restHttpConf.getRequestHeaderSize()); + } + if (restHttpConf.getResponseHeaderSize() > 0) { + httpConfig.setResponseHeaderSize(restHttpConf.getResponseHeaderSize()); + } + if (restHttpConf.getHeaderCacheSize() > 0) { + httpConfig.setHeaderCacheSize(restHttpConf.getHeaderCacheSize()); + } + return httpConfig; } @Override diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/grpc/GrpcServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/grpc/GrpcServer.java index 76b4c350a32..32ded068042 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/grpc/GrpcServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/grpc/GrpcServer.java @@ -19,7 +19,6 @@ import io.grpc.Server; import io.grpc.ServerBuilder; import org.opencb.opencga.server.AbstractStorageServer; -import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.slf4j.LoggerFactory; import java.nio.file.Path; @@ -31,29 +30,11 @@ public class GrpcServer extends AbstractStorageServer { private Server server; - public GrpcServer() { -// this(storageConfiguration.getServer().getGrpc(), storageConfiguration.getDefaultStorageEngineId()); - } - - public GrpcServer(int port, String defaultStorageEngine) { - super(port, defaultStorageEngine); - - logger = LoggerFactory.getLogger(this.getClass()); - } - public GrpcServer(Path configDir) { super(configDir); init(); } - @Deprecated - public GrpcServer(StorageConfiguration storageConfiguration) { -// super(storageConfiguration.getServer().getGrpc(), storageConfiguration.getDefaultStorageEngineId()); - this.storageConfiguration = storageConfiguration; - - logger = LoggerFactory.getLogger(this.getClass()); - } - private void init() { logger = LoggerFactory.getLogger(this.getClass()); if (configuration != null) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java index e100ebeb5eb..01be67e5a16 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/FileWSServer.java @@ -98,9 +98,12 @@ public Response downloadAndRegister( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(name = "body", value = "Fetch parameters", required = true) FileFetch fetchParams) { - return submitJob(FetchAndRegisterTask.ID, studyStr, fetchParams, jobId, jobDescription, dependsOn, jobTags); + return submitJob(FetchAndRegisterTask.ID, studyStr, fetchParams, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -709,8 +712,11 @@ public Response postlink( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(name = "body", value = "File parameters", required = true) PostLinkToolParams params) { - return submitJob(PostLinkSampleAssociation.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(PostLinkSampleAssociation.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -722,8 +728,11 @@ public Response linkAsync( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(name = "body", value = "File parameters", required = true) FileLinkToolParams params) { - return submitJob(FileLinkTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(FileLinkTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index e5bfe43f83b..536d692099e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -81,6 +81,7 @@ public Response retryJob( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTagsStr, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String jobScheduledStartTime, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "job", required = true) JobRetryParams params @@ -99,14 +100,24 @@ public Response retryJob( } else { jobTags = Collections.emptyList(); } - OpenCGAResult result = catalogManager.getJobManager().retry(study, params, - null, jobId, jobDescription, jobDependsOn, jobTags, token); + OpenCGAResult result = catalogManager.getJobManager().retry(study, params, null, jobId, jobDescription, jobDependsOn, + jobTags, jobScheduledStartTime, token); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); } } + @POST + @Path("/{job}/kill") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Send a signal to kill a pending or running job", response = Job.class) + public Response kill( + @ApiParam(value = ParamConstants.JOB_ID_DESCRIPTION, required = true) @PathParam("job") String jobId, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study) { + return run(() -> catalogManager.getJobManager().kill(study, jobId, token)); + } + @GET @Path("/{jobs}/info") @ApiOperation(value = "Get job information", response = Job.class) diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java index 686b40cad16..3dae031c245 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java @@ -852,42 +852,46 @@ public Response run(Callable> c) { } } - public Response submitJob(String toolId, String project, String study, Map paramsMap, - String jobName, String jobDescription, String jobDependsOne, String jobTags) { - return run(() -> submitJobRaw(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags)); + public Response submitJob(String toolId, String project, String study, Map paramsMap, String jobName, + String jobDescription, String jobDependsOne, String jobTags, String jobScheduledStartTime, String jobPriority, + Boolean dryRun) { + return run(() -> submitJobRaw(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags, jobScheduledStartTime, jobPriority, dryRun)); } public Response submitJob(String toolId, String study, ToolParams bodyParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr) { - return submitJob(toolId, null, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr); + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return submitJob(toolId, null, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun); } public Response submitJobAdmin(String toolId, ToolParams bodyParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr) { + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, + Boolean dryRun) { return run(() -> { JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(token); catalogManager.getAuthorizationManager().checkIsOpencgaAdministrator(jwtPayload, "submit job from tool '" + toolId + "'"); - return submitJobRaw(toolId, null, ADMIN_STUDY_FQN, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr); + return submitJobRaw(toolId, null, ADMIN_STUDY_FQN, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun); }); } public Response submitJob(String toolId, String project, String study, ToolParams bodyParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr) { - return run(() -> submitJobRaw(toolId, project, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr)); + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return run(() -> submitJobRaw(toolId, project, study, bodyParams, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun)); } - protected DataResult submitJobRaw(String toolId, String project, String study, ToolParams bodyParams, - String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr) + protected DataResult submitJobRaw(String toolId, String project, String study, ToolParams bodyParams, String jobId, + String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) throws CatalogException { Map paramsMap = bodyParams.toParams(); if (StringUtils.isNotEmpty(study)) { paramsMap.putIfAbsent(ParamConstants.STUDY_PARAM, study); } - return submitJobRaw(toolId, project, study, paramsMap, jobId, jobDescription, jobDependsOnStr, jobTagsStr); + return submitJobRaw(toolId, project, study, paramsMap, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun); } - protected DataResult submitJobRaw(String toolId, String project, String study, Map paramsMap, - String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr) + protected DataResult submitJobRaw(String toolId, String project, String study, Map paramsMap, String jobId, + String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) throws CatalogException { if (StringUtils.isNotEmpty(project) && StringUtils.isEmpty(study)) { @@ -918,8 +922,13 @@ protected DataResult submitJobRaw(String toolId, String project, String stu } else { jobDependsOn = Collections.emptyList(); } + Enums.Priority priority = Enums.Priority.MEDIUM; + if (!StringUtils.isEmpty(jobPriority)) { + priority = Enums.Priority.getPriority(jobPriority.toUpperCase()); + } return catalogManager.getJobManager() - .submit(study, toolId, Enums.Priority.MEDIUM, paramsMap, jobId, jobDescription, jobDependsOn, jobTags, token); + .submit(study, toolId, priority, paramsMap, jobId, jobDescription, jobDependsOn, jobTags, null, + jobScheduledStartTime, dryRun, token); } public Response createPendingResponse() { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java index 9deeb489110..15085581bca 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OrganizationWSServer.java @@ -27,8 +27,12 @@ import org.opencb.opencga.core.models.notes.NoteCreateParams; import org.opencb.opencga.core.models.notes.NoteUpdateParams; import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.models.organizations.OrganizationConfiguration; import org.opencb.opencga.core.models.organizations.OrganizationCreateParams; import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams; +import org.opencb.opencga.core.models.user.OrganizationUserUpdateParams; +import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.models.user.UserStatusUpdateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; @@ -96,6 +100,34 @@ public Response update( } } + @POST + @Path("/{organization}/configuration/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update the Organization configuration attributes", response = OrganizationConfiguration.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateConfiguration( + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION, required = true) @PathParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "Action to be performed if the array of authenticationOrigins is being updated.", + allowableValues = "ADD,REMOVE,SET,REPLACE", defaultValue = "ADD") @QueryParam("authenticationOriginsAction") ParamUtils.UpdateAction authOriginsAction, + @ApiParam(value = "JSON containing the params to be updated.", required = true) OrganizationConfiguration parameters) { + try { + if (authOriginsAction == null) { + authOriginsAction = ParamUtils.UpdateAction.ADD; + } + Map actionMap = new HashMap<>(); + actionMap.put(OrganizationDBAdaptor.AUTH_ORIGINS_FIELD, authOriginsAction); + queryOptions.put(Constants.ACTIONS, actionMap); + OpenCGAResult result = catalogManager.getOrganizationManager().updateConfiguration(organizationId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + @POST @Path("/create") @Consumes(MediaType.APPLICATION_JSON) @@ -196,4 +228,47 @@ public Response deleteNote( } } + @POST + @Path("/user/{user}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update the user information", response = User.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateUserInformation( + @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId, + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the User fields to be updated.", required = true) OrganizationUserUpdateParams parameters) { + try { + OpenCGAResult result = catalogManager.getOrganizationManager().updateUser(organizationId, userId, parameters, queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/user/{user}/status/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update the user status", response = User.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, dataType = "string", paramType = "query") + }) + public Response updateUserStatus( + @ApiParam(value = ParamConstants.USER_DESCRIPTION, required = true) @PathParam("user") String userId, + @ApiParam(value = ParamConstants.ORGANIZATION_DESCRIPTION) @QueryParam(ParamConstants.ORGANIZATION) String organizationId, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing the User fields to be updated.", required = true) UserStatusUpdateParams parameters) { + try { + OpenCGAResult result = catalogManager.getUserManager().changeStatus(organizationId, userId, parameters.getStatus(), queryOptions, token); + return createOkResponse(result); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + } \ No newline at end of file diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java index 2ec062bb84f..7d36dec4386 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/PanelWSServer.java @@ -82,8 +82,11 @@ public Response importPanel( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(name = "body", value = "Panel parameters") PanelImportParams params) { - return submitJob(PanelImportTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(PanelImportTask.ID, studyStr, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } // @POST diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java index 64bea0e07e1..f6cc3fe8882 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/StudyWSServer.java @@ -547,8 +547,11 @@ public Response executeTemplate( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = TemplateParams.DESCRIPTION, required = true) TemplateParams params) { - return submitJob(TemplateRunner.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(TemplateRunner.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java index 99fa4402fc9..277f289a6b6 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/AlignmentWebService.java @@ -83,8 +83,11 @@ public Response indexRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = AlignmentIndexParams.DESCRIPTION, required = true) AlignmentIndexParams params) { - return submitJob(AlignmentIndexOperation.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentIndexOperation.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- @@ -177,9 +180,12 @@ public Response coverageRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = CoverageIndexParams.DESCRIPTION, required = true) CoverageIndexParams params) { - return submitJob(AlignmentCoverageAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentCoverageAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); // logger.debug("ObjectMap: {}", params); // @@ -386,9 +392,12 @@ public Response geneCoverageStatsRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = AlignmentGeneCoverageStatsParams.DESCRIPTION, required = true) AlignmentGeneCoverageStatsParams params) { - return submitJob(AlignmentGeneCoverageStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentGeneCoverageStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- @@ -404,9 +413,12 @@ public Response qcRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = AlignmentQcParams.DESCRIPTION, required = true) AlignmentQcParams params) { - return submitJob(AlignmentQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AlignmentQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- @@ -422,8 +434,11 @@ public Response bwaRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = BwaWrapperParams.DESCRIPTION, required = true) BwaWrapperParams params) { - return submitJob(BwaWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(BwaWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -435,8 +450,11 @@ public Response samtoolsRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SamtoolsWrapperParams.DESCRIPTION, required = true) SamtoolsWrapperParams params) { - return submitJob(SamtoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SamtoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -448,8 +466,11 @@ public Response deeptoolsRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = DeeptoolsWrapperParams.DESCRIPTION, required = true) DeeptoolsWrapperParams params) { - return submitJob(DeeptoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(DeeptoolsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -461,8 +482,11 @@ public Response fastqcRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = FastqcWrapperParams.DESCRIPTION, required = true) FastqcWrapperParams params) { - return submitJob(FastqcWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(FastqcWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -474,8 +498,11 @@ public Response picardRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = PicardWrapperParams.DESCRIPTION, required = true) PicardWrapperParams params) { - return submitJob(PicardWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(PicardWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } //------------------------------------------------------------------------- diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java index d69c237e421..f893c1f5986 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/ClinicalWebService.java @@ -170,10 +170,13 @@ public Response load( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ClinicalAnalysisLoadParams.DESCRIPTION, required = true) ClinicalAnalysisLoadParams params) { try { // Execute load as a job - return submitJob(ClinicalAnalysisLoadTask.ID, study, params, jobId, jobDescription, dependsOn, jobTags); + return submitJob(ClinicalAnalysisLoadTask.ID, study, params, jobId, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } catch (Exception e) { return createErrorResponse("Load clinical analyses from file", e.getMessage()); } @@ -1161,6 +1164,9 @@ public Response rgaIndexRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.INDEX_AUXILIAR_COLLECTION_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INDEX_AUXILIAR_COLLECTION) boolean indexAuxiliarColl, @ApiParam(value = RgaAnalysisParams.DESCRIPTION, required = true) RgaAnalysisParams params) { @@ -1169,9 +1175,9 @@ public Response rgaIndexRun( if (StringUtils.isNotEmpty(study)) { paramsMap.putIfAbsent(ParamConstants.STUDY_PARAM, study); } - return submitJob(AuxiliarRgaAnalysis.ID, null, study, paramsMap, jobName, jobDescription, dependsOn, jobTags); + return submitJob(AuxiliarRgaAnalysis.ID, null, study, paramsMap, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } else { - return submitJob(RgaAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(RgaAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } } @@ -1292,8 +1298,11 @@ public Response interpretationTieringRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = TieringInterpretationAnalysisParams.DESCRIPTION, required = true) TieringInterpretationAnalysisParams params) { - return submitJob(TieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(TieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -1306,8 +1315,11 @@ public Response interpretationExomiserRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ExomiserInterpretationAnalysisParams.DESCRIPTION, required = true) ExomiserInterpretationAnalysisParams params) { - return submitJob(ExomiserInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(ExomiserInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1319,8 +1331,11 @@ public Response interpretationTeamRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = TeamInterpretationAnalysisParams.DESCRIPTION, required = true) TeamInterpretationAnalysisParams params) { - return submitJob(TeamInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(TeamInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1332,8 +1347,11 @@ public Response interpretationZettaRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ZettaInterpretationAnalysisParams.DESCRIPTION, required = true) ZettaInterpretationAnalysisParams params) { - return submitJob(ZettaInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(ZettaInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1345,7 +1363,10 @@ public Response interpretationCancerTieringRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = CancerTieringInterpretationAnalysisParams.DESCRIPTION, required = true) CancerTieringInterpretationAnalysisParams params) { - return submitJob(CancerTieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(CancerTieringInterpretationAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index a62b08e0413..be6ec91c91f 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -16,7 +16,6 @@ package org.opencb.opencga.server.rest.analysis; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; @@ -28,12 +27,8 @@ import org.opencb.biodata.models.variant.metadata.VariantMetadata; import org.opencb.biodata.models.variant.metadata.VariantSetStats; import org.opencb.commons.datastore.core.*; -import org.opencb.opencga.analysis.AnalysisUtils; -import org.opencb.opencga.analysis.ResourceUtils; -import org.opencb.opencga.analysis.alignment.AlignmentConstants; import org.opencb.opencga.analysis.family.qc.FamilyQcAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcAnalysis; -import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.sample.qc.SampleQcAnalysis; import org.opencb.opencga.analysis.variant.VariantExportTool; import org.opencb.opencga.analysis.variant.circos.CircosAnalysis; @@ -56,13 +51,10 @@ import org.opencb.opencga.analysis.variant.stats.CohortVariantStatsAnalysis; import org.opencb.opencga.analysis.variant.stats.SampleVariantStatsAnalysis; import org.opencb.opencga.analysis.variant.stats.VariantStatsAnalysis; -import org.opencb.opencga.analysis.wrappers.deeptools.DeeptoolsWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.exomiser.ExomiserWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.gatk.GatkWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.rvtests.RvtestsWrapperAnalysis; -import org.opencb.opencga.analysis.wrappers.samtools.SamtoolsWrapperAnalysis; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.AvroToAnnotationConverter; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -72,15 +64,14 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.VersionException; -import org.opencb.opencga.core.models.alignment.DeeptoolsWrapperParams; -import org.opencb.opencga.core.models.alignment.SamtoolsWrapperParams; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByGene; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.operations.variant.VariantFileDeleteParams; +import org.opencb.opencga.core.models.operations.variant.VariantIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantStatsExportParams; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.variant.*; @@ -104,7 +95,6 @@ import java.util.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.SAVED_FILTER_DESCR; -import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.geneRegionIntersect; import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @@ -196,8 +186,11 @@ public Response variantFileIndex( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantIndexParams.DESCRIPTION, required = true) VariantIndexParams params) { - return submitJob(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @Deprecated @@ -209,11 +202,14 @@ public Response variantFileDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Files to remove") @QueryParam("file") String file, @ApiParam(value = "Resume a previously failed indexation") @QueryParam("resume") boolean resume) throws WebServiceException { VariantFileDeleteParams params = new VariantFileDeleteParams(getIdList(file), resume); - return submitJob(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -408,8 +404,11 @@ public Response export( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantExportParams.DESCRIPTION, required = true) VariantExportParams params) { - return submitJob(VariantExportTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantExportTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -451,8 +450,11 @@ public Response statsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsAnalysisParams.DESCRIPTION, required = true) VariantStatsAnalysisParams params) { - return submitJob(VariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -465,8 +467,11 @@ public Response statsExport( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsExportParams.DESCRIPTION, required = true) VariantStatsExportParams params) { - return submitJob("variant-stats-export", project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob("variant-stats-export", project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } // public static class StatsDeleteParams extends ToolParams { @@ -568,8 +573,11 @@ public Response sampleRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleVariantFilterParams.DESCRIPTION, required = true) SampleVariantFilterParams params) { - return submitJob(SampleVariantFilterAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleVariantFilterAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -581,8 +589,11 @@ public Response sampleEligibility( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleEligibilityAnalysisParams.DESCRIPTION, required = true) SampleEligibilityAnalysisParams params) { - return submitJob(SampleEligibilityAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleEligibilityAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -594,8 +605,11 @@ public Response exomiserRun( @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ExomiserWrapperParams.DESCRIPTION, required = true) ExomiserWrapperParams params) { - return submitJob(ExomiserWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(ExomiserWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -689,8 +703,11 @@ public Response sampleStatsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleVariantStatsAnalysisParams.DESCRIPTION, required = true) SampleVariantStatsAnalysisParams params) { - return submitJob(SampleVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -738,8 +755,11 @@ public Response cohortStatsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = CohortVariantStatsAnalysisParams.DESCRIPTION, required = true) CohortVariantStatsAnalysisParams params) { - return submitJob(CohortVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(CohortVariantStatsAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -911,8 +931,11 @@ public Response gwasRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = GwasAnalysisParams.DESCRIPTION, required = true) GwasAnalysisParams params) { - return submitJob(GwasAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(GwasAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } // @POST @@ -938,8 +961,11 @@ public Response mutationalSignatureRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = MutationalSignatureAnalysisParams.DESCRIPTION, required = true) MutationalSignatureAnalysisParams params) { - return submitJob(MutationalSignatureAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(MutationalSignatureAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -1009,7 +1035,7 @@ public Response mutationalSignatureQuery( MutationalSignatureAnalysis mutationalSignatureAnalysis = new MutationalSignatureAnalysis(); mutationalSignatureAnalysis.setUp(opencgaHome.toString(), catalogManager, storageEngineFactory, new ObjectMap(), outDir, null, - token); + false, token); mutationalSignatureAnalysis.setStudy(query.getString(STUDY.key())); mutationalSignatureAnalysis.setSignatureParams(params); @@ -1053,8 +1079,11 @@ public Response hrDetectRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = HRDetectAnalysisParams.DESCRIPTION, required = true) HRDetectAnalysisParams params) { - return submitJob(HRDetectAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(HRDetectAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1066,8 +1095,11 @@ public Response mendelianErrorRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = MendelianErrorAnalysisParams.DESCRIPTION, required = true) MendelianErrorAnalysisParams params) { - return submitJob(MendelianErrorAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(MendelianErrorAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1079,13 +1111,16 @@ public Response inferredSexRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = InferredSexAnalysisParams.DESCRIPTION, required = true) InferredSexAnalysisParams params) { return run(() -> { // Check before submitting the job InferredSexAnalysis.checkParameters(params.getIndividual(), params.getSample(), study, catalogManager, token); // Submit the inferred sex analysis - return submitJobRaw(InferredSexAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobRaw(InferredSexAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); }); } @@ -1098,8 +1133,11 @@ public Response relatednessRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = RelatednessAnalysisParams.DESCRIPTION, required = true) RelatednessAnalysisParams params) { - return submitJob(RelatednessAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(RelatednessAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1111,8 +1149,11 @@ public Response familyQcRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = FamilyQcAnalysisParams.DESCRIPTION, required = true) FamilyQcAnalysisParams params) { - return submitJob(FamilyQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(FamilyQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1124,6 +1165,9 @@ public Response individualQcRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = IndividualQcAnalysisParams.DESCRIPTION, required = true) IndividualQcAnalysisParams params) { return run(() -> { // Check before submitting the job @@ -1131,7 +1175,7 @@ public Response individualQcRun( catalogManager, token); // Submit the individual QC analysis - return submitJobRaw(IndividualQcAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobRaw(IndividualQcAnalysis.ID, null, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); }); } @@ -1144,9 +1188,12 @@ public Response sampleQcRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = SampleQcAnalysisParams.DESCRIPTION, required = true) SampleQcAnalysisParams params) { - return submitJob(SampleQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(SampleQcAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1158,8 +1205,11 @@ public Response plinkRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = PlinkWrapperParams.DESCRIPTION, required = true) PlinkWrapperParams params) { - return submitJob(PlinkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(PlinkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1171,8 +1221,11 @@ public Response rvtestsRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = RvtestsWrapperParams.DESCRIPTION, required = true) RvtestsWrapperParams params) { - return submitJob(RvtestsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(RvtestsWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1184,8 +1237,11 @@ public Response gatkRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = GatkWrapperParams.DESCRIPTION, required = true) GatkWrapperParams params) { - return submitJob(GatkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(GatkWrapperAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -1197,8 +1253,11 @@ public Response knockoutRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = KnockoutAnalysisParams.DESCRIPTION, required = true) KnockoutAnalysisParams params) { - return submitJob(KnockoutAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(KnockoutAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @GET @@ -1238,10 +1297,13 @@ public Response genomePlotRun( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = GenomePlotAnalysisParams.DESCRIPTION, required = true) GenomePlotAnalysisParams params) { // To be sure: do not update quality control sample params.setSample(null); - return submitJob(GenomePlotAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(GenomePlotAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java index 77574d73ad9..a2143f45090 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/operations/VariantOperationWebService.java @@ -109,9 +109,12 @@ public Response variantIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantIndexParams.DESCRIPTION) VariantIndexParams params) { - return submitOperation(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -122,9 +125,12 @@ public Response variantDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFileDeleteParams.DESCRIPTION) VariantFileDeleteParams params) { - return submitOperation(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantFileDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -135,9 +141,12 @@ public Response variantStudyDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantStudyDeleteParams.DESCRIPTION) VariantStudyDeleteParams params) { - return submitOperation(VariantStudyDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantStudyDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -148,9 +157,12 @@ public Response variantIndexLauncher( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFileIndexJobLauncherParams.DESCRIPTION) VariantFileIndexJobLauncherParams params) { - return submitOperation(VariantFileIndexJobLauncherTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantFileIndexJobLauncherTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -161,9 +173,12 @@ public Response variantMetadataSynchronize( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantStorageMetadataSynchronizeParams.DESCRIPTION) VariantStorageMetadataSynchronizeParams params) { - return submitJobAdmin(VariantStorageMetadataSynchronizeOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobAdmin(VariantStorageMetadataSynchronizeOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -174,8 +189,11 @@ public Response variantMetadataRepair( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStorageMetadataRepairToolParams.DESCRIPTION) VariantStorageMetadataRepairToolParams params) { - return submitJobAdmin(VariantStorageMetadataRepairTool.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitJobAdmin(VariantStorageMetadataRepairTool.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -187,8 +205,11 @@ public Response statsIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsAnalysisParams.DESCRIPTION, required = true) VariantStatsIndexParams params) { - return submitJob(VariantStatsIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantStatsIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -200,8 +221,11 @@ public Response statsDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantStatsDeleteParams.DESCRIPTION, required = true) VariantStatsDeleteParams params) { - return submitJob(VariantStatsDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitJob(VariantStatsDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -214,10 +238,13 @@ public Response secondaryIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondaryAnnotationIndexParams.DESCRIPTION) VariantSecondaryAnnotationIndexParams params) { - return variantSecondaryAnnotationIndex(jobName, jobDescription, dependsOn, jobTags, project, study, params); + return variantSecondaryAnnotationIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, project, study, params); } @POST @@ -228,10 +255,13 @@ public Response variantSecondaryAnnotationIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondaryAnnotationIndexParams.DESCRIPTION) VariantSecondaryAnnotationIndexParams params) { - return submitOperation(VariantSecondaryAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSecondaryAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -242,12 +272,15 @@ public Response secondaryIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Samples to remove. Needs to provide all the samples in the secondary index.") @QueryParam("samples") String samples) { HashMap params = new HashMap<>(); params.put(ParamConstants.STUDY_PARAM, study); params.put("samples", samples); - return submitOperation(VariantSecondaryIndexSamplesDeleteOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSecondaryIndexSamplesDeleteOperationTool.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -258,10 +291,13 @@ public Response annotation( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.PROJECT_DESCRIPTION) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantAnnotationIndexParams.DESCRIPTION) VariantAnnotationIndexParams params) { - return submitOperation(VariantAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantAnnotationIndexOperationTool.ID, project, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -272,6 +308,9 @@ public Response annotationDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = "Annotation identifier") @QueryParam("annotationId") String annotationId ) { @@ -279,7 +318,7 @@ public Response annotationDelete( params.put(ParamConstants.PROJECT_PARAM, project); params.put("annotationId", annotationId); return submitOperationToProject(VariantAnnotationDeleteOperationTool.ID, project, params, jobName, jobDescription, dependsOn, - jobTags); + jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -290,10 +329,13 @@ public Response annotationSave( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantCatalogQueryUtils.PROJECT_DESC) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = VariantAnnotationSaveParams.DESCRIPTION) VariantAnnotationSaveParams params) { return submitOperationToProject(VariantAnnotationSaveOperationTool.ID, project, params, jobName, jobDescription, dependsOn, - jobTags); + jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -304,9 +346,12 @@ public Response scoreIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantScoreIndexParams.DESCRIPTION) VariantScoreIndexParams params) { - return submitOperation(VariantScoreIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantScoreIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @DELETE @@ -317,6 +362,9 @@ public Response scoreDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Unique name of the score within the study") @QueryParam("name") String name, @ApiParam(value = "Resume a previously failed remove") @QueryParam("resume") boolean resume, @@ -327,7 +375,7 @@ public Response scoreDelete( params.put("name", name); if (resume) params.put("resume", ""); if (force) params.put("force", ""); - return submitOperation(VariantScoreDeleteParams.ID, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantScoreDeleteParams.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @Deprecated @@ -360,9 +408,12 @@ public Response sampleGenotypeIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondarySampleIndexParams.DESCRIPTION) VariantSecondarySampleIndexParams params) { - return sampleIndex(jobName, jobDescription, dependsOn, jobTags, study, params); + return sampleIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, study, params); } @Deprecated @@ -374,9 +425,12 @@ public Response sampleIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondarySampleIndexParams.DESCRIPTION) VariantSecondarySampleIndexParams params) { - return variantSecondarySampleIndex(jobName, jobDescription, dependsOn, jobTags, study, params); + return variantSecondarySampleIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, study, params); } @POST @@ -387,9 +441,12 @@ public Response variantSecondarySampleIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSecondarySampleIndexParams.DESCRIPTION) VariantSecondarySampleIndexParams params) { - return submitOperation(VariantSecondarySampleIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSecondarySampleIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -400,9 +457,12 @@ public Response variantSampleDelete( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantSampleDeleteParams.DESCRIPTION) VariantSampleDeleteParams params) { - return submitOperation(VariantSampleDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantSampleDeleteOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -414,9 +474,12 @@ public Response familyGenotypeIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFamilyIndexParams.DESCRIPTION) VariantFamilyIndexParams params) { - return familyIndex(jobName, jobDescription, dependsOn, jobTags, study, params); + return familyIndex(jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, study, params); } @Deprecated @@ -428,9 +491,12 @@ public Response familyIndex( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantFamilyIndexParams.DESCRIPTION) VariantFamilyIndexParams params) { - return submitOperation(VariantFamilyIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantFamilyIndexOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -441,9 +507,12 @@ public Response aggregateFamily( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantAggregateFamilyParams.DESCRIPTION) VariantAggregateFamilyParams params) { - return submitOperation(VariantAggregateFamilyOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantAggregateFamilyOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -454,9 +523,12 @@ public Response aggregate( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = VariantAggregateParams.DESCRIPTION) VariantAggregateParams params) { - return submitOperation(VariantAggregateOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperation(VariantAggregateOperationTool.ID, study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -467,9 +539,12 @@ public Response julie( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = ParamConstants.PROJECT_PARAM) @QueryParam(ParamConstants.PROJECT_PARAM) String project, @ApiParam(value = JulieParams.DESCRIPTION, required = true) JulieParams params) { - return submitOperationToProject(JulieTool.ID, project, params, jobName, jobDescription, dependsOn, jobTags); + return submitOperationToProject(JulieTool.ID, project, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @POST @@ -480,22 +555,25 @@ public Response variantPrune( @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, @ApiParam(value = VariantPruneParams.DESCRIPTION) VariantPruneParams params) { - return submitOperationToProject(VariantPruneOperationTool.ID, params.getProject(), params, jobName, jobDescription, dependsOn, jobTags); + return submitOperationToProject(VariantPruneOperationTool.ID, params.getProject(), params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } - public Response submitOperation(String toolId, String study, ToolParams params, - String jobName, String jobDescription, String jobDependsOn, String jobTags) { - return submitOperation(toolId, null, study, params, jobName, jobDescription, jobDependsOn, jobTags); + public Response submitOperation(String toolId, String study, ToolParams params, String jobName, String jobDescription, + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return submitOperation(toolId, null, study, params, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, dryRun); } public Response submitOperationToProject(String toolId, String project, ToolParams params, String jobName, String jobDescription, - String jobDependsOn, String jobTags) { - return submitOperation(toolId, project, null, params, jobName, jobDescription, jobDependsOn, jobTags); + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { + return submitOperation(toolId, project, null, params, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, dryRun); } public Response submitOperation(String toolId, String project, String study, ToolParams params, String jobName, String jobDescription, - String jobDependsOn, String jobTags) { + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { try { Map paramsMap = params.toParams(); if (StringUtils.isNotEmpty(study)) { @@ -504,26 +582,31 @@ public Response submitOperation(String toolId, String project, String study, Too if (StringUtils.isNotEmpty(project)) { paramsMap.put(ParamConstants.PROJECT_PARAM, project); } - return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags); + return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, + jobPriority, dryRun); } catch (Exception e) { return createErrorResponse(e); } } public Response submitOperationToProject(String toolId, String project, Map paramsMap, String jobName, - String jobDescription, String jobDependsOn, String jobTags) { - return submitOperation(toolId, project, null, paramsMap, jobName, jobDescription, jobDependsOn, jobTags); + String jobDescription, String jobDependsOn, String jobTags, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) { + return submitOperation(toolId, project, null, paramsMap, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, + jobPriority, dryRun); } public Response submitOperation(String toolId, Map paramsMap, String jobName, String jobDescription, - String jobDependsOn, String jobTags) { + String jobDependsOn, String jobTags, String jobScheduledStartTime, String jobPriority, Boolean dryRun) { String project = (String) paramsMap.get(ParamConstants.PROJECT_PARAM); String study = (String) paramsMap.get(ParamConstants.STUDY_PARAM); - return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags); + return submitOperation(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, + jobPriority, dryRun); } public Response submitOperation(String toolId, String project, String study, Map paramsMap, String jobName, - String jobDescription, String jobDependsOne, String jobTags) { + String jobDescription, String jobDependsOne, String jobTags, String jobScheduledStartTime, + String jobPriority, Boolean dryRun) { Map dynamicParamsMap = new HashMap<>(); for (String key : this.params.keySet()) { String prefix = "dynamic_"; @@ -534,6 +617,6 @@ public Response submitOperation(String toolId, String project, String study, Map if (dynamicParamsMap.size() > 0) { paramsMap.put("dynamicParams", dynamicParamsMap); } - return submitJob(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags); + return submitJob(toolId, project, study, paramsMap, jobName, jobDescription, jobDependsOne, jobTags, jobScheduledStartTime, jobPriority, dryRun); } } diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 1834d5c7e64..df69dcc79dd 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index 02953389c2d..12400612512 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index a7a2c8adbbb..0a392fe9020 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml @@ -188,6 +188,12 @@ com.databricks SnpEff + + + distlib + distlib + + diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/AlignmentStorageOptions.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/AlignmentStorageOptions.java index 672a7f03bca..a137d4c4e0f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/AlignmentStorageOptions.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/AlignmentStorageOptions.java @@ -1,11 +1,11 @@ package org.opencb.opencga.storage.core.alignment; -import org.opencb.biodata.tools.alignment.BamManager; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.ConfigurationOption; public enum AlignmentStorageOptions implements ConfigurationOption { - BIG_WIG_WINDOWS_SIZE("bigWigWindowsSize", BamManager.DEFAULT_WINDOW_SIZE); + BIG_WIG_WINDOWS_SIZE("bigWigWindowsSize", ParamConstants.COVERAGE_WINDOW_SIZE_DEFAULT); private final String key; private final Object value; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/local/LocalAlignmentStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/local/LocalAlignmentStoragePipeline.java index 009718fcbfb..d45ccbc130c 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/local/LocalAlignmentStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/alignment/local/LocalAlignmentStoragePipeline.java @@ -87,7 +87,7 @@ public URI transform(URI input, URI pedigree, URI output) throws Exception { // 3) Create the BigWig file containing the coverage using the bamCoverage from the DeepTools package Path bwPath = workspace.resolve(path.getFileName() + BamManager.COVERAGE_BIGWIG_EXTENSION); int windowSize = configuration.getInt(AlignmentStorageOptions.BIG_WIG_WINDOWS_SIZE.key(), - AlignmentStorageOptions.BIG_WIG_WINDOWS_SIZE.defaultValue()); + Integer.parseInt(AlignmentStorageOptions.BIG_WIG_WINDOWS_SIZE.defaultValue())); bamManager.calculateBigWigCoverage(bwPath, windowSize); return input; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/exceptions/StorageEngineException.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/exceptions/StorageEngineException.java index 547b83c55af..0ac128118a9 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/exceptions/StorageEngineException.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/exceptions/StorageEngineException.java @@ -41,6 +41,11 @@ public static StorageEngineException alreadyLoaded(int fileId, String fileName) return unableToExecute("Already loaded", fileId, fileName); } + public static StorageEngineException invalidFileStatus(int fileId, String fileName) { + return unableToExecute("File is in INVALID status. Unable to load. File needs to be deleted from the variant-storage", + fileId, fileName); + } + public static StorageEngineException otherOperationInProgressException(TaskMetadata operation, String jobOperationName, List fileIds, VariantStorageMetadataManager mm) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 103e8568bce..680d30a20db 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -679,6 +679,7 @@ public FileMetadata updateFileMetadata(int studyId, int fi update.update(fileMetadata); lock.checkLocked(); unsecureUpdateFileMetadata(studyId, fileMetadata); + fileIdIndexedCache.put(studyId, fileId, fileMetadata.isIndexed()); return fileMetadata; } finally { lock.unlock(); @@ -807,7 +808,6 @@ public void addIndexedFiles(int studyId, List fileIds) throws StorageEn .getName(); logger.info("Register file " + name + " as INDEXED"); } - fileDBAdaptor.addIndexedFiles(studyId, fileIds); fileIdsFromSampleIdCache.clear(); fileIdIndexedCache.clear(); } @@ -821,6 +821,7 @@ public void removeIndexedFiles(int studyId, Collection fileIds) throws fileMetadata.setIndexStatus(TaskMetadata.Status.NONE); fileMetadata.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); fileMetadata.setAnnotationStatus(TaskMetadata.Status.NONE); + fileMetadata.getAttributes().remove(LOAD_ARCHIVE.key()); if (fileMetadata.getType() == FileMetadata.Type.VIRTUAL) { partialFiles.addAll(fileMetadata.getAttributes().getAsIntegerList(FileMetadata.VIRTUAL_FILES)); } @@ -833,6 +834,7 @@ public void removeIndexedFiles(int studyId, Collection fileIds) throws fileMetadata.setIndexStatus(TaskMetadata.Status.NONE); fileMetadata.setSecondaryAnnotationIndexStatus(TaskMetadata.Status.NONE); fileMetadata.setAnnotationStatus(TaskMetadata.Status.NONE); + fileMetadata.getAttributes().remove(LOAD_ARCHIVE.key()); }); // deleteVariantFileMetadata(studyId, fileId); } @@ -851,7 +853,6 @@ public void removeIndexedFiles(int studyId, Collection fileIds) throws } }); } - fileDBAdaptor.removeIndexedFiles(studyId, fileIds); } public Iterable fileMetadataIterable(int studyId) { @@ -1135,6 +1136,16 @@ public Iterable getInvalidCohorts(int studyId) { return () -> Iterators.filter(cohortIterator(studyId), CohortMetadata::isInvalid); } + public Iterable getCalculatedOrPartialCohorts(int studyId) { + return () -> Iterators.filter(cohortIterator(studyId), + cohortMetadata -> { + TaskMetadata.Status status = cohortMetadata.getStatsStatus(); + return status == TaskMetadata.Status.READY + || status == TaskMetadata.Status.RUNNING + || status == TaskMetadata.Status.ERROR; + }); + } + public CohortMetadata setSamplesToCohort(int studyId, String cohortName, Collection samples) throws StorageEngineException { return updateCohortSamples(studyId, cohortName, samples, false); } @@ -1244,6 +1255,7 @@ private CohortMetadata updateCohortSamples(int studyId, String cohortName, Colle if (!oldSamples.equals(sampleIdsList) || !oldFiles.equals(fileIds)) { // Cohort has been modified! Invalidate stats cohort.setStatsStatus(TaskMetadata.Status.ERROR); + cohort.getAttributes().put(CohortMetadata.INVALID_STATS_NUM_SAMPLES, oldSamples.size()); } } } @@ -1272,14 +1284,22 @@ public TaskMetadata getTask(int studyId, String taskName, List fileIds) return task; } + public Iterable taskIterable(int studyId) { + return () -> taskIterator(studyId, null, false); + } + public Iterator taskIterator(int studyId) { - return taskIterator(studyId, null); + return taskIterator(studyId, null, false); } public Iterator taskIterator(int studyId, List statusFilter) { return taskIterator(studyId, statusFilter, false); } + public Iterator taskIterator(int studyId, TaskMetadata.Status... statusFilter) { + return taskIterator(studyId, Arrays.asList(statusFilter), false); + } + public Iterator taskIterator(int studyId, List statusFilter, boolean reversed) { return taskDBAdaptor.taskIterator(studyId, statusFilter, reversed); } @@ -1288,6 +1308,13 @@ public Iterable getRunningTasks(int studyId) { return taskDBAdaptor.getRunningTasks(studyId); } + public Iterable getActiveTasks(int studyId) { + return () -> taskIterator(studyId, + TaskMetadata.Status.RUNNING, + TaskMetadata.Status.DONE, + TaskMetadata.Status.ERROR); + } + public void unsecureUpdateTask(int studyId, TaskMetadata task) throws StorageEngineException { task.setStudyId(studyId); if (task.getId() == 0) { @@ -1735,6 +1762,9 @@ private int registerFile(int studyId, String filePath, FileMetadata.Type type) t if (fileId != null) { updateFileMetadata(studyId, fileId, fileMetadata -> { + if (fileMetadata.getIndexStatus() == TaskMetadata.Status.INVALID) { + throw StorageEngineException.invalidFileStatus(fileMetadata.getId(), fileName); + } if (fileMetadata.isIndexed()) { throw StorageEngineException.alreadyLoaded(fileMetadata.getId(), fileName); } @@ -2072,47 +2102,35 @@ private TaskMetadata getRunningTaskCompatibleOrFail(int studyId, String jobOpera TaskMetadata.Type type, Predicate allowConcurrent) throws StorageEngineException { TaskMetadata resumeTask = null; - Iterator iterator = taskIterator(studyId, Arrays.asList( - TaskMetadata.Status.DONE, - TaskMetadata.Status.RUNNING, - TaskMetadata.Status.ERROR)); - while (iterator.hasNext()) { - TaskMetadata task = iterator.next(); + for (TaskMetadata task : getActiveTasks(studyId)) { TaskMetadata.Status currentStatus = task.currentStatus(); - switch (currentStatus) { - case READY: - logger.warn("Unexpected READY task. IGNORE"); - // Ignore ready operations - break; - case DONE: - case RUNNING: - if (!resume) { - if (task.sameOperation(fileIds, type, jobOperationName)) { - throw StorageEngineException.currentOperationInProgressException(task, this); - } else { - if (allowConcurrent.test(task)) { - break; - } else { - throw StorageEngineException.otherOperationInProgressException(task, jobOperationName, fileIds, this); - } - } - } - // DO NOT BREAK!. Resuming last loading, go to error case. - case ERROR: - if (!task.sameOperation(fileIds, type, jobOperationName)) { - if (allowConcurrent.test(task)) { - break; - } else { - throw StorageEngineException.otherOperationInProgressException(task, jobOperationName, fileIds, this, resume); - } - } else { - logger.info("Resuming last batch operation \"" + task.getName() + "\" due to error."); - resumeTask = task; - } - break; - default: - throw new IllegalArgumentException("Unknown Status " + currentStatus); + if (currentStatus != TaskMetadata.Status.DONE + && currentStatus != TaskMetadata.Status.RUNNING + && currentStatus != TaskMetadata.Status.ERROR) { + logger.warn("Unexpected {} task. IGNORE", currentStatus); + // Ignore ready operations + continue; + } + + if (task.sameOperation(fileIds, type, jobOperationName)) { + if (currentStatus == TaskMetadata.Status.ERROR) { + // Automatically resume ERROR status tasks + logger.info("Resuming last batch operation \"" + task.getName() + "\" due to error."); + resumeTask = task; + } else if (resume) { + // Force resume + logger.info("Manually resuming last batch operation \"" + task.getName() + "\" in status " + currentStatus + "."); + resumeTask = task; + } else { + // Already being executed + throw StorageEngineException.currentOperationInProgressException(task, this); + } + } else { + // Check if it can be executed concurrently + if (!allowConcurrent.test(task)) { + throw StorageEngineException.otherOperationInProgressException(task, jobOperationName, fileIds, this); + } } } return resumeTask; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/FileMetadataDBAdaptor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/FileMetadataDBAdaptor.java index ca6a5964b59..8cb916df1c6 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/FileMetadataDBAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/adaptors/FileMetadataDBAdaptor.java @@ -75,10 +75,6 @@ public String description() { LinkedHashSet getIndexedFiles(int studyId, boolean includePartial); - default void addIndexedFiles(int studyId, List fileIds) {} - - default void removeIndexedFiles(int studyId, Collection fileIds) {}; - default DataResult count() { return count(new Query()); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/CohortMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/CohortMetadata.java index 4814fc82b63..a4e9de4116e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/CohortMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/CohortMetadata.java @@ -8,6 +8,7 @@ * @author Jacobo Coll <jacobo167@gmail.com> */ public class CohortMetadata extends StudyResourceMetadata { + public static final String INVALID_STATS_NUM_SAMPLES = "invalidStatsNumSamples"; // private int studyId; // private int id; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/TaskMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/TaskMetadata.java index e7d32e12b5d..c0d0cf9376c 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/TaskMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/TaskMetadata.java @@ -34,10 +34,39 @@ public class TaskMetadata { public enum Status { NONE, + /** + * Active task. + * Running, but not finished + */ RUNNING, - DONE, // Finished, but some work still needed (optional) + /** + * Active task. + * Finished, but some work still needed (optional status) + */ + DONE, + /** + * Active task. + * Currently, paused. + * Errors found during the execution. Needs to be resumed or cleaned. + */ + ERROR, + /** + * Finished. + * Ready to be used + */ READY, - ERROR + /** + * Finished. + * Task was aborted, cancelled or rolled back. + * Any needed clean might be executed by other running tasks + */ + ABORTED, + /** + * Finished. + * Task finished with invalid results. + * Similar to "ERROR" status, but this can't be resumed. Needs to be cleaned first. + */ + INVALID, } public enum Type { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java index ed0848a0e0b..f8dfc2d1f9f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java @@ -343,6 +343,7 @@ public List getVariants(List variantsStr) { List variants = new ArrayList<>(variantsStr.size()); List> response = null; try { + // FIXME: This method should call genomic/variant/snp/search response = checkNulls(cellBaseClient.getVariantClient().get(variantsStr, new QueryOptions(QueryOptions.INCLUDE, VariantField.CHROMOSOME.fieldName() + "," diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index a3ab7e539b5..dd24ee1334d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -31,7 +31,7 @@ import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngine; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -946,35 +946,78 @@ public void removeSamples(String study, List samples, URI outdir) throws */ protected TaskMetadata preRemove(String study, List files, List samples) throws StorageEngineException { AtomicReference batchFileOperation = new AtomicReference<>(); + boolean force = getOptions().getBoolean(FORCE.key(), FORCE.defaultValue()); + boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); VariantStorageMetadataManager metadataManager = getMetadataManager(); - metadataManager.updateStudyMetadata(study, studyMetadata -> { - List fileIds = new ArrayList<>(files.size()); - for (String file : files) { - FileMetadata fileMetadata = metadataManager.getFileMetadata(studyMetadata.getId(), file); - if (fileMetadata == null) { - throw VariantQueryException.fileNotFound(file, study); - } - if (fileMetadata.getType() == FileMetadata.Type.PARTIAL) { - String virtualFileName = metadataManager.getFileName( - studyMetadata.getId(), - fileMetadata.getAttributes().getInt(FileMetadata.VIRTUAL_PARENT)); - throw new StorageEngineException("Unable to remove " + FileMetadata.Type.PARTIAL + " file. " - + "Try removing its virtual file : '" + virtualFileName + "'"); - } - fileIds.add(fileMetadata.getId()); - if (!fileMetadata.isIndexed()) { + + int studyId = metadataManager.getStudyId(study); + List fileIds = new ArrayList<>(files.size()); + for (String file : files) { + FileMetadata fileMetadata = metadataManager.getFileMetadata(studyId, file); + if (fileMetadata == null) { + throw VariantQueryException.fileNotFound(file, study); + } + if (fileMetadata.getType() == FileMetadata.Type.PARTIAL) { + String virtualFileName = metadataManager.getFileName( + studyId, + fileMetadata.getAttributes().getInt(FileMetadata.VIRTUAL_PARENT)); + throw new StorageEngineException("Unable to remove " + FileMetadata.Type.PARTIAL + " file. " + + "Try removing its virtual file : '" + virtualFileName + "'"); + } + if (fileMetadata.getIndexStatus() == TaskMetadata.Status.NONE) { + if (force) { + logger.info("Force remove. Removing non indexed file: " + fileMetadata.getName()); + } else { throw new StorageEngineException("Unable to remove non indexed file: " + fileMetadata.getName()); } } + if (fileMetadata.getIndexStatus() == TaskMetadata.Status.RUNNING) { + if (force) { + logger.info("Force remove. Removing file while being indexed: " + fileMetadata.getName()); + } else { + throw new StorageEngineException("Unable to remove file while being indexed: " + fileMetadata.getName()); + } + } + fileIds.add(fileMetadata.getId()); + } - boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); + metadataManager.updateStudyMetadata(study, studyMetadata -> { + List tasksToAbort = new ArrayList<>(); + HashSet fileIdsSet = new HashSet<>(fileIds); batchFileOperation.set(metadataManager.addRunningTask( studyMetadata.getId(), REMOVE_OPERATION_NAME, fileIds, resume, - TaskMetadata.Type.REMOVE)); + TaskMetadata.Type.REMOVE, runningTask -> { + if (runningTask.getType() == TaskMetadata.Type.LOAD) { + if (force && fileIdsSet.containsAll(runningTask.getFileIds())) { + // Abort running load task as all files are being removed + tasksToAbort.add(runningTask); + return true; + } else { + if (force) { + logger.error("Unable to force remove the file while other files are being loaded."); + } + // Unable to remove files. Other files are being loaded. + return false; + } + } else { + // Unable to remove files. Other tasks are running. + return false; + } + })); + // If the task was successfully created, abort other tasks. + for (TaskMetadata task : tasksToAbort) { + List fileNames = task.getFileIds().stream() + .map(fileId -> metadataManager.getFileName(studyMetadata.getId(), fileId)) + .collect(Collectors.toList()); + TaskMetadata.Status status = TaskMetadata.Status.ABORTED; + logger.info("Force remove. Setting status to " + status + " of running task " + + task.getName() + "('id=" + task.getId() + ") for files " + fileNames); + metadataManager.setStatus(studyMetadata.getId(), task.getId(), status); + } return studyMetadata; }); @@ -1192,8 +1235,36 @@ public VariantQueryResult getCompoundHeterozygous(String study, String return get(query, options); } - public DataResult getSampleData(String variant, String study, QueryOptions options) throws StorageEngineException { - return new VariantSampleDataManager(getDBAdaptor()).getSampleData(variant, study, options); + public DataResult getSampleData(String variantStr, String study, QueryOptions options) throws StorageEngineException { + final Variant variant = getVariant(variantStr); + return getVariantSampleDataManager().getSampleData(variant, study, options); + } + + public Variant getVariant(String variantStr) { + final Variant variant; + if (VariantQueryUtils.isVariantId(variantStr)) { + variant = VariantQueryUtils.toVariant(variantStr, true); + } else if (VariantQueryUtils.isVariantAccession(variantStr)) { + VariantQueryResult result = get(new Query(VariantQueryParam.ANNOT_XREF.key(), variantStr), + new QueryOptions(QueryOptions.INCLUDE, VariantField.ID).append(QueryOptions.LIMIT, 1).append(QueryOptions.COUNT, true)); + if (result.getNumMatches() > 1) { + throw new VariantQueryException("Not unique variant identifier '" + variantStr + "'." + + " Found " + result.getNumMatches() + " results"); + } else if (result.getNumResults() == 1) { + variant = result.first(); + } else { + throw VariantQueryException.variantNotFound(variantStr); + } + } else { + throw new VariantQueryException("Variant not valid. Variant = '" + variantStr + "'. Supported values:" + + " {chr}:{start}:{end}:{ref}:{alt}, rs{id}"); + } + variant.setId(variant.toString()); + return variant; + } + + protected VariantSampleDataManager getVariantSampleDataManager() throws StorageEngineException { + return new VariantSampleDataManager(getDBAdaptor()); } public VariantQueryResult get(Query query, QueryOptions options) { @@ -1205,8 +1276,8 @@ public VariantQueryResult get(Query query, QueryOptions options) { } addDefaultLimit(options, getOptions()); addDefaultSampleLimit(query, getOptions()); - query = preProcessQuery(query, options); - return getVariantQueryExecutor(query, options).get(query, options); + ParsedVariantQuery variantQuery = parseQuery(query, options); + return getVariantQueryExecutor(variantQuery).get(variantQuery); } @Override @@ -1223,8 +1294,8 @@ public MultiVariantDBIterator iterator(Iterator variants, Query query, QueryO public VariantDBIterator iterator(Query query, QueryOptions options) { query = VariantQueryUtils.copy(query); options = VariantQueryUtils.copy(options); - query = preProcessQuery(query, options); - return getVariantQueryExecutor(query, options).iterator(query, options); + ParsedVariantQuery variantQuery = parseQuery(query, options); + return getVariantQueryExecutor(variantQuery).iterator(variantQuery); } public final List getVariantQueryExecutors() throws StorageEngineException { @@ -1246,7 +1317,7 @@ protected List initVariantQueryExecutors() throws StorageE executors.add(new CompoundHeterozygousQueryExecutor( getMetadataManager(), getStorageEngineId(), getOptions(), this)); executors.add(new BreakendVariantQueryExecutor( - getMetadataManager(), getStorageEngineId(), getOptions(), new DBAdaptorVariantQueryExecutor( + getStorageEngineId(), getOptions(), new DBAdaptorVariantQueryExecutor( getDBAdaptor(), getStorageEngineId(), getOptions()), getDBAdaptor())); executors.add(new SamplesSearchIndexVariantQueryExecutor( getDBAdaptor(), getVariantSearchManager(), getStorageEngineId(), dbName, configuration, getOptions())); @@ -1265,12 +1336,22 @@ protected List initVariantQueryExecutors() throws StorageE * @return VariantQueryExecutor to use */ public VariantQueryExecutor getVariantQueryExecutor(Query query, QueryOptions options) { + return getVariantQueryExecutor(parseQuery(query, options)); + } + + /** + * Determine which {@link VariantQueryExecutor} should be used to execute the given query. + * + * @param variantQuery Parsed variant query + * @return VariantQueryExecutor to use + */ + public VariantQueryExecutor getVariantQueryExecutor(ParsedVariantQuery variantQuery) { try { for (VariantQueryExecutor executor : getVariantQueryExecutors()) { - if (executor.canUseThisExecutor(query, options)) { + if (executor.canUseThisExecutor(variantQuery.getQuery(), variantQuery.getInputOptions())) { logger.info("Using VariantQueryExecutor : " + executor.getClass().getName()); - logger.info(" Query : " + VariantQueryUtils.printQuery(query)); - logger.info(" Options : " + options.toJson()); + logger.info(" Query : " + VariantQueryUtils.printQuery(variantQuery.getInputQuery())); + logger.info(" Options : " + variantQuery.getInputOptions().toJson()); return executor; } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 19b0631659f..5b37d2512b1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -61,6 +61,7 @@ import org.opencb.opencga.storage.core.io.plain.StringDataWriter; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; +import org.opencb.opencga.storage.core.metadata.models.FileMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -176,9 +177,14 @@ public URI preTransform(URI input) throws StorageEngineException, IOException, F Integer fileId = getMetadataManager().getFileId(studyMetadata.getId(), fileName, true); if (fileId != null) { // File is indexed. Mark as non indexed. - getMetadataManager().updateFileMetadata(studyMetadata.getId(), fileId, fileMetadata -> { - fileMetadata.setIndexStatus(TaskMetadata.Status.NONE); - }); + FileMetadata fileMetadata = getMetadataManager().getFileMetadata(studyMetadata.getId(), fileId); + if (fileMetadata.getIndexStatus() == TaskMetadata.Status.INVALID) { + throw StorageEngineException.invalidFileStatus(fileId, fileName); + } else if (fileMetadata.getIndexStatus() != TaskMetadata.Status.NONE) { + getMetadataManager().updateFileMetadata(studyMetadata.getId(), fileId, fm -> { + fm.setIndexStatus(TaskMetadata.Status.NONE); + }); + } logger.info("File '{}' already loaded. Force reload!", fileName); } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptor.java index c2ee42c9b00..e54efca912a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptor.java @@ -23,12 +23,12 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; import org.opencb.opencga.storage.core.variant.stats.VariantStatsWrapper; @@ -37,8 +37,6 @@ import java.util.List; import java.util.Map; -import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.addSamplesMetadataIfRequested; - /** * @author Ignacio Medina * @author Jacobo Coll @@ -46,21 +44,6 @@ */ public interface VariantDBAdaptor extends VariantIterable, AutoCloseable { - /** - * This method inserts Variants into the given Study. If the Study already exists then it just adds the new Sample - * genotypes, also new variants are inserted. If it is a new Study then Sample genotypes are added to the new Study. - * - * @param variants List of variants in OpenCB data model to be inserted - * @param studyName Name or alias of the study - * @param options Query modifiers, accepted values are: include, exclude, limit, skip, sort and count - * @return A DataResult with the number of inserted variants - */ - @Deprecated - default DataResult insert(List variants, String studyName, QueryOptions options) { - throw new UnsupportedOperationException(); - } - - /** /** * Fetch all variants resulting of executing the query in the database. Returned fields are taken from * the 'include' and 'exclude' fields at options. @@ -71,26 +54,29 @@ default DataResult insert(List variants, String studyName, QueryOptions * @return A DataResult with the result of the query */ default VariantQueryResult get(Iterator variants, Query query, QueryOptions options) { - DataResult queryResult = iterator(variants, query, options).toDataResult(); - return addSamplesMetadataIfRequested(queryResult, query, options, getMetadataManager()); + ParsedVariantQuery variantQuery = new VariantQueryParser(null, getMetadataManager()).parseQuery(query, options, true); + try (VariantDBIterator iterator = iterator(variants, query, options)) { + return iterator.toDataResult(variantQuery); + } catch (Exception e) { + throw VariantQueryException.internalException(e); + } } @Deprecated default VariantDBIterator iterator(Query query, QueryOptions options) { - return iterator(new VariantQueryParser(null, getMetadataManager()).parseQuery(query, options, true), options); + return iterator(new VariantQueryParser(null, getMetadataManager()).parseQuery(query, options, true)); } - VariantDBIterator iterator(ParsedVariantQuery query, QueryOptions options); + VariantDBIterator iterator(ParsedVariantQuery query); /** * Fetch all variants resulting of executing the query in the database. Returned fields are taken from * the 'include' and 'exclude' fields at options. * * @param query Query to be executed in the database to filter variants - * @param options Query modifiers, accepted values are: include, exclude, limit, skip, sort and count * @return A DataResult with the result of the query */ - VariantQueryResult get(ParsedVariantQuery query, QueryOptions options); + VariantQueryResult get(ParsedVariantQuery query); /** * Fetch all variants resulting of executing the query in the database. Returned fields are taken from @@ -102,7 +88,7 @@ default VariantDBIterator iterator(Query query, QueryOptions options) { */ @Deprecated default VariantQueryResult get(Query query, QueryOptions options) { - return get(new VariantQueryParser(null, getMetadataManager()).parseQuery(query, options, true), options); + return get(new VariantQueryParser(null, getMetadataManager()).parseQuery(query, options, true)); } /** diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQuery.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQuery.java index 65c2bda05e6..09b1867854e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQuery.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQuery.java @@ -50,6 +50,10 @@ public VariantQuery region(String value) { put(VariantQueryParam.REGION.key(), value); return this; } + public VariantQuery region(String... value) { + put(VariantQueryParam.REGION.key(), Arrays.asList(value)); + return this; + } public VariantQuery region(Region... value) { put(VariantQueryParam.REGION.key(), Arrays.asList(value)); return this; @@ -105,6 +109,15 @@ public VariantQuery sample(String value) { return sample(VariantQueryUtils.QueryOperation.OR, value); } + public VariantQuery sample(List value) { + return sample(VariantQueryUtils.QueryOperation.OR, value); + } + + public VariantQuery sample(VariantQueryUtils.QueryOperation operation, List value) { + put(VariantQueryParam.SAMPLE.key(), value.stream().collect(Collectors.joining(operation.separator()))); + return this; + } + public VariantQuery sample(String... value) { return sample(VariantQueryUtils.QueryOperation.OR, value); } @@ -178,8 +191,9 @@ public VariantQuery includeSampleId(boolean value) { put(VariantQueryParam.INCLUDE_SAMPLE_ID.key(), value); return this; } - public String includeSampleId() { - return getString(VariantQueryParam.INCLUDE_SAMPLE_ID.key()); + + public boolean includeSampleId() { + return getBoolean(VariantQueryParam.INCLUDE_SAMPLE_ID.key()); } public VariantQuery sampleMetadata(boolean value) { @@ -212,20 +226,24 @@ public boolean includeGenotype() { return getBoolean(VariantQueryParam.INCLUDE_GENOTYPE.key()); } - public VariantQuery sampleLimit(String value) { + public VariantQuery sampleLimit(int value) { put(VariantQueryParam.SAMPLE_LIMIT.key(), value); return this; } - public String sampleLimit() { - return getString(VariantQueryParam.SAMPLE_LIMIT.key()); + public int sampleLimit() { + return getInt(VariantQueryParam.SAMPLE_LIMIT.key()); } public VariantQuery sampleSkip(String value) { put(VariantQueryParam.SAMPLE_SKIP.key(), value); return this; } - public String sampleSkip() { - return getString(VariantQueryParam.SAMPLE_SKIP.key()); + public VariantQuery sampleSkip(int value) { + put(VariantQueryParam.SAMPLE_SKIP.key(), value); + return this; + } + public int sampleSkip() { + return getInt(VariantQueryParam.SAMPLE_SKIP.key()); } public VariantQuery file(String value) { @@ -623,12 +641,12 @@ public VariantQuery release(String value) { put(VariantQueryParam.RELEASE.key(), value); return this; } - public VariantQuery release(int value) { + public VariantQuery release(Integer value) { put(VariantQueryParam.RELEASE.key(), value); return this; } - public String release() { - return getString(VariantQueryParam.RELEASE.key()); + public Integer release() { + return getInt(VariantQueryParam.RELEASE.key()); } @Override diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryException.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryException.java index d90698f9e44..20ef94c7f2c 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryException.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryException.java @@ -217,6 +217,10 @@ public static VariantQueryException fileNotFound(Object file, Object study) { return new VariantQueryException("File " + quote(file) + " not found in study '" + study + "'"); } + public static VariantQueryException fileNotIndexed(Object file, Object study) { + return new VariantQueryException("File " + quote(file) + " not indexed in study '" + study + "'"); + } + public static VariantQueryException scoreNotFound(Object score, Object study) { return new VariantQueryException("Variant Score " + quote(score) + " not found in study '" + study + "'"); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java index adc0b72ea44..1973d7f383d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryParam.java @@ -41,7 +41,7 @@ public final class VariantQueryParam implements QueryParam { private static final String ACCEPTS_AND_OR = "Accepts AND (" + AND + ") and OR (" + OR + ") operators."; public static final String ID_DESCR - = "List of IDs, these can be rs IDs (dbSNP) or variants in the format chrom:start:ref:alt, e.g. rs116600158,19:7177679:C:T"; + = "List of variant IDs in the format chrom:start:ref:alt, e.g. 19:7177679:C:T"; public static final VariantQueryParam ID = new VariantQueryParam("id", TEXT_ARRAY, ID_DESCR); public static final String REGION_DESCR @@ -122,7 +122,7 @@ public final class VariantQueryParam implements QueryParam { public static final String INCLUDE_SAMPLE_ID_DESCR = "Include sampleId on each result"; - public static final VariantQueryParam INCLUDE_SAMPLE_ID = new VariantQueryParam("includeSampleId", TEXT_ARRAY, INCLUDE_SAMPLE_ID_DESCR); + public static final VariantQueryParam INCLUDE_SAMPLE_ID = new VariantQueryParam("includeSampleId", BOOLEAN, INCLUDE_SAMPLE_ID_DESCR); public static final String SAMPLE_METADATA_DESCR = "Return the samples metadata group by study. Sample names will appear in the same order as their corresponding genotypes."; @@ -216,7 +216,7 @@ public final class VariantQueryParam implements QueryParam { public static final String ANNOT_XREF_DESCR = "List of any external reference, these can be genes, proteins or variants. " - + "Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, ..."; + + "Accepted IDs include HGNC, Ensembl genes, dbSNP, ClinVar, HPO, Cosmic, HGVS ..."; public static final VariantQueryParam ANNOT_XREF = new VariantQueryParam("xref", TEXT_ARRAY, ANNOT_XREF_DESCR); public static final String GENE_DESCR diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/LimitVariantDBIterator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/LimitVariantDBIterator.java index 3f69fe91924..07ece2a578d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/LimitVariantDBIterator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/LimitVariantDBIterator.java @@ -1,16 +1,32 @@ package org.opencb.opencga.storage.core.variant.adaptors.iterators; +import org.opencb.biodata.models.variant.Variant; + +import java.util.NoSuchElementException; + public class LimitVariantDBIterator extends DelegatedVariantDBIterator { private final int limit; + private int count; LimitVariantDBIterator(VariantDBIterator delegated, int limit) { super(delegated); this.limit = limit; + this.count = 0; } @Override public boolean hasNext() { - return getCount() < limit && super.hasNext(); + return count < limit && super.hasNext(); + } + + @Override + public Variant next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } else { + ++this.count; + return super.next(); + } } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/VariantDBIterator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/VariantDBIterator.java index eae841fc766..f070c4c7bb1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/VariantDBIterator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/iterators/VariantDBIterator.java @@ -16,14 +16,14 @@ package org.opencb.opencga.storage.core.variant.adaptors.iterators; -import com.google.common.collect.Iterators; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.VariantAvro; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.utils.iterators.CloseableIterator; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,21 +131,34 @@ public final void forEachRemaining(Consumer action) { } } - public final DataResult toDataResult() { + + // TODO: The VariantDBIterator should be able to return the samples in the result + // This class should contain a ParsedVariantQuery +// public final VariantQueryResult toVariantQueryResult() { +// } + + public final VariantQueryResult toDataResult(ParsedVariantQuery variantQuery) { List result = new ArrayList<>(); this.forEachRemaining(result::add); int numResults = result.size(); int numTotalResults = -1; // Unknown numTotalResults - return new DataResult<>((int) getTimeFetching(TimeUnit.MILLISECONDS), Collections.emptyList(), numResults, result, numTotalResults); + DataResult dataResult = new DataResult<>((int) + getTimeFetching(TimeUnit.MILLISECONDS), + Collections.emptyList(), + numResults, + result, + numTotalResults); + return new VariantQueryResult<>(dataResult, variantQuery); } - public final VariantQueryResult toDataResult(Map> samples) { - return new VariantQueryResult<>(toDataResult(), samples); + public final List toList() { + List result = new ArrayList<>(); + this.forEachRemaining(result::add); + return result; } - protected interface TimeFunction { R call() throws E; } @@ -192,7 +205,11 @@ public VariantDBIterator filter(Predicate filter) { } public VariantDBIterator localSkip(int skip) { - Iterators.advance(this, skip); + int i = 0; + while (i < skip && hasNext()) { + next(); + i++; + } return this; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/sample/VariantSampleDataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/sample/VariantSampleDataManager.java index d0e09c582e5..b575d1f63e5 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/sample/VariantSampleDataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/sample/VariantSampleDataManager.java @@ -40,13 +40,13 @@ public VariantSampleDataManager(VariantDBAdaptor dbAdaptor) { } - public final DataResult getSampleData(String variant, String study, QueryOptions options) { + public final DataResult getSampleData(Variant variant, String study, QueryOptions options) { options = options == null ? new QueryOptions() : options; int sampleLimit = options.getInt(SAMPLE_BATCH_SIZE, SAMPLE_BATCH_SIZE_DEFAULT); return getSampleData(variant, study, options, sampleLimit); } - public final DataResult getSampleData(String variant, String study, QueryOptions options, int sampleLimit) { + public final DataResult getSampleData(Variant variant, String study, QueryOptions options, int sampleLimit) { options = options == null ? new QueryOptions() : options; @@ -77,7 +77,7 @@ public final DataResult getSampleData(String variant, String study, Que } protected DataResult getSampleData( - String variantStr, String study, QueryOptions options, List includeSamples, Set genotypes, + Variant variant, String study, QueryOptions options, List includeSamples, Set genotypes, int sampleLimit) { options = options == null ? new QueryOptions() : options; Set includeFields = VariantField.getIncludeFields(options); @@ -98,7 +98,7 @@ protected DataResult getSampleData( int queries = 0; while (true) { queries++; - Query query = new Query(VariantQueryParam.ID.key(), variantStr) + Query query = new Query(VariantQueryParam.ID.key(), variant.toString()) .append(VariantQueryParam.STUDY.key(), study) .append(VariantQueryParam.INCLUDE_GENOTYPE.key(), options.get(VariantQueryParam.INCLUDE_GENOTYPE.key())) .append(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), options.get(VariantQueryParam.INCLUDE_SAMPLE_DATA.key())) @@ -130,7 +130,7 @@ protected DataResult getSampleData( DataResult result = dbAdaptor.get(query, variantQueryOptions); if (result.getNumResults() == 0) { - throw VariantQueryException.variantNotFound(variantStr); + throw VariantQueryException.variantNotFound(variant.toString()); } dbTime += result.getTime(); Variant partialVariant = result.first(); @@ -199,7 +199,7 @@ protected DataResult getSampleData( } } - Variant variant = new Variant(variantStr); + variant = new Variant(variant.toString()); variant.setAnnotation(annotation); StudyEntry studyEntry = new StudyEntry(study); variant.addStudyEntry(studyEntry); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtils.java new file mode 100644 index 00000000000..a4d40cf3d56 --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtils.java @@ -0,0 +1,150 @@ +package org.opencb.opencga.storage.core.variant.annotation.converters; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.variant.avro.*; +import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class VariantAnnotationModelUtils { + + /** + * Extracts all the XRefs from a VariantAnnotation object. + * Includes: + * - annotation.id + * - annotation.xrefs.id + * - annotation.hgvs + * - annotation.consequenceTypes.geneName + * - annotation.consequenceTypes.geneId + * - annotation.consequenceTypes.ensemblGeneId + * - annotation.consequenceTypes.transcriptId + * - annotation.consequenceTypes.ensemblTranscriptId + * - annotation.consequenceTypes.hgvs + * - annotation.consequenceTypes.proteinVariantAnnotation.proteinId + * - annotation.consequenceTypes.proteinVariantAnnotation.uniprotAccession + * - annotation.consequenceTypes.proteinVariantAnnotation.uniprotName + * - annotation.consequenceTypes.proteinVariantAnnotation.uniprotVariantId + * - annotation.consequenceTypes.proteinVariantAnnotation.features.id + * - annotation.traitAssociation.id + * - annotation.geneTraitAssociation.id + * - annotation.geneTraitAssociation.hpo + * - annotation.pharmacogenomics.id + * - annotation.pharmacogenomics.name + * + * @param variantAnnotation VariantAnnotation object + * @return Set of XRefs + */ + private static final Pattern HGVS_PATTERN = Pattern.compile("\\([^()]*\\)"); + + public Set extractXRefs(VariantAnnotation variantAnnotation) { + Set xrefs = new HashSet<>(100); + + if (variantAnnotation == null) { + return xrefs; + } + + xrefs.add(variantAnnotation.getId()); + + if (CollectionUtils.isNotEmpty(variantAnnotation.getXrefs())) { + for (Xref xref : variantAnnotation.getXrefs()) { + if (xref != null) { + xrefs.add(xref.getId()); + } + } + } + + List consequenceTypes = variantAnnotation.getConsequenceTypes(); + + if (CollectionUtils.isNotEmpty(variantAnnotation.getHgvs())) { + xrefs.addAll(variantAnnotation.getHgvs()); + + // TODO Remove this code when CellBase 6.4.0 returns the expected HGVS + for (String hgvs: variantAnnotation.getHgvs()) { + if (VariantQueryUtils.isTranscript(hgvs)) { + // 1. Remove the content between parentheses, e.g. ENST00000680783.1(ENSG00000135744):c.776T>C + if (hgvs.contains("(")) { + Matcher matcher = HGVS_PATTERN.matcher(hgvs); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, ""); + } + matcher.appendTail(result); + xrefs.add(result.toString()); + } + + // 2. Add the HGVS with the Ensembl and gene name, e.g. ENSG00000135744:c.776T>C, AGT:c.776T>C + if (CollectionUtils.isNotEmpty(consequenceTypes)) { + for (ConsequenceType conseqType : consequenceTypes) { + if (conseqType != null && conseqType.getHgvs() != null && conseqType.getHgvs().contains(hgvs)) { + String[] fields = hgvs.split(":", 2); + if (StringUtils.isNotEmpty(conseqType.getGeneId())) { + xrefs.add(conseqType.getGeneId() + ":" + fields[1]); + } + if (StringUtils.isNotEmpty(conseqType.getGeneName())) { + xrefs.add(conseqType.getGeneName() + ":" + fields[1]); + } + break; + } + } + } + } + } + } + + if (CollectionUtils.isNotEmpty(consequenceTypes)) { + for (ConsequenceType conseqType : consequenceTypes) { + xrefs.add(conseqType.getGeneName()); + xrefs.add(conseqType.getGeneId()); + xrefs.add(conseqType.getTranscriptId()); + xrefs.add(conseqType.getEnsemblGeneId()); + xrefs.add(conseqType.getEnsemblTranscriptId()); + + ProteinVariantAnnotation protVarAnnotation = conseqType.getProteinVariantAnnotation(); + if (protVarAnnotation != null) { + xrefs.add(protVarAnnotation.getProteinId()); + xrefs.add(protVarAnnotation.getUniprotAccession()); + xrefs.add(protVarAnnotation.getUniprotName()); + xrefs.add(protVarAnnotation.getUniprotVariantId()); + + if (protVarAnnotation.getFeatures() != null) { + for (ProteinFeature proteinFeature : protVarAnnotation.getFeatures()) { + xrefs.add(proteinFeature.getId()); + } + } + } + } + } + + if (CollectionUtils.isNotEmpty(variantAnnotation.getTraitAssociation())) { + for (EvidenceEntry evidenceEntry : variantAnnotation.getTraitAssociation()) { + xrefs.add(evidenceEntry.getId()); + } + } + + if (CollectionUtils.isNotEmpty(variantAnnotation.getGeneTraitAssociation())) { + for (GeneTraitAssociation geneTrait : variantAnnotation.getGeneTraitAssociation()) { + xrefs.add(geneTrait.getId()); + xrefs.add(geneTrait.getHpo()); + } + } + + if (CollectionUtils.isNotEmpty(variantAnnotation.getPharmacogenomics())) { + for (Pharmacogenomics pharmacogenomics : variantAnnotation.getPharmacogenomics()) { + xrefs.add(pharmacogenomics.getId()); + xrefs.add(pharmacogenomics.getName()); + } + } + + // Remove empty strings and nulls + xrefs.remove(""); + xrefs.remove(null); + + return xrefs; + } + +} diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/mixin/ConsequenceTypeMixin.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/mixin/ConsequenceTypeMixin.java index ad643f53068..ab4946fc572 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/mixin/ConsequenceTypeMixin.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/io/json/mixin/ConsequenceTypeMixin.java @@ -4,12 +4,11 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.util.StdConverter; import org.opencb.biodata.models.variant.avro.ConsequenceType; -import org.opencb.cellbase.client.rest.ParentRestClient; import java.util.List; @JsonDeserialize( - converter = ParentRestClient.ConsequenceTypeMixin.ConsequenceTypeConverter.class + converter = ConsequenceTypeMixin.ConsequenceTypeConverter.class ) public interface ConsequenceTypeMixin { class ConsequenceTypeConverter extends StdConverter { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java index 9c89ffb5b61..8468ab34317 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/ParsedVariantQuery.java @@ -1,53 +1,76 @@ package org.opencb.opencga.storage.core.variant.query; +import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.variant.Variant; +import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyResourceMetadata; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; import java.util.*; import java.util.stream.Collectors; +import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.ID_INTERSECT; public class ParsedVariantQuery { private Query inputQuery; private QueryOptions inputOptions; - private Query query; + private VariantQuery query; private boolean optimized = false; + private List events = new ArrayList<>(); + private VariantQueryProjection projection; private final VariantStudyQuery studyQuery; -// private VariantAnnotationQuery annotationQuery; + private Integer limit; + private int skip; + private boolean count; + private int approximateCountSamplingSize; + private List geneRegions; + private List regions; + private List> clinicalCombination; + private List clinicalCombinationList; + // private VariantAnnotationQuery annotationQuery; public ParsedVariantQuery() { this.inputQuery = new Query(); this.inputOptions = new QueryOptions(); - this.query = new Query(); + this.query = new VariantQuery(); studyQuery = new VariantStudyQuery(); } public ParsedVariantQuery(Query inputQuery, QueryOptions inputOptions) { this.inputQuery = inputQuery; this.inputOptions = inputOptions; - this.query = inputQuery; + this.query = new VariantQuery(inputQuery); studyQuery = new VariantStudyQuery(); } public ParsedVariantQuery(ParsedVariantQuery other) { this.inputQuery = new Query(other.inputQuery); this.inputOptions = new QueryOptions(other.inputOptions); - this.query = new Query(other.query); + this.query = new VariantQuery(other.query); this.projection = other.projection; - this.studyQuery = other.studyQuery; + this.studyQuery = new VariantStudyQuery(other.getStudyQuery()); this.optimized = other.optimized; + this.limit = other.limit; + this.skip = other.skip; + this.count = other.count; + this.approximateCountSamplingSize = other.approximateCountSamplingSize; + this.geneRegions = new ArrayList<>(other.geneRegions); + this.regions = new ArrayList<>(other.regions); + this.clinicalCombination = new ArrayList<>(other.clinicalCombination); + this.clinicalCombinationList = new ArrayList<>(other.clinicalCombinationList); } public Query getInputQuery() { @@ -59,11 +82,11 @@ public ParsedVariantQuery setInputQuery(Query inputQuery) { return this; } - public Query getQuery() { + public VariantQuery getQuery() { return query; } - public ParsedVariantQuery setQuery(Query query) { + public ParsedVariantQuery setQuery(VariantQuery query) { this.query = query; return this; } @@ -77,6 +100,15 @@ public ParsedVariantQuery setOptimized(boolean optimized) { return this; } + public List getEvents() { + return events; + } + + public ParsedVariantQuery setEvents(List events) { + this.events = events; + return this; + } + public VariantQueryProjection getProjection() { return projection; } @@ -107,6 +139,24 @@ public VariantQueryXref getXrefs() { return VariantQueryParser.parseXrefs(query); } + public List getRegions() { + return regions; + } + + public ParsedVariantQuery setRegions(List regions) { + this.regions = regions; + return this; + } + + public List getGeneRegions() { + return geneRegions; + } + + public ParsedVariantQuery setGeneRegions(List geneRegions) { + this.geneRegions = geneRegions; + return this; + } + public List getConsequenceTypes() { return VariantQueryUtils.parseConsequenceTypes(query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key())); } @@ -119,6 +169,76 @@ public List getTranscriptFlags() { return query.getAsStringList(VariantQueryParam.ANNOT_TRANSCRIPT_FLAG.key()); } + public Integer getLimit() { + return limit; + } + + public int getLimitOr(int defaultValue) { + return limit == null ? defaultValue : limit; + } + + public ParsedVariantQuery setLimit(Integer limit) { + this.limit = limit; + return this; + } + + public int getSkip() { + return skip; + } + + public ParsedVariantQuery setSkip(int skip) { + this.skip = skip; + return this; + } + + public boolean getCount() { + return count; + } + + public ParsedVariantQuery setCount(boolean count) { + this.count = count; + return this; + } + + public int getApproximateCountSamplingSize() { + return approximateCountSamplingSize; + } + + public ParsedVariantQuery setApproximateCountSamplingSize(int approximateCountSamplingSize) { + this.approximateCountSamplingSize = approximateCountSamplingSize; + return this; + } + + public ParsedQuery> getPopulationFrequencyAlt() { + return VariantQueryParser.parseFreqFilter(query, ANNOT_POPULATION_ALTERNATE_FREQUENCY); + } + + public ParsedQuery> getPopulationFrequencyRef() { + return VariantQueryParser.parseFreqFilter(query, ANNOT_POPULATION_REFERENCE_FREQUENCY); + } + + public ParsedQuery> getPopulationFrequencyMaf() { + return VariantQueryParser.parseFreqFilter(query, ANNOT_POPULATION_MINOR_ALLELE_FREQUENCY); + } + + public List> getClinicalCombinations() { + return clinicalCombination; + } + + public ParsedVariantQuery setClinicalCombination(List> clinicalCombination) { + this.clinicalCombination = clinicalCombination; + return this; + } + + public List getClinicalCombinationsList() { + return clinicalCombinationList; + } + + public ParsedVariantQuery setClinicalCombinationList(List clinicalCombinationList) { + this.clinicalCombinationList = clinicalCombinationList; + return this; + } + public static class VariantStudyQuery { private ParsedQuery studies; private ParsedQuery>> genotypes; @@ -131,6 +251,13 @@ public static class VariantStudyQuery { public VariantStudyQuery() { } + public VariantStudyQuery(VariantStudyQuery studyQuery) { + this.studies = studyQuery.studies; + this.genotypes = studyQuery.genotypes; + this.sampleDataQuery = studyQuery.sampleDataQuery; + this.defaultStudy = studyQuery.defaultStudy; + } + public ParsedQuery getStudies() { return studies; } @@ -140,6 +267,14 @@ public VariantStudyQuery setStudies(ParsedQuery studies) { return this; } + public String getStudyOrFail() { + if (studies == null || studies.size() != 1) { + throw new VariantQueryException("Require exactly one study"); + } else { + return studies.get(0); + } + } + public ParsedQuery>> getGenotypes() { return genotypes; } @@ -226,5 +361,16 @@ public List getIDsAndXrefs() { public boolean isEmpty() { return genes.isEmpty() && variants.isEmpty() && ids.isEmpty() && otherXrefs.isEmpty(); } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("VariantQueryXref{"); + sb.append("genes=").append(genes); + sb.append(", variants=").append(variants); + sb.append(", ids=").append(ids); + sb.append(", otherXrefs=").append(otherXrefs); + sb.append('}'); + return sb.toString(); + } } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java index 1564ad0fabf..641e365a51d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParser.java @@ -3,12 +3,15 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.ClinicalSignificance; import org.opencb.biodata.models.variant.avro.VariantType; +import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.models.variant.metadata.VariantFileHeaderComplexLine; +import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; @@ -19,7 +22,9 @@ import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; import org.opencb.opencga.storage.core.utils.CellBaseUtils; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; @@ -148,21 +153,39 @@ public ParsedVariantQuery parseQuery(Query query, QueryOptions options) { return parseQuery(query, options, false); } - public ParsedVariantQuery parseQuery(Query query, QueryOptions options, boolean skipPreProcess) { - if (query == null) { - query = new Query(); + public ParsedVariantQuery parseQuery(Query inputQuery, QueryOptions options, boolean skipPreProcess) { + if (inputQuery == null) { + inputQuery = new Query(); } if (options == null) { options = new QueryOptions(); } - ParsedVariantQuery variantQuery = new ParsedVariantQuery(new Query(query), new QueryOptions(options)); + ParsedVariantQuery variantQuery = new ParsedVariantQuery(new Query(inputQuery), new QueryOptions(options)); + int limit = options.getInt(QueryOptions.LIMIT, -1); + variantQuery.setLimit(limit == -1 ? null : limit); + variantQuery.setSkip(options.getInt(QueryOptions.SKIP, 0)); + variantQuery.setCount(options.getBoolean(QueryOptions.COUNT, false)); + variantQuery.setApproximateCountSamplingSize(options.getInt( + VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), + VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.defaultValue())); + + variantQuery.setProjection(projectionParser.parseVariantQueryProjection(inputQuery, options)); + VariantQuery query; if (!skipPreProcess) { - query = preProcessQuery(query, options); + query = new VariantQuery(preProcessQuery(inputQuery, options, variantQuery.getProjection())); + } else { + query = new VariantQuery(inputQuery); } variantQuery.setQuery(query); - variantQuery.setProjection(projectionParser.parseVariantQueryProjection(query, options)); + + List geneRegions = Region.parseRegions(query.getString(ANNOT_GENE_REGIONS.key())); + variantQuery.setGeneRegions(geneRegions == null ? Collections.emptyList() : geneRegions); + List regions = Region.parseRegions(query.region(), true); + variantQuery.setRegions(regions == null ? Collections.emptyList() : regions); + variantQuery.setClinicalCombination(VariantQueryParser.parseClinicalCombination(query, false)); + variantQuery.setClinicalCombinationList(VariantQueryParser.parseClinicalCombinationsList(query, false)); ParsedVariantQuery.VariantStudyQuery studyQuery = variantQuery.getStudyQuery(); @@ -173,7 +196,7 @@ public ParsedVariantQuery parseQuery(Query query, QueryOptions options, boolean } if (isValidParam(query, GENOTYPE)) { HashMap> map = new HashMap<>(); - QueryOperation op = VariantQueryUtils.parseGenotypeFilter(query.getString(GENOTYPE.key()), map); + QueryOperation op = VariantQueryUtils.parseGenotypeFilter(query.genotype(), map); if (defaultStudy == null) { List studyNames = metadataManager.getStudyNames(); @@ -209,13 +232,17 @@ public ParsedVariantQuery parseQuery(Query query, QueryOptions options, boolean return variantQuery; } - public Query preProcessQuery(Query originalQuery, QueryOptions options) { + public final Query preProcessQuery(Query originalQuery, QueryOptions options) { + return preProcessQuery(originalQuery, options, null); + } + + protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { // Copy input query! Do not modify original query! Query query = VariantQueryUtils.copy(originalQuery); preProcessAnnotationParams(query); - preProcessStudyParams(query, options); + preProcessStudyParams(query, options, projection); if (options != null && options.getLong(QueryOptions.LIMIT) < 0) { throw VariantQueryException.malformedParam(QueryOptions.LIMIT, options.getString(QueryOptions.LIMIT), @@ -361,7 +388,7 @@ private VariantType parseVariantType(String type) { } } - protected void preProcessStudyParams(Query query, QueryOptions options) { + protected void preProcessStudyParams(Query query, QueryOptions options, VariantQueryProjection projection) { StudyMetadata defaultStudy = getDefaultStudy(query); QueryOperation formatOperator = null; if (isValidParam(query, SAMPLE_DATA)) { @@ -433,6 +460,19 @@ protected void preProcessStudyParams(Query query, QueryOptions options) { } } + if (isValidParam(query, FILE)) { + ParsedQuery files = splitValue(query, FILE); + for (String file : files.getValues()) { + Pair fileIdPair = metadataManager.getFileIdPair(file, false, defaultStudy); + if (fileIdPair == null) { + throw VariantQueryException.fileNotFound(file, defaultStudy.getName()); + } + if (!metadataManager.isFileIndexed(fileIdPair.getKey(), fileIdPair.getValue())) { + throw VariantQueryException.fileNotIndexed(file, metadataManager.getStudyName(fileIdPair.getKey())); + } + } + } + QueryOperation genotypeOperator = null; VariantQueryParam genotypeParam = null; @@ -661,22 +701,23 @@ protected void preProcessStudyParams(Query query, QueryOptions options) { if (!isValidParam(query, INCLUDE_STUDY) || !isValidParam(query, INCLUDE_SAMPLE) || !isValidParam(query, INCLUDE_FILE) - || !isValidParam(query, SAMPLE_SKIP) - || !isValidParam(query, SAMPLE_LIMIT) + || isValidParam(query, SAMPLE_SKIP) + || isValidParam(query, SAMPLE_LIMIT) ) { - VariantQueryProjection selectVariantElements = - VariantQueryProjectionParser.parseVariantQueryFields(query, options, metadataManager); + if (projection == null) { + projection = projectionParser.parseVariantQueryProjection(query, options); + } // Apply the sample pagination. // Remove the sampleLimit and sampleSkip to avoid applying the pagination twice query.remove(SAMPLE_SKIP.key()); query.remove(SAMPLE_LIMIT.key()); - query.put(NUM_TOTAL_SAMPLES.key(), selectVariantElements.getNumTotalSamples()); - query.put(NUM_SAMPLES.key(), selectVariantElements.getNumSamples()); + query.put(NUM_TOTAL_SAMPLES.key(), projection.getNumTotalSamples()); + query.put(NUM_SAMPLES.key(), projection.getNumSamples()); if (!isValidParam(query, INCLUDE_STUDY)) { List includeStudy = new ArrayList<>(); - for (Integer studyId : selectVariantElements.getStudyIds()) { - includeStudy.add(selectVariantElements.getStudy(studyId).getStudyMetadata().getName()); + for (Integer studyId : projection.getStudyIds()) { + includeStudy.add(projection.getStudy(studyId).getStudyMetadata().getName()); } if (includeStudy.isEmpty()) { query.put(INCLUDE_STUDY.key(), NONE); @@ -684,22 +725,17 @@ protected void preProcessStudyParams(Query query, QueryOptions options) { query.put(INCLUDE_STUDY.key(), includeStudy); } } - if (!isValidParam(query, INCLUDE_SAMPLE) || selectVariantElements.getSamplePagination()) { - List includeSample = selectVariantElements.getSamples() - .entrySet() - .stream() - .flatMap(e -> e.getValue() - .stream() - .map(s -> metadataManager.getSampleName(e.getKey(), s))) - .collect(Collectors.toList()); + if (!isValidParam(query, INCLUDE_SAMPLE) || projection.getSamplePagination()) { + List includeSample = projection.getSampleNames().values() + .stream().flatMap(Collection::stream).collect(Collectors.toList()); if (includeSample.isEmpty()) { query.put(INCLUDE_SAMPLE.key(), NONE); } else { query.put(INCLUDE_SAMPLE.key(), includeSample); } } - if (!isValidParam(query, INCLUDE_FILE) || selectVariantElements.getSamplePagination()) { - List includeFile = selectVariantElements.getFiles() + if (!isValidParam(query, INCLUDE_FILE) || projection.getSamplePagination()) { + List includeFile = projection.getFiles() .entrySet() .stream() .flatMap(e -> e.getValue() @@ -851,15 +887,28 @@ public static ParsedVariantQuery.VariantQueryXref parseXrefs(Query query) { if (variant != null) { xrefs.getVariants().add(variant); } else { - if (isVariantAccession(value) || isClinicalAccession(value) || isGeneAccession(value)) { + if (isVariantAccession(value) + || isClinicalAccession(value) + || isGeneAccession(value) + || isHGVS(value) + || isProteinFeatureId(value)) { xrefs.getOtherXrefs().add(value); } else { genes.add(value); } } } + } + if (!xrefs.getVariants().isEmpty()) { + List normalizedVariants = normalizeVariants(xrefs.getVariants()); + for (Variant normalizedVariant : normalizedVariants) { + if (!xrefs.getVariants().contains(normalizedVariant)) { + xrefs.getVariants().add(normalizedVariant); + } + } } + if (isValidParam(query, ANNOT_GENE_ROLE_IN_CANER_GENES)) { List thisGenes = query.getAsStringList(ANNOT_GENE_ROLE_IN_CANER_GENES.key()); if (thisGenes.size() != 1 || !thisGenes.get(0).equals(NONE)) { @@ -886,6 +935,21 @@ public static ParsedVariantQuery.VariantQueryXref parseXrefs(Query query) { return xrefs; } + public static Variant normalizeVariant(Variant variant) { + return normalizeVariants(Collections.singletonList(variant)).get(0); + } + + public static List normalizeVariants(List variants) { + VariantNormalizer variantNormalizer = new VariantNormalizer(); + List normalizedVariants; + try { + normalizedVariants = variantNormalizer.normalize(variants, false); + } catch (NonStandardCompliantSampleField e) { + throw VariantQueryException.internalException(e); + } + return normalizedVariants; + } + public static ParsedQuery> parseFreqFilter(Query query, QueryParam queryParam) { return VariantQueryUtils.splitValue(query, queryParam) .map(VariantQueryUtils::parseKeyOpValue) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryResult.java similarity index 64% rename from opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java rename to opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryResult.java index c955b3ca379..ee432deb3b7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/response/VariantQueryResult.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryResult.java @@ -14,13 +14,17 @@ * limitations under the License. */ -package org.opencb.opencga.core.response; +package org.opencb.opencga.storage.core.variant.query; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -45,25 +49,19 @@ public class VariantQueryResult extends OpenCGAResult { private static final String APPROXIMATE_COUNT = "approximateCount"; private static final String APPROXIMATE_COUNT_SAMPLING_SIZE = "approximateCountSamplingSize"; - public VariantQueryResult() { + protected VariantQueryResult() { } - public VariantQueryResult(long time, int numResults, long numMatches, List events, List result) { - this(time, numResults, numMatches, events, result, null, null, null, null, null); + public VariantQueryResult(long time, int numResults, long numMatches, List events, List result, String source, + ParsedVariantQuery variantQuery) { + this(time, numResults, numMatches, events, result, source, null, null, null, variantQuery); } public VariantQueryResult(long time, int numResults, long numMatches, List events, List result, - Map> samples, String source) { - this(time, numResults, numMatches, events, result, samples, source, null, null, null); - } - - public VariantQueryResult(long time, int numResults, long numMatches, List events, List result, - Map> samples, String source, Boolean approximateCount, - Integer approximateCountSamplingSize, Integer numTotalSamples) { + String source, Boolean approximateCount, Integer approximateCountSamplingSize, Integer numTotalSamples, + ParsedVariantQuery variantQuery) { super((int) time, events, numResults, result, numMatches); - if (samples != null) { - setSamples(samples); - } + if (source != null) { setSource(source); } @@ -73,15 +71,18 @@ public VariantQueryResult(long time, int numResults, long numMatches, List()); + } + if (variantQuery != null) { + addSamplesMetadataIfRequested(variantQuery); + } } - public VariantQueryResult(DataResult dataResult) { + private VariantQueryResult(DataResult dataResult, List results) { super(dataResult.getTime(), dataResult.getEvents(), dataResult.getNumMatches(), @@ -90,27 +91,66 @@ public VariantQueryResult(DataResult dataResult) { dataResult.getNumDeleted(), dataResult.getNumErrors(), dataResult.getAttributes()); - setResults(dataResult.getResults()); - setNumResults(dataResult.getNumResults()); + setResults(results); + setNumResults(results.size()); + if (getEvents() == null) { + setEvents(new ArrayList<>()); + } + } + + public VariantQueryResult(DataResult dataResult, ParsedVariantQuery variantQuery) { + this(dataResult, dataResult.getResults()); + if (variantQuery != null) { + addSamplesMetadataIfRequested(variantQuery); + } + } + + public VariantQueryResult(VariantQueryResult dataResult, List results) { + this((DataResult) dataResult, results); + } + + public VariantQueryResult(DataResult dataResult, String source, ParsedVariantQuery variantQuery) { + this(dataResult, variantQuery); + setSource(source); + } + + /* + * @deprecated Missing ParsedVariantQuery. + * Use {@link #VariantQueryResult(long, int, long, List, List, String, Boolean, Integer, Integer, ParsedVariantQuery)} + */ + @Deprecated + public VariantQueryResult(long time, int numResults, long numMatches, List events, List result, String source) { + this(time, numResults, numMatches, events, result, source, null, null, null, (ParsedVariantQuery) null); } - public VariantQueryResult(DataResult queryResult, Map> samples) { - this(queryResult, samples, null); + /* + * @deprecated Missing ParsedVariantQuery. + * Use {@link #VariantQueryResult(DataResult, ParsedVariantQuery)} + */ + @Deprecated + public VariantQueryResult(DataResult dataResult) { + this(dataResult, (ParsedVariantQuery) null); } - public VariantQueryResult(DataResult dataResult, Map> samples, String source) { - this(dataResult); - setSamples(samples); - if (getNumMatches() >= 0) { - setApproximateCount(false); + private void addSamplesMetadataIfRequested(ParsedVariantQuery query) { + VariantQueryProjection projection = query.getProjection(); + + // Ensure is modifiable + if (getEvents() == null || Collections.emptyList().getClass().equals(getEvents().getClass())) { + setEvents(new ArrayList<>()); } - if (samples != null) { - this.setNumSamples(samples.values().stream().mapToInt(List::size).sum()); - this.setNumTotalSamples(getNumSamples()); + if (!query.getEvents().isEmpty()) { + getEvents().addAll(query.getEvents()); } - if (source != null) { - this.setSource(source); + if (!projection.getEvents().isEmpty()) { + getEvents().addAll(projection.getEvents()); + } + + if (query.getQuery().sampleMetadata()) { + setSamples(query.getProjection().getSampleNames()); } + setNumSamples(projection.getNumSamples()); + setNumTotalSamples(projection.getNumTotalSamples()); } public Map> getSamples() { @@ -173,7 +213,9 @@ public VariantQueryResult setApproximateCount(Boolean approximateCount) { } public Integer getApproximateCountSamplingSize() { - return getAttributes().containsKey(APPROXIMATE_COUNT_SAMPLING_SIZE) ? getAttributes().getInt(APPROXIMATE_COUNT_SAMPLING_SIZE) : null; + return getAttributes().containsKey(APPROXIMATE_COUNT_SAMPLING_SIZE) + ? getAttributes().getInt(APPROXIMATE_COUNT_SAMPLING_SIZE) + : null; } public VariantQueryResult setApproximateCountSamplingSize(Integer approximateCountSamplingSize) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryUtils.java index 114379c77a9..9cf773d001f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryUtils.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryUtils.java @@ -28,12 +28,13 @@ import org.opencb.biodata.models.variant.VariantBuilder; import org.opencb.biodata.models.variant.annotation.ConsequenceTypeMappings; import org.opencb.biodata.models.variant.avro.*; -import org.opencb.commons.datastore.core.*; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.QueryParam; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; -import org.opencb.opencga.core.response.VariantQueryResult; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; @@ -466,6 +467,33 @@ public static boolean isVariantAccession(String value) { return value.startsWith("rs") || value.startsWith("VAR_"); } + /** + * Determines if the given value might be a known transcript or not. + * Ensembl transcripts start with `ENST` + * RefSeq transcripts start with `NM_` and `XM_` + * See ... + * + * @param value Value to check + * @return If is a known transcript + */ + public static boolean isTranscript(String value) { + return value.startsWith("ENST") || value.startsWith("NM_") || value.startsWith("XM_"); + } + + /** + * Determines if the given value is a HGVS. + * + * @param value Value to check + * @return If is a known accession + */ + public static boolean isHGVS(String value) { + // Check regex ':[cnpg].' + // HGVC examples : + // - "1:g.65325832G>A" + // - "1:g.65325832_65325833insA" + return value.contains(":c.") || value.contains(":n.") || value.contains(":p.") || value.contains(":g."); + } + /** * Determines if the given value is a known clinical accession or not. *

@@ -494,6 +522,18 @@ public static boolean isGeneAccession(String value) { return isHpo(value) || value.startsWith("OMIM:") || value.startsWith("umls:"); } + /** + * Determines if the given value is a valid protein feature id. + *

+ * Protein feature id starts with 'PRO_', 'VAR_' or 'VSP_' + * + * @param value Value to check + * @return If is a known accession + */ + public static boolean isProteinFeatureId(String value) { + return value.startsWith("PRO_") | value.startsWith("VAR_") | value.startsWith("VSP_"); + } + /** * Determines if the given value is a HPO term or not. *

@@ -552,6 +592,14 @@ public static Variant toVariant(String value) { return variant; } + public static Variant toVariant(String variantStr, boolean normalize) { + Variant variant = toVariant(variantStr); + if (normalize && variant != null) { + return VariantQueryParser.normalizeVariant(variant); + } + return variant; + } + public static String[] splitStudyResource(String value) { int idx = value.lastIndexOf(STUDY_RESOURCE_SEPARATOR); if (idx <= 0 || idx == value.length() - 1) { @@ -583,37 +631,6 @@ public static boolean isOutputMultiStudy(Query query, QueryOptions options, Coll } } - public static VariantQueryResult addSamplesMetadataIfRequested(DataResult result, Query query, QueryOptions options, - VariantStorageMetadataManager variantStorageMetadataManager) { - return addSamplesMetadataIfRequested(new VariantQueryResult<>(result, null), query, options, variantStorageMetadataManager); - } - - public static VariantQueryResult addSamplesMetadataIfRequested(VariantQueryResult result, Query query, QueryOptions options, - VariantStorageMetadataManager variantStorageMetadataManager) { - if (query.getBoolean(SAMPLE_METADATA.key(), false)) { - int numTotalSamples = query.getInt(NUM_TOTAL_SAMPLES.key(), -1); - int numSamples = query.getInt(NUM_SAMPLES.key(), -1); - Map> samplesMetadata = VariantQueryProjectionParser - .getIncludeSampleNames(query, options, variantStorageMetadataManager); - if (numTotalSamples < 0 && numSamples < 0) { - numTotalSamples = samplesMetadata.values().stream().mapToInt(List::size).sum(); - VariantQueryProjectionParser.skipAndLimitSamples(query, samplesMetadata); - numSamples = samplesMetadata.values().stream().mapToInt(List::size).sum(); - } - return result.setNumSamples(numSamples) - .setNumTotalSamples(numTotalSamples) - .setSamples(samplesMetadata); - } else { - int numTotalSamples = query.getInt(NUM_TOTAL_SAMPLES.key(), -1); - int numSamples = query.getInt(NUM_SAMPLES.key(), -1); - if (numTotalSamples >= 0 && numSamples >= 0) { - return result.setNumSamples(numSamples) - .setNumTotalSamples(numTotalSamples); - } - return result; - } - } - /** * Gets a list of elements sample data keys to return. * @@ -1572,6 +1589,13 @@ public static String printQuery(Query query) { query.put(ANNOT_GENE_REGIONS.key(), "numGeneRegions : " + ((Collection) geneRegions).size()); } } + if (isValidParam(query, ID_INTERSECT)) { + query = new Query(query); + Object idIntersect = query.get(ID_INTERSECT.key()); + if (idIntersect instanceof Collection) { + query.put(ID_INTERSECT.key(), "numIdIntersect : " + ((Collection) idIntersect).size()); + } + } try { return QUERY_MAPPER.writeValueAsString(query); } catch (JsonProcessingException e) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/AbstractTwoPhasedVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/AbstractTwoPhasedVariantQueryExecutor.java index a49fe60c3e4..d71311a107d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/AbstractTwoPhasedVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/AbstractTwoPhasedVariantQueryExecutor.java @@ -7,7 +7,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIteratorWithCounts; @@ -133,7 +133,7 @@ protected boolean shouldGetApproximateCount(QueryOptions options, boolean iterat // } protected int getLimit(QueryOptions options) { - return options.getInt(QueryOptions.LIMIT); + return options.getInt(QueryOptions.LIMIT, -1); } protected int getSkip(QueryOptions options) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java index 0e8c9ead98c..6eb237ea4b3 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/BreakendVariantQueryExecutor.java @@ -1,6 +1,5 @@ package org.opencb.opencga.storage.core.variant.query.executors; -import com.google.common.collect.Iterators; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.variant.StudyEntry; @@ -13,14 +12,13 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.filters.VariantFilterBuilder; @@ -33,12 +31,12 @@ public class BreakendVariantQueryExecutor extends VariantQueryExecutor { private final VariantDBAdaptor variantDBAdaptor; private final VariantFilterBuilder filterBuilder; - public BreakendVariantQueryExecutor(VariantStorageMetadataManager metadataManager, String storageEngineId, ObjectMap options, + public BreakendVariantQueryExecutor(String storageEngineId, ObjectMap options, VariantQueryExecutor delegatedQueryExecutor, VariantDBAdaptor variantDBAdaptor) { - super(metadataManager, storageEngineId, options); + super(variantDBAdaptor.getMetadataManager(), storageEngineId, options); this.delegatedQueryExecutor = delegatedQueryExecutor; this.variantDBAdaptor = variantDBAdaptor; - filterBuilder = new VariantFilterBuilder(metadataManager); + filterBuilder = new VariantFilterBuilder(); } @Override @@ -48,35 +46,39 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) throws Stor } @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean getIterator) throws StorageEngineException { - int limit = options.getInt(QueryOptions.LIMIT); - int skip = options.getInt(QueryOptions.SKIP); - boolean count = options.getBoolean(QueryOptions.COUNT); - int approximateCountSamplingSize = options.getInt( - VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), - VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.defaultValue()); - Query baseQuery = baseQuery(query); - Predicate variantLocalFilter = filterBuilder.buildFilter(query, options); - + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean getIterator) throws StorageEngineException { + int limit = variantQuery.getLimitOr(-1); + int skip = variantQuery.getSkip(); + boolean count = variantQuery.getCount() && !getIterator; + int approximateCountSamplingSize = variantQuery.getApproximateCountSamplingSize(); + Query baseQuery = baseQuery(variantQuery.getQuery()); + Predicate variantLocalFilter = filterBuilder.buildFilter(variantQuery); + + // Copy to avoid modifications to input query + ParsedVariantQuery delegatedVariantQuery = new ParsedVariantQuery(variantQuery); + QueryOptions options = new QueryOptions(variantQuery.getInputOptions()); + options.remove(QueryOptions.SKIP); + delegatedVariantQuery.setSkip(0); + if (limit >= 0) { + int tmpLimit = skip + limit * 4; + if (count && tmpLimit < approximateCountSamplingSize) { + tmpLimit = approximateCountSamplingSize; + } + options.put(QueryOptions.LIMIT, tmpLimit); + delegatedVariantQuery.setLimit(tmpLimit); + } + delegatedVariantQuery.setInputOptions(options); if (getIterator) { - VariantDBIterator iterator = delegatedQueryExecutor.iterator(query, options); + VariantDBIterator iterator = delegatedQueryExecutor.iterator(delegatedVariantQuery); iterator = iterator.mapBuffered(l -> getBreakendPairs(0, baseQuery, variantLocalFilter, l), 100); iterator = iterator.localLimitSkip(limit, skip); return iterator; } else { - // Copy to avoid modifications to input options - options = new QueryOptions(options); - options.remove(QueryOptions.SKIP); - int tmpLimit = skip + limit * 2; - if (count && tmpLimit < approximateCountSamplingSize) { - tmpLimit = approximateCountSamplingSize; - } - options.put(QueryOptions.LIMIT, tmpLimit); - VariantQueryResult queryResult = delegatedQueryExecutor.get(query, options); + VariantQueryResult queryResult = delegatedQueryExecutor.get(delegatedVariantQuery); List results = queryResult.getResults(); results = getBreakendPairs(0, baseQuery, variantLocalFilter, results); - if (queryResult.getNumMatches() < tmpLimit) { + if (queryResult.getNumMatches() < delegatedVariantQuery.getLimitOr(-1)) { // Exact count!! queryResult.setApproximateCount(false); queryResult.setNumMatches(results.size()); @@ -100,24 +102,6 @@ protected Object getOrIterator(Query query, QueryOptions options, boolean getIte } } - - protected VariantDBIterator iterator(Query query, QueryOptions options, int batchSize) throws StorageEngineException { - - int limit = options.getInt(QueryOptions.LIMIT); - int skip = options.getInt(QueryOptions.SKIP); - Query baseQuery = baseQuery(query); - Predicate variantLocalFilter = filterBuilder.buildFilter(query, options); - - VariantDBIterator iterator = delegatedQueryExecutor.iterator(query, options); - iterator = iterator.mapBuffered(l -> getBreakendPairs(0, baseQuery, variantLocalFilter, l), batchSize); - Iterators.advance(iterator, skip); - iterator = iterator.localSkip(skip); - if (limit > 0) { - iterator = iterator.localLimit(limit); - } - return iterator; - } - private Query baseQuery(Query query) { return subQuery(query, VariantQueryParam.STUDY, @@ -143,7 +127,6 @@ private List getBreakendPairs(int samplePosition, Query baseQuery, Pred } // Copy query to avoid propagating modifications baseQuery = new Query(baseQuery); -// System.out.println("variants = " + variants); List regions = new ArrayList<>(variants.size()); for (Variant variant : variants) { BreakendMate mate = variant.getSv().getBreakend().getMate(); @@ -190,7 +173,7 @@ private List getBreakendPairs(int samplePosition, Query baseQuery, Pred return variantPairs; } - private void addPair(Predicate filter, List variantPairs, Variant variant, Variant mateVariant) { + private boolean addPair(Predicate filter, List variantPairs, Variant variant, Variant mateVariant) { // Check for duplicated pairs if (VariantDBIterator.VARIANT_COMPARATOR.compare(variant, mateVariant) > 0) { // The mate variant is "before" the main variant @@ -200,10 +183,14 @@ private void addPair(Predicate filter, List variantPairs, Vari // But first the "mate" to respect order variantPairs.add(mateVariant); variantPairs.add(variant); + return true; + } else { + return false; } } else { variantPairs.add(variant); variantPairs.add(mateVariant); + return true; } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/ChromDensityVariantAggregationExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/ChromDensityVariantAggregationExecutor.java index b7a03ceac34..9606a74a85d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/ChromDensityVariantAggregationExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/ChromDensityVariantAggregationExecutor.java @@ -8,7 +8,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.solr.FacetQueryParser; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantIterable; @@ -144,7 +144,7 @@ protected VariantQueryResult aggregation(Query query, QueryOptions o regionBuckets.size(), regionBuckets); return new VariantQueryResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), 1, numMatches, Collections.emptyList(), - Collections.singletonList(field), null, null); + Collections.singletonList(field), null); } private VariantQueryException invalidNestedField(String nestedFieldName) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java index db0815b5135..c6f4b87a5f0 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/CompoundHeterozygousQueryExecutor.java @@ -3,21 +3,24 @@ import com.google.common.collect.Iterators; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.tools.pedigree.ModeOfInheritance; -import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; -import org.opencb.opencga.storage.core.variant.adaptors.*; +import org.opencb.opencga.storage.core.variant.adaptors.VariantField; +import org.opencb.opencga.storage.core.variant.adaptors.VariantIterable; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.UnionMultiVariantKeyIterator; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIteratorWithCounts; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,15 +69,11 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) throws Stor } @Override - public DataResult count(Query query) { - throw new UnsupportedOperationException(); - } - - @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator) { - Trio trio = getCompHetTrio(query); - return getOrIterator(query.getString(VariantQueryParam.STUDY.key()), trio.getChild(), trio.getFather(), trio.getMother(), - query, options, iterator); + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { + Trio trio = getCompHetTrio(variantQuery.getQuery()); + String study = variantQuery.getStudyQuery().getStudyOrFail(); + return getOrIterator(study, trio.getChild(), trio.getFather(), trio.getMother(), + variantQuery, iterator); } @Override @@ -84,25 +83,25 @@ protected long primaryCount(Query query, QueryOptions options) { .append(QueryOptions.INCLUDE, VariantField.ID.fieldName()))); } - public VariantQueryResult get(String study, String proband, String father, String mother, Query query, QueryOptions options) { - return (VariantQueryResult) getOrIterator(study, proband, father, mother, query, options, false); + public VariantQueryResult get(String study, String proband, String father, String mother, ParsedVariantQuery variantQuery) { + return (VariantQueryResult) getOrIterator(study, proband, father, mother, variantQuery, false); } - public VariantDBIterator iterator(String study, String proband, String father, String mother, Query query, QueryOptions options) { - return (VariantDBIterator) getOrIterator(study, proband, father, mother, query, options, true); + public VariantDBIterator iterator(String study, String proband, String father, String mother, ParsedVariantQuery variantQuery) { + return (VariantDBIterator) getOrIterator(study, proband, father, mother, variantQuery, true); } - private Object getOrIterator(String study, String proband, String father, String mother, Query query, QueryOptions inputOptions, + private Object getOrIterator(String study, String proband, String father, String mother, ParsedVariantQuery variantQuery, boolean iterator) { // Prepare query and options - int skip = getSkip(inputOptions); - int limit = inputOptions.containsKey(QueryOptions.LIMIT) ? getLimit(inputOptions) : (Integer.MAX_VALUE - skip); - int samplingSize = getSamplingSize(inputOptions, DEFAULT_SAMPLING_SIZE, iterator); - QueryOptions options = buildQueryOptions(inputOptions); + int skip = variantQuery.getSkip(); + int limit = variantQuery.getLimit() != null ? variantQuery.getLimit() : (Integer.MAX_VALUE - skip); + int samplingSize = getSamplingSize(variantQuery.getInputOptions(), DEFAULT_SAMPLING_SIZE, iterator); + QueryOptions options = buildQueryOptions(variantQuery.getInputOptions()); // Always sort results for compound heterozygous options.put(QueryOptions.SORT, true); - query = new Query(query); + Query query = new Query(variantQuery.getQuery()); List includeSample = getAndCheckIncludeSample(query, proband, father, mother); Set biotypes; @@ -158,12 +157,12 @@ private Object getOrIterator(String study, String proband, String father, String return VariantDBIterator.wrapper(variantIterator); } else { VariantQueryResult result = VariantDBIterator.wrapper(variantIterator) - .toDataResult(Collections.singletonMap(study, includeSample)); + .toDataResult(variantQuery); if ((limit + skip) < samplingSize && compoundHeterozygous.size() < samplingSize) { result.setApproximateCount(false); result.setNumMatches(compoundHeterozygous.size()); } else { - setNumTotalResults(unfilteredIterator, result, query, inputOptions, + setNumTotalResults(unfilteredIterator, result, query, variantQuery.getInputOptions(), unfilteredIterator.getCount(), compoundHeterozygous.size()); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java index 08705ccf69e..474cbc3fa9f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/DBAdaptorVariantQueryExecutor.java @@ -2,7 +2,9 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.*; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; @@ -37,11 +39,11 @@ public DBAdaptorVariantQueryExecutor(VariantDBAdaptor dbAdaptor, String storageE } @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator) { + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException { if (iterator) { - return dbAdaptor.iterator(query, options); + return dbAdaptor.iterator(variantQuery); } else { - VariantQueryResult result = dbAdaptor.get(query, options); + VariantQueryResult result = dbAdaptor.get(variantQuery); if (result.getSource() == null || result.getSource().isEmpty()) { result.setSource(storageEngineId); } @@ -49,11 +51,6 @@ protected Object getOrIterator(Query query, QueryOptions options, boolean iterat } } - @Override - public DataResult count(Query query) { - return dbAdaptor.count(query); - } - @Override public boolean canUseThisExecutor(Query query, QueryOptions options) { for (QueryParam unsupportedParam : UNSUPPORTED_PARAMS) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java index 57b1e64ae58..e286b4a07ce 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/NoOpVariantQueryExecutor.java @@ -1,10 +1,8 @@ package org.opencb.opencga.storage.core.variant.query.executors; -import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; @@ -126,14 +124,11 @@ private boolean checkStatsFilter(Query query, String sample, VariantQueryParam p } @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator) throws StorageEngineException { + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException { if (iterator) { return VariantDBIterator.emptyIterator(); } else { - VariantQueryResult result = new VariantQueryResult<>(0, 0, 0, Collections.emptyList(), Collections.emptyList()); - result.setSource(NO_OP); - VariantQueryUtils.addSamplesMetadataIfRequested(result, query, options, metadataManager); - return result; + return new VariantQueryResult<>(0, 0, 0, Collections.emptyList(), Collections.emptyList(), NO_OP, variantQuery); } } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java index a2c7f67d54e..8479dd683f7 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantAggregationExecutor.java @@ -4,7 +4,7 @@ import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java index 8e445cc30e2..26d53e89e5c 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutor.java @@ -1,18 +1,15 @@ package org.opencb.opencga.storage.core.variant.query.executors; import org.opencb.biodata.models.variant.Variant; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.variant.adaptors.VariantIterable; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; - -import java.util.Collections; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.QUERY_DEFAULT_TIMEOUT; import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.QUERY_MAX_TIMEOUT; @@ -22,7 +19,7 @@ * * @author Jacobo Coll <jacobo167@gmail.com> */ -public abstract class VariantQueryExecutor implements VariantIterable { +public abstract class VariantQueryExecutor { protected final VariantStorageMetadataManager metadataManager; protected final String storageEngineId; @@ -34,19 +31,17 @@ public VariantQueryExecutor(VariantStorageMetadataManager metadataManager, Strin this.options = options; } - public final VariantQueryResult get(Query query, QueryOptions options) { + public final VariantQueryResult get(ParsedVariantQuery query) { try { - return (VariantQueryResult) getOrIterator(query, options, false); + return (VariantQueryResult) getOrIterator(query, false); } catch (StorageEngineException e) { throw VariantQueryException.internalException(e); } } - @Override - public final VariantDBIterator iterator(Query query, QueryOptions options) { + public final VariantDBIterator iterator(ParsedVariantQuery variantQuery) { try { -// query = parser.preProcessQuery(query, options); - return (VariantDBIterator) getOrIterator(query, options, true); + return (VariantDBIterator) getOrIterator(variantQuery, true); } catch (StorageEngineException e) { throw VariantQueryException.internalException(e); } @@ -77,19 +72,7 @@ public static void setDefaultTimeout(QueryOptions queryOptions, ObjectMap config */ public abstract boolean canUseThisExecutor(Query query, QueryOptions options) throws StorageEngineException; - @Deprecated - public DataResult count(Query query) { - VariantQueryResult result = get(query, new QueryOptions(QueryOptions.COUNT, true).append(QueryOptions.LIMIT, 0)); - return new DataResult<>( - result.getTime(), - result.getEvents(), - 1, - Collections.singletonList(result.getNumMatches()), - result.getNumMatches(), - result.getAttributes()); - } - - protected abstract Object getOrIterator(Query query, QueryOptions options, boolean iterator) throws StorageEngineException; + protected abstract Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException; protected VariantStorageMetadataManager getMetadataManager() { return metadataManager; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/filters/VariantFilterBuilder.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/filters/VariantFilterBuilder.java index f8705d26eef..1f505864fd1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/filters/VariantFilterBuilder.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/filters/VariantFilterBuilder.java @@ -5,32 +5,26 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.ConsequenceType; import org.opencb.biodata.models.variant.avro.SequenceOntologyTerm; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.*; +import org.opencb.opencga.storage.core.variant.query.KeyOpValue; +import org.opencb.opencga.storage.core.variant.query.ParsedQuery; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import java.util.*; import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; - public class VariantFilterBuilder { - private final VariantStorageMetadataManager metadataManager; - - public VariantFilterBuilder(VariantStorageMetadataManager metadataManager) { - this.metadataManager = metadataManager; + public VariantFilterBuilder() { } - public Predicate buildFilter(Query query, QueryOptions queryOptions) { + public Predicate buildFilter(ParsedVariantQuery variantQuery) { List> filters = new LinkedList<>(); - addRegionFilters(query, filters); - addAnnotationFilters(query, filters); + addRegionFilters(variantQuery, filters); + addAnnotationFilters(variantQuery, filters); if (filters.isEmpty()) { return v -> true; @@ -39,18 +33,18 @@ public Predicate buildFilter(Query query, QueryOptions queryOptions) { } } - private void addRegionFilters(Query query, List> filters) { + private void addRegionFilters(ParsedVariantQuery variantQuery, List> filters) { List> regionFilters = new LinkedList<>(); - if (VariantQueryUtils.isValidParam(query, VariantQueryParam.REGION)) { - List regions = Region.parseRegions(query.getString(REGION.key()), true); + List regions = variantQuery.getRegions(); + if (!regions.isEmpty()) { regions = VariantQueryUtils.mergeRegions(regions); for (Region region : regions) { regionFilters.add(variant -> region.contains(variant.getChromosome(), variant.getStart())); } } - ParsedVariantQuery.VariantQueryXref variantQueryXref = VariantQueryParser.parseXrefs(query); - Predicate geneFilter = getGeneFilter(query, variantQueryXref.getGenes()); + ParsedVariantQuery.VariantQueryXref variantQueryXref = variantQuery.getXrefs(); + Predicate geneFilter = getGeneFilter(variantQuery, variantQueryXref.getGenes()); if (!variantQueryXref.getIds().isEmpty()) { Set ids = new HashSet<>(variantQueryXref.getIds()); regionFilters.add(variant -> ids.contains(variant.getAnnotation().getId())); @@ -65,15 +59,14 @@ private void addRegionFilters(Query query, List> filters) { if (!regionFilters.isEmpty()) { Set bts; - if (VariantQueryUtils.isValidParam(query, VariantQueryParam.ANNOT_BIOTYPE)) { - bts = new HashSet<>(query.getAsStringList(VariantQueryParam.ANNOT_BIOTYPE.key())); + if (!variantQuery.getBiotypes().isEmpty()) { + bts = new HashSet<>(variantQuery.getBiotypes()); } else { bts = null; } Set cts; - if (VariantQueryUtils.isValidParam(query, VariantQueryParam.ANNOT_CONSEQUENCE_TYPE)) { - cts = new HashSet<>(VariantQueryUtils - .parseConsequenceTypes(query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key()))); + if (!variantQuery.getConsequenceTypes().isEmpty()) { + cts = new HashSet<>(variantQuery.getConsequenceTypes()); } else { cts = null; } @@ -99,12 +92,12 @@ private void addRegionFilters(Query query, List> filters) { } } - private Predicate getGeneFilter(Query query, List genes) { + private Predicate getGeneFilter(ParsedVariantQuery variantQuery, List genes) { if (genes.isEmpty()) { return null; } - List geneRegions = Region.parseRegions(query.getString(VariantQueryUtils.ANNOT_GENE_REGIONS.key())); + List geneRegions = variantQuery.getGeneRegions(); Predicate geneRegionFilter; if (CollectionUtils.isEmpty(geneRegions)) { geneRegionFilter = null; @@ -115,15 +108,14 @@ private Predicate getGeneFilter(Query query, List genes) { Predicate geneFilter; Set bts; - if (VariantQueryUtils.isValidParam(query, VariantQueryParam.ANNOT_BIOTYPE)) { - bts = new HashSet<>(query.getAsStringList(VariantQueryParam.ANNOT_BIOTYPE.key())); + if (!variantQuery.getBiotypes().isEmpty()) { + bts = new HashSet<>(variantQuery.getBiotypes()); } else { bts = null; } Set cts; - if (VariantQueryUtils.isValidParam(query, VariantQueryParam.ANNOT_CONSEQUENCE_TYPE)) { - cts = new HashSet<>(VariantQueryUtils - .parseConsequenceTypes(query.getAsStringList(VariantQueryParam.ANNOT_CONSEQUENCE_TYPE.key()))); + if (!variantQuery.getConsequenceTypes().isEmpty()) { + cts = new HashSet<>(variantQuery.getConsequenceTypes()); } else { cts = null; } @@ -151,31 +143,30 @@ private Predicate getGeneFilter(Query query, List genes) { } } - private void addAnnotationFilters(Query query, List> filters) { -// ParsedVariantQuery.VariantQueryXref variantQueryXref = VariantQueryParser.parseXrefs(query); - addClinicalFilters(query, filters); + private void addAnnotationFilters(ParsedVariantQuery variantQuery, List> filters) { +// ParsedVariantQuery.VariantQueryXref variantQueryXref = variantQuery.getXrefs(); + addClinicalFilters(variantQuery, filters); - if (VariantQueryUtils.isValidParam(query, ANNOT_POPULATION_ALTERNATE_FREQUENCY)) { - ParsedQuery> freqQuery - = VariantQueryParser.parseFreqFilter(query, ANNOT_POPULATION_ALTERNATE_FREQUENCY); + ParsedQuery> freqQuery = variantQuery.getPopulationFrequencyAlt(); + if (!freqQuery.isEmpty()) { List freqFilters = freqQuery.mapValues(popFreq -> { String[] split = popFreq.getKey().split(VariantQueryUtils.STUDY_POP_FREQ_SEPARATOR); return new PopulationFrequencyVariantFilter.AltFreqFilter(split[0], split[1], popFreq.getOp(), popFreq.getValue()); }); filters.add(new PopulationFrequencyVariantFilter(freqQuery.getOperation(), freqFilters)); } - if (VariantQueryUtils.isValidParam(query, ANNOT_POPULATION_REFERENCE_FREQUENCY)) { - ParsedQuery> freqQuery - = VariantQueryParser.parseFreqFilter(query, ANNOT_POPULATION_REFERENCE_FREQUENCY); + + freqQuery = variantQuery.getPopulationFrequencyRef(); + if (!freqQuery.isEmpty()) { List freqFilters = freqQuery.mapValues(popFreq -> { String[] split = popFreq.getKey().split(VariantQueryUtils.STUDY_POP_FREQ_SEPARATOR); return new PopulationFrequencyVariantFilter.RefFreqFilter(split[0], split[1], popFreq.getOp(), popFreq.getValue()); }); filters.add(new PopulationFrequencyVariantFilter(freqQuery.getOperation(), freqFilters)); } - if (VariantQueryUtils.isValidParam(query, ANNOT_POPULATION_MINOR_ALLELE_FREQUENCY)) { - ParsedQuery> freqQuery - = VariantQueryParser.parseFreqFilter(query, ANNOT_POPULATION_MINOR_ALLELE_FREQUENCY); + + freqQuery = variantQuery.getPopulationFrequencyMaf(); + if (!freqQuery.isEmpty()) { List freqFilters = freqQuery.mapValues(popFreq -> { String[] split = popFreq.getKey().split(VariantQueryUtils.STUDY_POP_FREQ_SEPARATOR); return new PopulationFrequencyVariantFilter.MafFreqFilter(split[0], split[1], popFreq.getOp(), popFreq.getValue()); @@ -185,8 +176,8 @@ private void addAnnotationFilters(Query query, List> filters) } - private void addClinicalFilters(Query query, List> filters) { - List> clinicalCombinations = VariantQueryParser.parseClinicalCombination(query) + private void addClinicalFilters(ParsedVariantQuery variantQuery, List> filters) { + List> clinicalCombinations = variantQuery.getClinicalCombinations() .stream().map(HashSet::new).collect(Collectors.toList()); if (clinicalCombinations.isEmpty()) { return; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjection.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjection.java index 717967cc038..a31d810fffc 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjection.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjection.java @@ -1,6 +1,7 @@ package org.opencb.opencga.storage.core.variant.query.projection; import com.google.common.collect.Iterables; +import org.opencb.commons.datastore.core.Event; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; @@ -21,6 +22,7 @@ public final class VariantQueryProjection { private final int numSamples; private final int numTotalSamples; + private List events = new ArrayList<>(); public VariantQueryProjection(StudyMetadata studyMetadata, List samples, List files) { this.fields = VariantField.getIncludeFields(null); @@ -77,14 +79,28 @@ public Map> getSamples() { return studies.values().stream().collect(Collectors.toMap(s -> s.studyMetadata.getId(), s -> s.samples)); } + public Map> getSampleNames() { + return studies.values().stream().collect(Collectors.toMap(s -> s.studyMetadata.getName(), s -> s.sampleNames)); + } + @Deprecated public Map> getFiles() { return studies.values().stream().collect(Collectors.toMap(s -> s.studyMetadata.getId(), s -> s.files)); } + public List getEvents() { + return events; + } + + public VariantQueryProjection setEvents(List events) { + this.events = events; + return this; + } + public static class StudyVariantQueryProjection { private StudyMetadata studyMetadata; private List samples = Collections.emptyList(); + private List sampleNames = Collections.emptyList(); private Map> multiFileSampleFiles = Collections.emptyMap(); private Set multiFileSamples = Collections.emptySet(); private List files = Collections.emptyList(); @@ -129,6 +145,15 @@ public StudyVariantQueryProjection setSamples(List samples) { return this; } + public List getSampleNames() { + return sampleNames; + } + + public StudyVariantQueryProjection setSampleNames(List sampleNames) { + this.sampleNames = sampleNames; + return this; + } + public Map> getMultiFileSampleFiles() { return multiFileSampleFiles; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java index 371903a7626..bdcd501ad88 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java @@ -1,12 +1,14 @@ package org.opencb.opencga.storage.core.variant.query.projection; import org.apache.commons.collections4.CollectionUtils; +import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.FileMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; @@ -53,6 +55,7 @@ public static VariantQueryProjection parseVariantQueryFields( public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOptions options) { Set includeFields = VariantField.getIncludeFields(options); + List events = new ArrayList<>(); List includeStudies = getIncludeStudies(query, options, metadataManager, includeFields); Map studies = new HashMap<>(includeStudies.size()); @@ -68,13 +71,20 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti } Map> sampleIdsMap = getIncludeSampleIds(query, options, includeStudies, metadataManager); - for (VariantQueryProjection.StudyVariantQueryProjection study : studies.values()) { - study.setSamples(sampleIdsMap.get(study.getId())); - } int numTotalSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); skipAndLimitSamples(query, sampleIdsMap); int numSamples = sampleIdsMap.values().stream().mapToInt(List::size).sum(); + for (VariantQueryProjection.StudyVariantQueryProjection study : studies.values()) { + List sampleIds = sampleIdsMap.get(study.getId()); + study.setSamples(sampleIds); + List sampleNames = new ArrayList<>(sampleIds.size()); + for (Integer sampleId : sampleIds) { + sampleNames.add(metadataManager.getSampleName(study.getId(), sampleId)); + } + study.setSampleNames(sampleNames); + } + Map> fileIdsMap = getIncludeFiles(query, includeStudies, includeFields, metadataManager, sampleIdsMap); for (VariantQueryProjection.StudyVariantQueryProjection study : studies.values()) { @@ -125,22 +135,35 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti for (VariantQueryProjection.StudyVariantQueryProjection study : studies.values()) { int studyId = study.getId(); List cohorts = new LinkedList<>(); - for (CohortMetadata cohort : metadataManager.getCalculatedCohorts(studyId)) { + for (CohortMetadata cohort : metadataManager.getCalculatedOrPartialCohorts(studyId)) { cohorts.add(cohort.getId()); + TaskMetadata.Status status = cohort.getStatsStatus(); + if (status == TaskMetadata.Status.ERROR) { + String message = "Please note that the Cohort Stats for " + + "'" + study.getName() + ":" + cohort.getName() + "' are currently outdated."; + int numSampmles = cohort.getSamples().size(); + int invalidStatsNumSamples = cohort.getAttributes().getInt(CohortMetadata.INVALID_STATS_NUM_SAMPLES, -1); + if (invalidStatsNumSamples > 0) { + message += " The statistics have been calculated with " + invalidStatsNumSamples + " samples, " + + "while the total number of samples in the cohort is " + numSampmles + "."; + } + message += " To display updated statistics, please execute variant-stats-index."; + events.add(new Event(Event.Type.WARNING, message)); + } else if (status == TaskMetadata.Status.RUNNING) { + String message = "Please note that the Cohort Stats for " + + "'" + study.getName() + ":" + cohort.getName() + "' are currently being calculated."; + events.add(new Event(Event.Type.WARNING, message)); + } } -// metadataManager.cohortIterator(studyId).forEachRemaining(cohort -> { -// if (cohort.isReady()/* || cohort.isInvalid()*/) { -// cohorts.add(cohort.getId()); -// } -// }); study.setCohorts(cohorts); } } - return new VariantQueryProjection(includeFields, studies, numTotalSamples != numSamples, numSamples, numTotalSamples); + return new VariantQueryProjection(includeFields, studies, numTotalSamples != numSamples, numSamples, numTotalSamples) + .setEvents(events); } - public static void skipAndLimitSamples(Query query, Map> sampleIds) { + private void skipAndLimitSamples(Query query, Map> sampleIds) { if (VariantQueryUtils.isValidParam(query, VariantQueryParam.SAMPLE_SKIP)) { int skip = query.getInt(VariantQueryParam.SAMPLE_SKIP.key()); if (skip > 0) { @@ -483,6 +506,10 @@ public static boolean isIncludeSamplesDefined(Query query, Set fie return getIncludeSamplePartialStatus(query, fields) != null || getIncludeFilePartialStatus(query, fields) != null; } + /* + * @deprecated use VariantQueryProjection.getSampleNames() + */ + @Deprecated public static Map> getIncludeSampleNames(Query query, QueryOptions options, VariantStorageMetadataManager metadataManager) { if (VariantField.getIncludeFields(options).contains(VariantField.STUDIES)) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java index dc278d1e3db..200b2eb463d 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SamplesSearchIndexVariantQueryExecutor.java @@ -1,7 +1,5 @@ package org.opencb.opencga.storage.core.variant.search; -import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -10,11 +8,10 @@ import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import java.io.IOException; -import java.util.Collections; -import java.util.concurrent.TimeUnit; import static org.opencb.opencga.storage.core.variant.search.VariantSearchUtils.inferSpecificSearchIndexSamplesCollection; @@ -37,19 +34,9 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) throws Stor } @Override - public DataResult count(Query query) { - try { - StopWatch watch = StopWatch.createStarted(); - long count = searchManager.count(dbName, query); - int time = (int) watch.getTime(TimeUnit.MILLISECONDS); - return new DataResult<>(time, Collections.emptyList(), 1, Collections.singletonList(count), 1); - } catch (IOException | VariantSearchException e) { - throw new VariantQueryException("Error querying Solr", e); - } - } - - @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator) throws StorageEngineException { + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) throws StorageEngineException { + Query query = variantQuery.getQuery(); + QueryOptions options = variantQuery.getInputOptions(); String specificSearchIndexSamples = inferSpecificSearchIndexSamplesCollection(query, options, getMetadataManager(), dbName); if (specificSearchIndexSamples == null) { @@ -59,7 +46,7 @@ protected Object getOrIterator(Query query, QueryOptions options, boolean iterat if (iterator) { return searchManager.iterator(specificSearchIndexSamples, query, options); } else { - return searchManager.query(specificSearchIndexSamples, query, options); + return searchManager.query(specificSearchIndexSamples, variantQuery); } } catch (IOException | VariantSearchException e) { throw VariantQueryException.internalException(e); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantAggregationExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantAggregationExecutor.java index cd466046999..eba762ab0af 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantAggregationExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantAggregationExecutor.java @@ -4,7 +4,7 @@ import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.executors.VariantAggregationExecutor; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import org.slf4j.Logger; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java index 137c48cd163..12c86cc4e2b 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/SearchIndexVariantQueryExecutor.java @@ -8,7 +8,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.config.storage.StorageConfiguration; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -16,6 +15,8 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.search.solr.SolrNativeIterator; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; @@ -80,29 +81,16 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) throws Stor } @Override - public DataResult count(Query query) { - try { - StopWatch watch = StopWatch.createStarted(); - long count = searchManager.count(dbName, query); - int time = (int) watch.getTime(TimeUnit.MILLISECONDS); - return new DataResult<>(time, Collections.emptyList(), 1, Collections.singletonList(count), 1); - } catch (IOException | VariantSearchException e) { - throw new VariantQueryException("Error querying Solr", e); - } - } - - @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator) { - if (options == null) { - options = QueryOptions.empty(); - } + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { + Query query = variantQuery.getQuery(); + QueryOptions options = variantQuery.getInputOptions(); if (doQuerySearchManager(query, options)) { try { if (iterator) { return searchManager.iterator(dbName, query, options); } else { - return searchManager.query(dbName, query, options); + return searchManager.query(dbName, variantQuery); } } catch (IOException | VariantSearchException e) { throw new VariantQueryException("Error querying Solr", e); @@ -134,7 +122,7 @@ protected Object getOrIterator(Query query, QueryOptions options, boolean iterat approxCount = false; } else if (options.getBoolean(APPROXIMATE_COUNT.key()) || options.getBoolean(QueryOptions.COUNT)) { options.put(QueryOptions.COUNT, false); - VariantQueryResult result = approximateCount(query, options); + VariantQueryResult result = approximateCount(variantQuery); numTotalResults = result.first(); approxCount = result.getApproximateCount(); approxCountSamplingSize = result.getApproximateCountSamplingSize(); @@ -176,7 +164,9 @@ protected Object getOrIterator(Query query, QueryOptions options, boolean iterat } } - public VariantQueryResult approximateCount(Query query, QueryOptions options) { + public VariantQueryResult approximateCount(ParsedVariantQuery variantQuery) { + Query query = variantQuery.getQuery(); + QueryOptions options = variantQuery.getInputOptions(); long count; boolean approxCount = true; int sampling = 0; @@ -193,7 +183,7 @@ public VariantQueryResult approximateCount(Query query, QueryOptions optio Query searchEngineQuery = getSearchEngineQuery(query); Query engineQuery = getEngineQuery(query, options, getMetadataManager()); - VariantQueryResult nativeResult = searchManager + DataResult nativeResult = searchManager .nativeQuery(dbName, searchEngineQuery, queryOptions); List variantIds = nativeResult.getResults().stream().map(VariantSearchModel::getId).collect(Collectors.toList()); // Adjust numSamples if the results from SearchManager is smaller than numSamples @@ -202,7 +192,7 @@ public VariantQueryResult approximateCount(Query query, QueryOptions optio approxCount = false; sampling = variantIds.size(); } - long numSearchResults = nativeResult.getNumTotalResults(); + long numSearchResults = nativeResult.getNumMatches(); long numResults; if (variantIds.isEmpty()) { @@ -223,8 +213,8 @@ public VariantQueryResult approximateCount(Query query, QueryOptions optio throw new VariantQueryException("Error querying Solr", e); } int time = (int) watch.getTime(TimeUnit.MILLISECONDS); - return new VariantQueryResult<>(time, 1, 1, Collections.emptyList(), Collections.singletonList(count), null, - SEARCH_ENGINE_ID + '+' + getStorageEngineId(), approxCount, approxCount ? sampling : null, null); + return new VariantQueryResult<>(time, 1, 1, Collections.emptyList(), Collections.singletonList(count), + SEARCH_ENGINE_ID + '+' + getStorageEngineId(), approxCount, approxCount ? sampling : null, null, variantQuery); } /** @@ -308,7 +298,7 @@ protected Iterator variantIdIteratorFromSearch(Query query, int limit, i try { // Do not iterate for small queries if (limit < 10000) { - VariantQueryResult nativeResult = searchManager.nativeQuery(dbName, query, queryOptions); + DataResult nativeResult = searchManager.nativeQuery(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeResult.getNumMatches()); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java index a56c3e4c3f1..6f55a892f7c 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/VariantSearchToVariantConverter.java @@ -34,6 +34,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; +import org.opencb.opencga.storage.core.variant.annotation.converters.VariantAnnotationModelUtils; import org.opencb.opencga.storage.core.variant.annotation.converters.VariantTraitAssociationToEvidenceEntryConverter; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.slf4j.Logger; @@ -59,6 +60,8 @@ public class VariantSearchToVariantConverter implements ComplexTypeConverter includeFields; + private final VariantAnnotationModelUtils variantAnnotationModelUtils = new VariantAnnotationModelUtils(); + public VariantSearchToVariantConverter() { this.includeFields = null; } @@ -666,19 +669,6 @@ public VariantSearchModel convertToStorageType(Variant variant) { variantSearchModel.setEnd(variant.getEnd()); variantSearchModel.setType(variant.getType().toString()); - // This field contains all possible IDs: id, dbSNP, names, genes, transcripts, protein, clinvar, hpo, ... - // This will help when searching by variant id. This is added at the end of the method after collecting all IDs - Set xrefs = new HashSet<>(); - xrefs.add(variantSearchModel.getId()); - xrefs.add(variantSearchModel.getVariantId()); - if (variant.getNames() != null && !variant.getNames().isEmpty()) { - variant.getNames().forEach(name -> { - if (name != null) { - xrefs.add(name); - } - }); - } - // convert Study related information convertStudies(variant, variantSearchModel, other); @@ -698,11 +688,6 @@ public VariantSearchModel convertToStorageType(Variant variant) { // Process Variant Annotation VariantAnnotation variantAnnotation = variant.getAnnotation(); if (variantAnnotation != null) { - - if (StringUtils.isNotEmpty(variantAnnotation.getId())) { - xrefs.add(variantAnnotation.getId()); - } - // This object will store all info and descriptions for full-text search Set traits = new HashSet<>(); @@ -721,27 +706,6 @@ public VariantSearchModel convertToStorageType(Variant variant) { } variantSearchModel.setRelease(release); - // Add cytoband names - if (variantAnnotation.getCytoband() != null) { - for (Cytoband cytoband : variantAnnotation.getCytoband()) { - xrefs.add(cytoband.getChromosome() + cytoband.getName()); - } - } - - // Add all XRefs coming from the variant annotation - if (variantAnnotation.getXrefs() != null && !variantAnnotation.getXrefs().isEmpty()) { - variantAnnotation.getXrefs().forEach(xref -> { - if (xref != null) { - xrefs.add(xref.getId()); - } - }); - } - - // Add all HGVS coming from the variant annotation - if (ListUtils.isNotEmpty(variantAnnotation.getHgvs())) { - xrefs.addAll(variantAnnotation.getHgvs()); - } - // Set Genes and Consequence Types List consequenceTypes = variantAnnotation.getConsequenceTypes(); if (consequenceTypes != null) { @@ -782,10 +746,6 @@ public VariantSearchModel convertToStorageType(Variant variant) { } } - xrefs.add(gene); - xrefs.add(conseqType.getGeneId()); - xrefs.add(conseqType.getTranscriptId()); - if (StringUtils.isNotEmpty(conseqType.getBiotype())) { biotypes.add(conseqType.getBiotype()); @@ -860,19 +820,16 @@ public VariantSearchModel convertToStorageType(Variant variant) { trans.append(FIELD_SEP); if (StringUtils.isNotEmpty(protVarAnnotation.getUniprotAccession())) { trans.append(protVarAnnotation.getUniprotAccession()); - xrefs.add(protVarAnnotation.getUniprotAccession()); } trans.append(FIELD_SEP); if (StringUtils.isNotEmpty(protVarAnnotation.getUniprotName())) { trans.append(protVarAnnotation.getUniprotName()); - xrefs.add(protVarAnnotation.getUniprotName()); } trans.append(FIELD_SEP); if (StringUtils.isNotEmpty(protVarAnnotation.getUniprotVariantId())) { trans.append(protVarAnnotation.getUniprotVariantId()); - xrefs.add(protVarAnnotation.getUniprotVariantId()); } trans.append(FIELD_SEP).append(protVarAnnotation.getPosition() == null @@ -913,8 +870,6 @@ public VariantSearchModel convertToStorageType(Variant variant) { if (protVarAnnotation.getFeatures() != null) { for (ProteinFeature proteinFeature : protVarAnnotation.getFeatures()) { if (StringUtils.isNotEmpty(proteinFeature.getId())) { - // We store them in xrefs and traits, the number of these IDs is very small - xrefs.add(proteinFeature.getId()); traits.add("PD" + FIELD_SEP + proteinFeature.getId() + FIELD_SEP + proteinFeature.getDescription()); } @@ -997,9 +952,6 @@ public VariantSearchModel convertToStorageType(Variant variant) { List clinical = VariantQueryUtils.buildClinicalCombinations(variantAnnotation); for (EvidenceEntry ev : variantAnnotation.getTraitAssociation()) { if (ev.getSource() != null && StringUtils.isNotEmpty(ev.getSource().getName())) { - if (StringUtils.isNotEmpty(ev.getId())) { - xrefs.add(ev.getId()); - } if ("clinvar".equalsIgnoreCase(ev.getSource().getName())) { String clinSigSuffix = ""; if (ev.getVariantClassification() != null @@ -1069,6 +1021,18 @@ public VariantSearchModel convertToStorageType(Variant variant) { variantSearchModel.setOther(other); } + // This field contains all possible IDs: id, dbSNP, names, genes, transcripts, protein, clinvar, hpo, ... + // This will help when searching by variant id. This is added at the end of the method after collecting all IDs + Set xrefs = variantAnnotationModelUtils.extractXRefs(variant.getAnnotation()); + xrefs.add(variantSearchModel.getId()); + xrefs.add(variantSearchModel.getVariantId()); + if (variant.getNames() != null && !variant.getNames().isEmpty()) { + variant.getNames().forEach(name -> { + if (name != null) { + xrefs.add(name); + } + }); + } variantSearchModel.setXrefs(new ArrayList<>(xrefs)); return variantSearchModel; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java index 74ee51bbe3a..374f5f9ffd6 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java @@ -42,7 +42,8 @@ import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.utils.CellBaseUtils; @@ -278,25 +279,24 @@ public int delete(String collection, VariantDBIterator variantDBIterator, Progre * according a given query. * * @param collection Collection name - * @param query Query - * @param queryOptions Query options + * @param variantQuery Parsed variant query * @return List of Variant objects * @throws VariantSearchException VariantSearchException * @throws IOException IOException */ - public VariantQueryResult query(String collection, Query query, QueryOptions queryOptions) + public VariantQueryResult query(String collection, ParsedVariantQuery variantQuery) throws VariantSearchException, IOException { - SolrQuery solrQuery = solrQueryParser.parse(query, queryOptions); + SolrQuery solrQuery = solrQueryParser.parse(variantQuery.getQuery(), variantQuery.getInputOptions()); SolrCollection solrCollection = solrManager.getCollection(collection); DataResult queryResult; try { queryResult = solrCollection.query(solrQuery, VariantSearchModel.class, - new VariantSearchToVariantConverter(VariantField.getIncludeFields(queryOptions))); + new VariantSearchToVariantConverter(VariantField.getIncludeFields(variantQuery.getInputOptions()))); } catch (SolrServerException e) { throw new VariantSearchException("Error executing variant query", e); } - return new VariantQueryResult<>(queryResult, null, SEARCH_ENGINE_ID); + return new VariantQueryResult<>(queryResult, SEARCH_ENGINE_ID, variantQuery); } /** @@ -310,7 +310,7 @@ public VariantQueryResult query(String collection, Query query, QueryOp * @throws VariantSearchException VariantSearchException * @throws IOException IOException */ - public VariantQueryResult nativeQuery(String collection, Query query, QueryOptions queryOptions) + public DataResult nativeQuery(String collection, Query query, QueryOptions queryOptions) throws VariantSearchException, IOException { SolrQuery solrQuery = solrQueryParser.parse(query, queryOptions); SolrCollection solrCollection = solrManager.getCollection(collection); @@ -321,7 +321,7 @@ public VariantQueryResult nativeQuery(String collection, Que throw new VariantSearchException("Error executing variant query (nativeQuery)", e); } - return new VariantQueryResult<>(queryResult, null, SEARCH_ENGINE_ID); + return queryResult; } /** diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java index c85c63b1505..db6ead71f58 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/DefaultVariantStatisticsManager.java @@ -474,6 +474,7 @@ protected VariantStatsDBWriter newVariantStatisticsDBWriter(VariantDBAdaptor dbA // // } + @Deprecated void checkAndUpdateCalculatedCohorts(StudyMetadata studyMetadata, URI uri, boolean updateStats) throws IOException, StorageEngineException { Set cohortNames = readCohortsFromStatsFile(uri); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java index fe6765529fa..d4fdc948622 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java @@ -112,6 +112,7 @@ public void postCalculateStats( } } + @Deprecated public static void checkAndUpdateCalculatedCohorts( VariantStorageMetadataManager metadataManager, StudyMetadata studyMetadata, Collection cohorts, boolean updateStats) throws StorageEngineException { diff --git a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml index 226fc106783..b9970d18eaf 100644 --- a/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml +++ b/opencga-storage/opencga-storage-core/src/main/resources/storage-configuration.yml @@ -11,17 +11,15 @@ cellbase: ## URL host to annotate variants, for example: https://uk.ws.zettagenomics.com/cellbase/ url: "${OPENCGA.CELLBASE.REST.HOST}" version: "${OPENCGA.CELLBASE.VERSION}" - dataRelease: "2" + dataRelease: "7" ## Storage Query Server configuration. When CLI is launched in 'server' mode a RESTful web server ## is launched in the specified port. server: rest: port: ${OPENCGA.SERVER.REST.PORT} - logFile: null grpc: port: ${OPENCGA.SERVER.GRPC.PORT} - logFile: null ## Solr Search Configuration search: @@ -173,6 +171,7 @@ variant: #storage.hadoop.mr.executor.ssh.key: "~/.ssh/id_rsa" # Hadoop edge node ssh-key file storage.hadoop.mr.executor.ssh.password: "" # Hadoop edge node password. Only if ssh-key is not present. Requires sshpass to run storage.hadoop.mr.executor.ssh.remoteOpenCgaHome: # Remote opencga home location. Only if different than local location. + storage.hadoop.mr.executor.ssh.terminationGracePeriodSeconds: 120 # Termination grace period in seconds for the ssh executor. # Increase the ScannerTimeoutPeriod from 60000 (1min) to 300000 (5min) to avoid ScannerTimeoutExceptions # See opencb/opencga#352 for more info. diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java index 598e1fae586..32e1657fcb8 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java @@ -1,6 +1,7 @@ package org.opencb.opencga.storage.core.variant; import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -17,6 +18,7 @@ import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; @@ -45,7 +47,7 @@ public abstract class VariantStorageEngineBNDTest extends VariantStorageBaseTest public void before() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); if (!loaded) { clearDB(DB_NAME); loadFiles(); @@ -56,7 +58,7 @@ public void before() throws Exception { protected void loadFiles() throws Exception { variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); studyMetadata = new StudyMetadata(1, "s1"); // variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); input1 = getResourceUri("variant-test-bnd.vcf"); @@ -75,13 +77,13 @@ public void checkAllAnnotated() throws Exception { @Test public void getPairs() throws Exception { - getPairs(new Query()); - getPairs(new Query(VariantQueryParam.REGION.key(), "2")); - getPairs(new Query(VariantQueryParam.REGION.key(), "17")); - getPairs(new Query(VariantQueryParam.REGION.key(), "2,13,5")); - getPairs(new Query(VariantQueryParam.REGION.key(), "2").append(VariantQueryParam.GENE.key(), "VPS53")); - getPairs(new Query(VariantQueryParam.GENE.key(), "VPS53")); - getPairs(new Query(VariantQueryParam.GENE.key(), "BRCA2")); + getPairs(new VariantQuery()); + getPairs(new VariantQuery().region("2")); + getPairs(new VariantQuery().region("17")); + getPairs(new VariantQuery().region("2", "13", "5")); + getPairs(new VariantQuery().region("2").gene("LINC00423")); + getPairs(new VariantQuery().gene("LINC00423")); + getPairs(new VariantQuery().gene("ENSG00000263015")); } public void getPairs(Query inputQuery) throws Exception { @@ -115,8 +117,8 @@ public void getPairs(Query inputQuery) throws Exception { } assertNull(prevMateid); assertNull(prevId); - assertThat(duplicatedVariants, CoreMatchers.not(CoreMatchers.hasItem(CoreMatchers.anything()))); - + MatcherAssert.assertThat(duplicatedVariants, CoreMatchers.not(CoreMatchers.hasItem(CoreMatchers.anything()))); + assertNotEquals(0, variants.size()); // Check pagination testPagination(variantsList, query, 1); testPagination(variantsList, query, 2); @@ -130,17 +132,19 @@ private void testPagination(List variantsList, Query query, int batchSiz List actualVariantsGet = new ArrayList<>(variantsList.size()); List actualVariantsIterator = new ArrayList<>(variantsList.size()); for (int i = 0; i < variantsList.size(); i += batchSize) { + System.out.println(" --- limit = " + batchSize + " skip = " + i); QueryOptions options = new QueryOptions(QueryOptions.LIMIT, batchSize) .append(QueryOptions.SKIP, i); - List results = variantStorageEngine.get(query, options).getResults(); - System.out.println("get = " + options.toJson() + " -> " + results.size() + " " + results); + List results; + results = variantStorageEngine.get(query, options).getResults(); + System.out.println("get = " + options.toJson() + " -> " + results.size() + " result " + results.stream().map(Variant::toString).collect(Collectors.joining("\", \"", "[ \"", "\" ]"))); results.stream().map(Variant::toString).forEach(actualVariantsGet::add); assertTrue(results.size() <= batchSize); results = new ArrayList<>(batchSize); variantStorageEngine.iterator(query, options).forEachRemaining(results::add); - System.out.println("it = " + options.toJson() + " -> " + results.size() + " " + results); - results.stream().map(Variant::toString).forEach(actualVariantsGet::add); + System.out.println("it = " + options.toJson() + " -> " + results.size() + " result " + results.stream().map(Variant::toString).collect(Collectors.joining("\", \"", "[ \"", "\" ]"))); + results.stream().map(Variant::toString).forEach(actualVariantsIterator::add); assertTrue(results.size() <= batchSize); } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java index fe7cebbd6a4..64fc14c4c6a 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineSVTest.java @@ -16,6 +16,7 @@ import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; @@ -39,11 +40,14 @@ public abstract class VariantStorageEngineSVTest extends VariantStorageBaseTest { protected static StudyMetadata studyMetadata; + protected static StudyMetadata studyMetadata2; protected static boolean loaded = false; protected static StoragePipelineResult pipelineResult1; protected static StoragePipelineResult pipelineResult2; + protected static StoragePipelineResult pipelineResult3; protected static URI input1; protected static URI input2; + protected static URI input3; @Before public void before() throws Exception { @@ -66,12 +70,22 @@ protected void loadFiles() throws Exception { variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical"); pipelineResult1 = runDefaultETL(input1, variantStorageEngine, studyMetadata, new QueryOptions() .append(VariantStorageOptions.ANNOTATE.key(), true) + .append(VariantStorageOptions.STATS_CALCULATE.key(), true) .append(VariantStorageOptions.ASSEMBLY.key(), "grch38") ); input2 = getResourceUri("variant-test-sv_2.vcf"); pipelineResult2 = runDefaultETL(input2, variantStorageEngine, studyMetadata, new QueryOptions() .append(VariantStorageOptions.ANNOTATE.key(), true) + .append(VariantStorageOptions.STATS_CALCULATE.key(), true) .append(VariantStorageOptions.ASSEMBLY.key(), "grch38")); + + input3 = getResourceUri("variant-test-sv-large.vcf"); + studyMetadata2 = new StudyMetadata(2, "s2"); + pipelineResult3 = runDefaultETL(input3, variantStorageEngine, studyMetadata2, new QueryOptions() + .append(VariantStorageOptions.ANNOTATE.key(), true) + .append(VariantStorageOptions.STATS_CALCULATE.key(), true) + .append(VariantStorageOptions.ASSEMBLY.key(), "grch38")); + } @Test @@ -88,7 +102,7 @@ public void checkCount() throws Exception { + 1 + 1 // negative cipos ; - int count = variantStorageEngine.getDBAdaptor().count().first().intValue(); + int count = variantStorageEngine.count(new VariantQuery().study(studyMetadata.getName())).first().intValue(); assertEquals(expected, count); } @@ -146,7 +160,10 @@ protected void checkCorrectness(URI file) throws StorageEngineException, NonStan @Test public void exportVcf() throws Exception { - variantStorageEngine.exportData(null, VariantWriterFactory.VariantOutputFormat.VCF, null, new Query(VariantQueryParam.UNKNOWN_GENOTYPE.key(), "./."), new QueryOptions(QueryOptions.SORT, true)); + variantStorageEngine.exportData(null, VariantWriterFactory.VariantOutputFormat.VCF, null, + new VariantQuery().unknownGenotype("./.").study(studyMetadata.getName()), new QueryOptions(QueryOptions.SORT, true)); + variantStorageEngine.exportData(null, VariantWriterFactory.VariantOutputFormat.VCF, null, + new VariantQuery().unknownGenotype("./.").study(studyMetadata2.getName()), new QueryOptions(QueryOptions.SORT, true)); } protected Map readVariants(URI input) throws StorageEngineException, NonStandardCompliantSampleField { diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java index 5e29cd2b127..28b7d0921ab 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java @@ -32,7 +32,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -81,7 +81,7 @@ public void before() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION); - variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38); dbAdaptor = variantStorageEngine.getDBAdaptor(); if (!loaded) { loadFile(); @@ -293,7 +293,7 @@ public void testCount() throws Exception { .append(ANNOT_CONSERVATION.key(), "gerp>0.1"); long realCount = dbAdaptor.count(query).first(); VariantQueryResult result = variantQueryExecutor - .get(query, new QueryOptions(COUNT, true).append(LIMIT, 0)); + .get(variantStorageEngine.parseQuery(query, new QueryOptions(COUNT, true).append(LIMIT, 0))); assertEquals(0, result.getResults().size()); assertEquals(realCount, result.getNumMatches()); } @@ -304,7 +304,7 @@ public void testApproxCount() throws Exception { .append(ANNOT_CONSERVATION.key(), "gerp>0.1"); long realCount = dbAdaptor.count(query).first(); VariantQueryResult result = variantQueryExecutor - .approximateCount(query, new QueryOptions(VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), realCount * 0.1)); + .approximateCount(variantStorageEngine.getVariantQueryParser().parseQuery(query, new QueryOptions(VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), realCount * 0.1))); long approxCount = result.first(); System.out.println("approxCount = " + approxCount); System.out.println("realCount = " + realCount); @@ -319,7 +319,7 @@ public void testExactApproxCount() throws Exception { .append(ANNOT_CONSERVATION.key(), "gerp>0.1"); long realCount = dbAdaptor.count(query).first(); VariantQueryResult result = variantQueryExecutor - .approximateCount(query, new QueryOptions(VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), allVariants.getNumResults())); + .approximateCount(variantStorageEngine.getVariantQueryParser().parseQuery(query, new QueryOptions(VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), allVariants.getNumResults()))); long approxCount = result.first(); System.out.println("approxCount = " + approxCount); System.out.println("realCount = " + realCount); @@ -333,7 +333,7 @@ public void testExactApproxCountToSearch() throws Exception { Query query = new Query(ANNOT_CONSERVATION.key(), "gerp>0.1"); long realCount = dbAdaptor.count(query).first(); VariantQueryResult result = variantQueryExecutor - .approximateCount(query, new QueryOptions(VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), allVariants.getNumResults())); + .approximateCount(variantStorageEngine.getVariantQueryParser().parseQuery(query, new QueryOptions(VariantStorageOptions.APPROXIMATE_COUNT_SAMPLING_SIZE.key(), allVariants.getNumResults()))); long approxCount = result.first(); System.out.println("approxCount = " + approxCount); System.out.println("realCount = " + realCount); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileSpecificSamplesCollectionTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileSpecificSamplesCollectionTest.java index 4c267a35d15..58cc058dc4b 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileSpecificSamplesCollectionTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileSpecificSamplesCollectionTest.java @@ -5,7 +5,8 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -55,19 +56,19 @@ protected void load() throws Exception { protected VariantQueryResult query(Query query, QueryOptions options) { try { - query = variantStorageEngine.preProcessQuery(query, options); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); VariantStorageMetadataManager scm = dbAdaptor.getMetadataManager(); - String collection = VariantSearchUtils.inferSpecificSearchIndexSamplesCollection(query, options, scm, DB_NAME); + String collection = VariantSearchUtils.inferSpecificSearchIndexSamplesCollection(variantQuery.getQuery(), options, scm, DB_NAME); // Do not execute this test if the query is not covered by the specific search index collection - Assume.assumeThat(query.toJson(), collection, CoreMatchers.notNullValue()); + Assume.assumeThat(variantQuery.getQuery().toJson(), collection, CoreMatchers.notNullValue()); if (options.getInt(QueryOptions.LIMIT, 0) <= 0) { options = new QueryOptions(options); options.put(QueryOptions.LIMIT, 100000); } - return variantStorageEngine.getVariantSearchManager().query(collection, query, options); + return variantStorageEngine.getVariantSearchManager().query(collection, variantQuery); } catch (StorageEngineException | VariantSearchException | IOException e) { e.printStackTrace(); Assert.fail(e.getMessage()); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java index 6291344e0e3..905f64a5373 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorMultiFileTest.java @@ -10,7 +10,8 @@ import org.opencb.biodata.models.variant.avro.SampleEntry; import org.opencb.biodata.models.variant.stats.VariantStats; import org.opencb.commons.datastore.core.*; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.FileMetadata; @@ -111,9 +112,8 @@ protected void load() throws Exception { } protected VariantQueryResult query(Query query, QueryOptions options) { - options = options == null ? QueryOptions.empty() : options; - query = variantStorageEngine.preProcessQuery(query, options); - return dbAdaptor.get(query, options); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); + return dbAdaptor.get(variantQuery); } protected ObjectMap getOptions() { @@ -378,13 +378,13 @@ public void testSampleLimitSkip() throws Exception { System.out.println("samples(ALL) = " + result.getSamples()); for (int i : new int[]{1, 3, 6, 8, 10}) { - result = query(new Query(VariantQueryParam.SAMPLE_SKIP.key(), i).append(VariantQueryParam.INCLUDE_SAMPLE.key(), ALL).append(SAMPLE_METADATA.key(), true), options); + result = query(new VariantQuery().sampleSkip(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(SKIP=" + i + ") = " + result.getSamples()); assertEquals(Math.max(0, 8 - i), result.getSamples().values().stream().mapToInt(List::size).sum()); assertEquals(Math.max(0, 8 - i), result.getNumSamples().intValue()); assertEquals(8, result.getNumTotalSamples().intValue()); - result = query(new Query(VariantQueryParam.SAMPLE_LIMIT.key(), i).append(VariantQueryParam.INCLUDE_SAMPLE.key(), ALL).append(SAMPLE_METADATA.key(), true), options); + result = query(new VariantQuery().sampleLimit(i).includeSampleAll().sampleMetadata(true), options); // System.out.println("samples(LIMIT=" + i + ") = " + result.getSamples()); assertEquals(Math.min(8, i), result.getSamples().values().stream().mapToInt(List::size).sum()); assertEquals(Math.min(8, i), result.getNumSamples().intValue()); @@ -1329,6 +1329,16 @@ public void testSampleData() throws Exception { assertEquals(0, variant.getStudies().get(0).getFiles().size()); } + @Test + public void testSampleDataUnnormalized() throws Exception { + // Check unnormalized queries + Variant variant = variantStorageEngine.getSampleData("1:10352:T:TA", study1, new QueryOptions()).first(); + assertEquals("1:10353:-:A", variant.toString()); + System.out.println("variant = " + variant.toJson()); + assertNotNull(variant.getStudies().get(0).getStats(DEFAULT_COHORT)); + assertEquals(4, variant.getStudies().get(0).getSamples().size()); + assertEquals(4, variant.getStudies().get(0).getFiles().size()); + } @Test public void testCount() throws StorageEngineException { diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorTest.java index 1f305574a64..984cc5bfb83 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantDBAdaptorTest.java @@ -30,14 +30,11 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.avro.*; -import org.opencb.biodata.models.variant.exceptions.NonStandardCompliantSampleField; import org.opencb.biodata.models.variant.stats.VariantStats; -import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -47,6 +44,8 @@ import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.annotation.annotators.CellBaseRestVariantAnnotator; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.executors.NoOpVariantQueryExecutor; import org.opencb.opencga.storage.core.variant.query.filters.VariantFilterBuilder; @@ -268,8 +267,8 @@ public void multiIterator() throws Exception { VariantDBIterator iterator = dbAdaptor.iterator(variantsToQuery.iterator(), new Query(), new QueryOptions()); - DataResult queryResult = iterator.toDataResult(); - assertEquals(variantsToQuery.size(), queryResult.getResults().size()); + List variants = iterator.toList(); + assertEquals(variantsToQuery.size(), variants.size()); } @Test @@ -531,11 +530,14 @@ public long filterPopulation(DataResult queryResult, Predicate public void testGetAllVariants_variantId() { int i = 0; List variants = new ArrayList<>(); + Map normalizedVariants = new HashMap<>(); for (Variant variant : allVariants.getResults()) { - if (i++ % 10 == 0) { - if (!variant.isSymbolic()) { - variants.add(variant); - } + if ((i++ % 10) == 0) { + variants.add(variant); + } + OriginalCall call = variant.getStudies().get(0).getFiles().get(0).getCall(); + if (call != null) { + normalizedVariants.put(variant.toString(), call.getVariantId()); } } List result = query(new Query(ID.key(), variants), new QueryOptions()).getResults(); @@ -554,32 +556,61 @@ public void testGetAllVariants_variantId() { } } assertEquals(expectedList, actualList); + + normalizedVariants.forEach((key, value) -> { + System.out.println(key + " = " + value); + }); + List resultNormalized = query(new Query(ID.key(), normalizedVariants.values()).append(INCLUDE_FILE.key(), ALL), new QueryOptions()).getResults(); + assertEquals(normalizedVariants.size(), resultNormalized.size()); + assertTrue(!resultNormalized.isEmpty()); + for (Variant variant : resultNormalized) { + String expected = normalizedVariants.get(variant.toString()); + String actual = variant.getStudies().get(0).getFiles().get(0).getCall().getVariantId(); + assertEquals(expected, actual); + } } @Test public void testGetAllVariants_xref() { - Query query = new Query(ANNOT_XREF.key(), "3:108634973:C:A,rs2032582,HP:0001250,VAR_048225,Q9BY64,ENSG00000250026,TMPRSS11B,COSM1421316"); - queryResult = query(query, null); - assertThat(queryResult, everyResult(allVariantsSummary, anyOf( - hasAnnotation(at("3:108634973:C:A")), - with("id", Variant::getId, is("rs2032582")), - hasAnnotation(with("GeneTraitAssociation", VariantAnnotation::getGeneTraitAssociation, - hasItem(with("HPO", GeneTraitAssociation::getHpo, is("HP:0001250"))))), - hasAnnotation(with("ConsequenceType", VariantAnnotation::getConsequenceTypes, - hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, - with("UniprotVariantId", ProteinVariantAnnotation::getUniprotVariantId, is("VAR_048225")))))), - hasAnnotation(with("ConsequenceType", VariantAnnotation::getConsequenceTypes, - hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, - with("UniprotName", ProteinVariantAnnotation::getUniprotAccession, is("Q9BY64")))))), - hasAnnotation(with("ConsequenceType", VariantAnnotation::getConsequenceTypes, - hasItem(with("EnsemblGene", ConsequenceType::getGeneId, is("ENSG00000250026"))))), - hasAnnotation(with("ConsequenceType", VariantAnnotation::getConsequenceTypes, - hasItem(with("GeneName", ConsequenceType::getGeneName, is("TMPRSS11B"))))) -// hasAnnotation(with("VariantTraitAssociation", VariantAnnotation::getVariantTraitAssociation, -// with("Cosmic", VariantTraitAssociation::getCosmic, -// hasItem(with("MutationId", Cosmic::getMutationId, is("COSM1421316")))))) - ))); + Map> matchers = new HashMap<>(); + matchers.put("3:108634973:C:A", hasAnnotation(at("3:108634973:C:A"))); + matchers.put("rs2032582", with("id", Variant::getId, is("rs2032582"))); + matchers.put("HP:0001250", hasAnnotation(with("GeneTraitAssociation", VariantAnnotation::getGeneTraitAssociation, + hasItem(with("HPO", GeneTraitAssociation::getHpo, is("HP:0001250")))))); + matchers.put("VAR_048225", hasAnnotation(with("ConsequenceType", VariantAnnotation::getConsequenceTypes, + hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, + with("UniprotVariantId", ProteinVariantAnnotation::getUniprotVariantId, is("VAR_048225"))))))); + matchers.put("Q9BY64", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( + with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, + with("UniprotAccession", ProteinVariantAnnotation::getUniprotAccession, + is("Q9BY64"))))))); + matchers.put("ENSG00000250026", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( + with("GeneId", ConsequenceType::getGeneId, + is("ENSG00000250026")))))); + matchers.put("TMPRSS11B", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( + with("GeneName", ConsequenceType::getGeneName, + is("TMPRSS11B")))))); + matchers.put("COSM1421316", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem( + with("Cosmic", EvidenceEntry::getId, is("COSM1421316")))))); + + // Query one by one + for (Map.Entry> entry : matchers.entrySet()) { + Query query = new Query(ANNOT_XREF.key(), entry.getKey()); + queryResult = query(query, null); + assertThat(entry.getKey(), queryResult, everyResult(allVariantsSummary, allOf(entry.getValue()))); + query = new Query(ID.key(), entry.getKey()); + queryResult = query(query, null); + assertThat(entry.getKey(), queryResult, everyResult(allVariantsSummary, allOf(entry.getValue()))); + } + + // Query by all xrefs + Query query = new Query(ANNOT_XREF.key(), matchers.keySet()); + queryResult = query(query, null); + assertThat(queryResult, everyResult(allVariantsSummary, anyOf(matchers.values()))); } @Test @@ -2420,7 +2451,8 @@ private void testQuery(Query query, QueryOptions options) { } private Matcher withFilter(Query query) { - return VariantMatchers.withFilter(new VariantFilterBuilder(metadataManager).buildFilter(query, null), query.toJson()); + ParsedVariantQuery parsedVariantQuery = variantStorageEngine.parseQuery(query, new QueryOptions()); + return VariantMatchers.withFilter(new VariantFilterBuilder().buildFilter(parsedVariantQuery), query.toJson()); } /* @Test diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantMatchers.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantMatchers.java index 7f790ffe609..11c9a6f8927 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantMatchers.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantMatchers.java @@ -139,6 +139,20 @@ public static Matcher overlaps(Variant variant) { return overlaps(new Region(variant.getChromosome(), variant.getStart(), variant.getEnd()), true); } + public static Matcher samePosition(Variant variant) { + return new TypeSafeDiagnosingMatcher() { + @Override + protected boolean matchesSafely(Variant item, Description mismatchDescription) { + return variant.sameGenomicVariant(item); + } + + @Override + public void describeTo(Description description) { + description.appendText("has same genomic region " + variant); + } + }; + } + public static Matcher overlaps(Region region) { return overlaps(region, true); } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryTest.java index de17c52a8a0..5cfdde4ca13 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryTest.java @@ -28,6 +28,12 @@ public void test() throws Exception { if (param.type() == QueryParam.Type.BOOLEAN || param.type() == QueryParam.Type.BOOLEAN_ARRAY) { methodSet = getMethodSafe(param.key(), boolean.class); expectedValue = true; + } else if (param.type() == QueryParam.Type.INTEGER || param.type() == QueryParam.Type.INTEGER_ARRAY) { + methodSet = getMethodSafe(param.key(), int.class); + expectedValue = 42; + if (methodSet == null) { + methodSet = getMethodSafe(param.key(), Integer.class); + } } else { expectedValue = RandomStringUtils.random(10); methodSet = getMethodSafe(param.key(), String.class); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java index 395e41d2675..6194d25c6d8 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/VariantQueryUsingSearchIndexTest.java @@ -6,7 +6,7 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java index b36e8fa90ad..8491d03b41e 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java @@ -4,7 +4,9 @@ import org.opencb.biodata.models.variant.avro.AdditionalAttribute; import org.opencb.biodata.models.variant.avro.ConsequenceType; import org.opencb.biodata.models.variant.avro.VariantAnnotation; +import org.opencb.biodata.models.variant.avro.Xref; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.CryptoUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import org.opencb.opencga.storage.core.variant.annotation.annotators.VariantAnnotator; @@ -29,6 +31,10 @@ public DummyTestAnnotator(StorageConfiguration configuration, ProjectMetadata pr fail = options.getBoolean(FAIL, false); } + static String getRs(Variant variant) { + return "rs" + variant.toString().hashCode(); + } + @Override public List annotate(List variants) throws VariantAnnotatorException { if (fail) { @@ -48,6 +54,7 @@ public List annotate(List variants) throws VariantAn ct.setExonOverlap(Collections.emptyList()); ct.setTranscriptFlags(Collections.emptyList()); a.setConsequenceTypes(Collections.singletonList(ct)); + a.setXrefs(Collections.singletonList(new Xref(getRs(v), "dbSNP"))); a.setAdditionalAttributes( Collections.singletonMap(GROUP_NAME.key(), new AdditionalAttribute(Collections.singletonMap(VARIANT_ID.key(), v.toString())))); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java index 68461f5d9c2..ba33f63dcff 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java @@ -4,6 +4,7 @@ import org.apache.commons.lang.StringUtils; import org.junit.Assume; import org.junit.Test; +import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.EvidenceEntry; import org.opencb.biodata.models.variant.avro.VariantAnnotation; import org.opencb.commons.datastore.core.DataResult; @@ -452,6 +453,10 @@ public void testQueries(VariantStorageEngine variantStorageEngine) throws Storag assertThat(annotation.getConsequenceTypes(), VariantMatchers.isEmpty()); } + for (Variant variant : variantStorageEngine) { + Variant thisVariant = variantStorageEngine.getVariant(DummyTestAnnotator.getRs(variant)); + assertThat(thisVariant, VariantMatchers.samePosition(variant)); + } // Get annotations from a deleted snapshot thrown.expectMessage("Variant Annotation snapshot \"v1\" not found!"); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtilsTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtilsTest.java new file mode 100644 index 00000000000..f1167213be9 --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/converters/VariantAnnotationModelUtilsTest.java @@ -0,0 +1,46 @@ +package org.opencb.opencga.storage.core.variant.annotation.converters; + +import junit.framework.TestCase; +import org.junit.experimental.categories.Category; +import org.opencb.biodata.models.variant.avro.ConsequenceType; +import org.opencb.biodata.models.variant.avro.VariantAnnotation; +import org.opencb.biodata.models.variant.avro.Xref; +import org.opencb.opencga.core.testclassification.duration.ShortTests; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; + +@Category(ShortTests.class) +public class VariantAnnotationModelUtilsTest extends TestCase { + + public void testXrefsHgvs() throws Exception { + VariantAnnotation variantAnnotation = new VariantAnnotation(); + variantAnnotation.setId("id"); + variantAnnotation.setXrefs(Collections.singletonList(new Xref("xref1", "source"))); + variantAnnotation.setHgvs(Arrays.asList( + "ENST00000680783.1(ENSG00000135744):c.776T>C", + "ENSP00000451720.1:p.Asn134Lys" + )); + ConsequenceType ct = new ConsequenceType(); + ct.setGeneName("GENE"); + ct.setHgvs(variantAnnotation.getHgvs()); + ct.setGeneId(null); + variantAnnotation.setConsequenceTypes(Arrays.asList(ct, new ConsequenceType())); + Set xrefs = new VariantAnnotationModelUtils().extractXRefs(variantAnnotation); + + assertEquals(7, xrefs.size()); + // Default fields + assertTrue(xrefs.contains("id")); + assertTrue(xrefs.contains("xref1")); + assertTrue(xrefs.contains("GENE")); + + // Untouched hgvs, not starting with ENST or NM_ + assertTrue(xrefs.contains("ENSP00000451720.1:p.Asn134Lys")); + + assertTrue(xrefs.contains("ENST00000680783.1(ENSG00000135744):c.776T>C")); + assertTrue(xrefs.contains("ENST00000680783.1:c.776T>C")); + assertTrue(xrefs.contains("GENE:c.776T>C")); + + } +} \ No newline at end of file diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyStudyMetadataDBAdaptor.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyStudyMetadataDBAdaptor.java index f954026b234..3d519030b19 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyStudyMetadataDBAdaptor.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyStudyMetadataDBAdaptor.java @@ -222,7 +222,7 @@ public Iterator taskIterator(int studyId, List get(ParsedVariantQuery query, QueryOptions options) { + public VariantQueryResult get(ParsedVariantQuery query) { List variants = new ArrayList<>(); - iterator(query, options).forEachRemaining(variants::add); + iterator(query).forEachRemaining(variants::add); - return new VariantQueryResult<>(0, variants.size(), variants.size(), Collections.emptyList(), variants, null, - DummyVariantStorageEngine.STORAGE_ENGINE_ID); + return new VariantQueryResult<>(0, variants.size(), variants.size(), Collections.emptyList(), variants, + DummyVariantStorageEngine.STORAGE_ENGINE_ID, query); } @Override @@ -102,7 +102,8 @@ public DataResult distinct(Query query, String field) { } @Override - public VariantDBIterator iterator(ParsedVariantQuery variantQuery, QueryOptions options) { + public VariantDBIterator iterator(ParsedVariantQuery variantQuery) { + QueryOptions options = variantQuery.getInputOptions(); logger.info("Query " + variantQuery.getQuery().toJson()); logger.info("QueryOptions " + options.toJson()); logger.info("dbName " + dbName); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java index 8fae5f12310..2611cfc337a 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java @@ -101,10 +101,17 @@ public static Set set(T... ts) { @Test public void testParseClinicalCombinations() { - assertEquals(Arrays.asList("cosmic"), VariantQueryParser.parseClinicalCombinationsList(new Query(ANNOT_CLINICAL.key(), "cosmic"))); - assertEquals(Arrays.asList("clinvar", "cosmic"), VariantQueryParser.parseClinicalCombinationsList(new Query(ANNOT_CLINICAL.key(), "clinvar,cosmic"))); - assertEquals(Arrays.asList("cosmic_pathogenic"), VariantQueryParser.parseClinicalCombinationsList(new Query(ANNOT_CLINICAL.key(), "cosmic").append(ANNOT_CLINICAL_SIGNIFICANCE.key(), "pathogenic"))); - assertEquals(Arrays.asList("cosmic_confirmed"), VariantQueryParser.parseClinicalCombinationsList(new Query(ANNOT_CLINICAL.key(), "cosmic").append(ANNOT_CLINICAL_CONFIRMED_STATUS.key(), true))); - assertEquals(Arrays.asList("clinvar_pathogenic_confirmed", "clinvar_likely_pathogenic_confirmed", "cosmic_pathogenic_confirmed", "cosmic_likely_pathogenic_confirmed"), VariantQueryParser.parseClinicalCombinationsList(new Query(ANNOT_CLINICAL.key(), "clinvar,cosmic").append(ANNOT_CLINICAL_SIGNIFICANCE.key(), "pathogenic,likely_pathogenic").append(ANNOT_CLINICAL_CONFIRMED_STATUS.key(), true))); + assertEquals(Arrays.asList("cosmic"), variantQueryParser.parseQuery( + new Query(ANNOT_CLINICAL.key(), "cosmic"), new QueryOptions()).getClinicalCombinationsList()); + assertEquals(Arrays.asList("clinvar", "cosmic"), variantQueryParser.parseQuery( + new Query(ANNOT_CLINICAL.key(), "clinvar,cosmic"), new QueryOptions()).getClinicalCombinationsList()); + assertEquals(Arrays.asList("cosmic_pathogenic"), variantQueryParser.parseQuery( + new Query(ANNOT_CLINICAL.key(), "cosmic").append(ANNOT_CLINICAL_SIGNIFICANCE.key(), "pathogenic"), new QueryOptions()).getClinicalCombinationsList()); + assertEquals(Arrays.asList("cosmic_confirmed"), variantQueryParser.parseQuery( + new Query(ANNOT_CLINICAL.key(), "cosmic").append(ANNOT_CLINICAL_CONFIRMED_STATUS.key(), true), new QueryOptions()).getClinicalCombinationsList()); + assertEquals(Arrays.asList("clinvar_pathogenic_confirmed", "clinvar_likely_pathogenic_confirmed", "cosmic_pathogenic_confirmed", "cosmic_likely_pathogenic_confirmed"), variantQueryParser.parseQuery(new Query() + .append(ANNOT_CLINICAL.key(), "clinvar,cosmic") + .append(ANNOT_CLINICAL_SIGNIFICANCE.key(), "pathogenic,likely_pathogenic") + .append(ANNOT_CLINICAL_CONFIRMED_STATUS.key(), true), new QueryOptions()).getClinicalCombinationsList()); } } \ No newline at end of file diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java new file mode 100644 index 00000000000..1fc7617422c --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java @@ -0,0 +1,290 @@ +package org.opencb.opencga.storage.core.variant.query.executors; + +import org.hamcrest.Matcher; +import org.junit.*; +import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.VariantFileMetadata; +import org.opencb.biodata.models.variant.avro.*; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.storage.core.StoragePipelineResult; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; +import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; +import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; +import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; +import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource; +import org.opencb.opencga.storage.core.variant.stats.DefaultVariantStatisticsManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Paths; +import java.util.*; + +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opencb.opencga.storage.core.variant.adaptors.VariantMatchers.*; + +@Ignore +public abstract class VariantQueryExecutorTest extends VariantStorageBaseTest { + + private static Logger logger = LoggerFactory.getLogger(VariantQueryExecutorTest.class); + + protected boolean fileIndexed; + private StudyMetadata studyMetadata; + private VariantFileMetadata fileMetadata; + private int numVariants; + private DBAdaptorVariantQueryExecutor dbQueryExecutor; + private List variantQueryExecutors; + + public static VariantSolrExternalResource solr = null; + + + public void initSolr() throws Exception { + if (solr != null) { + solr = new VariantSolrExternalResource(); + solr.before(); + } + } + + @AfterClass + public static void afterClass() { + if (solr != null) { + solr.after(); + } + } + + @Before + public void setUp() throws Exception { + initSolr(); + VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); + VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager(); + if (solr != null) { + solr.configure(variantStorageEngine); + } + if (!fileIndexed) { + studyMetadata = newStudyMetadata(); +// variantSource = new VariantSource(smallInputUri.getPath(), "testAlias", "testStudy", "Study for testing purposes"); + clearDB(DB_NAME); + ObjectMap params = new ObjectMap() + .append(VariantStorageOptions.ASSEMBLY.key(), "GRCH38") + .append(VariantStorageOptions.ANNOTATE.key(), true) + .append(VariantStorageOptions.ANNOTATION_CHECKPOINT_SIZE.key(), 500) + .append(VariantStorageOptions.STATS_CALCULATE.key(), true); + + StoragePipelineResult etlResult = runDefaultETL(smallInputUri, getVariantStorageEngine(), studyMetadata, params); + fileMetadata = variantStorageEngine.getVariantReaderUtils().readVariantFileMetadata(Paths.get(etlResult.getTransformResult().getPath()).toUri()); + numVariants = getExpectedNumLoadedVariants(fileMetadata); + fileIndexed = true; + Integer indexedFileId = metadataManager.getIndexedFiles(studyMetadata.getId()).iterator().next(); + + //Calculate stats + QueryOptions options = new QueryOptions(VariantStorageOptions.STUDY.key(), STUDY_NAME) + .append(VariantStorageOptions.LOAD_BATCH_SIZE.key(), 100) + .append(DefaultVariantStatisticsManager.OUTPUT, outputUri) + .append(DefaultVariantStatisticsManager.OUTPUT_FILE_NAME, "cohort1.cohort2.stats"); + Iterator iterator = metadataManager.getFileMetadata(studyMetadata.getId(), indexedFileId).getSamples().iterator(); + + /** Create cohorts **/ + HashSet cohort1 = new HashSet<>(); + cohort1.add(metadataManager.getSampleName(studyMetadata.getId(), iterator.next())); + cohort1.add(metadataManager.getSampleName(studyMetadata.getId(), iterator.next())); + + HashSet cohort2 = new HashSet<>(); + cohort2.add(metadataManager.getSampleName(studyMetadata.getId(), iterator.next())); + cohort2.add(metadataManager.getSampleName(studyMetadata.getId(), iterator.next())); + + Map> cohorts = new HashMap<>(); + cohorts.put("cohort1", cohort1); + cohorts.put("cohort2", cohort2); + metadataManager.registerCohorts(studyMetadata.getName(), cohorts); + + variantStorageEngine.calculateStats(studyMetadata.getName(), + new ArrayList<>(cohorts.keySet()), options); + + if (solr != null) { + solr.configure(variantStorageEngine); + variantStorageEngine.secondaryIndex(); + Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive()); + } + + variantQueryExecutors = variantStorageEngine.getVariantQueryExecutors(); + dbQueryExecutor = null; + for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { + if (variantQueryExecutor instanceof DBAdaptorVariantQueryExecutor) { + dbQueryExecutor = (DBAdaptorVariantQueryExecutor) variantQueryExecutor; + break; + } + } + Assert.assertNotNull(dbQueryExecutor); + } + } + + @Test + public void testXRefRs() throws StorageEngineException { + Map> matchers = new LinkedHashMap<>(); + matchers.put("3:108634973:C:A", hasAnnotation(at("3:108634973:C:A"))); +// matchers.put("rs2032582", with("id", Variant::getId, is("rs2032582"))); + matchers.put("HP:0001250", hasAnnotation(with("GeneTraitAssociation", VariantAnnotation::getGeneTraitAssociation, + hasItem(with("HPO", GeneTraitAssociation::getHpo, is("HP:0001250")))))); + matchers.put("VAR_031174", hasAnnotation(with("ConsequenceType", VariantAnnotation::getConsequenceTypes, + hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, + with("UniprotVariantId", ProteinVariantAnnotation::getUniprotVariantId, is("VAR_031174"))))))); +// matchers.put("Q9BY64", hasAnnotation( +// with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( +// with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, +// with("UniprotAccession", ProteinVariantAnnotation::getUniprotAccession, +// is("Q9BY64"))))))); + matchers.put("ENSG00000170925", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( + with("GeneId", ConsequenceType::getGeneId, + is("ENSG00000170925")))))); + matchers.put("TEX13B", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem( + with("GeneName", ConsequenceType::getGeneName, + is("TEX13B")))))); + matchers.put("RCV000155534", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem( + with("Clinvar", EvidenceEntry::getId, is("RCV000155534")))))); + matchers.put("ENST00000341832.11(ENSG00000248333):c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem( + is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G"))))); + matchers.put("ENST00000341832.11:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem( + // The variant annotation may not have the "alternate" hgvs + is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G"))))); + matchers.put("ENSG00000248333:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem( + // The variant annotation may not have the "alternate" hgvs + is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G"))))); + matchers.put("VSP_039324", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, + hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, + with("Features", ProteinVariantAnnotation::getFeatures, + hasItem(with("id", ProteinFeature::getId, is("VSP_039324"))))))))); + matchers.put("VAR_081776", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, + hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, + with("Features", ProteinVariantAnnotation::getFeatures, + hasItem(with("id", ProteinFeature::getId, is("VAR_081776"))))))))); + matchers.put("PRO_0000211180", hasAnnotation( + with("ConsequenceType", VariantAnnotation::getConsequenceTypes, + hasItem(with("ProteinVariantAnnotation", ConsequenceType::getProteinVariantAnnotation, + with("Features", ProteinVariantAnnotation::getFeatures, + hasItem(with("id", ProteinFeature::getId, is("PRO_0000211180"))))))))); + + for (Map.Entry> entry : matchers.entrySet()) { + testQuery(new VariantQuery().xref(entry.getKey()), new QueryOptions(), entry.getValue()); + VariantQueryResult result = testQuery(new VariantQuery().xref(entry.getKey()) + .study(studyMetadata.getName()) + .includeSampleId(true) + .includeSampleAll(), new QueryOptions(), entry.getValue()); + if (result.getNumResults() == 1) { + for (SampleEntry sample : result.first().getStudies().get(0).getSamples()) { + if (GenotypeClass.MAIN_ALT.test(sample.getData().get(0))) { + testQuery(new VariantQuery().xref(entry.getKey()) + .study(studyMetadata.getName()) + .includeSampleId(true) + .sample(sample.getSampleId()), + new QueryOptions(), entry.getValue()); + } + } + } else { + List samples = result.first().getStudies().get(0).getOrderedSamplesName(); + testQuery(new VariantQuery().xref(entry.getKey()) + .study(studyMetadata.getName()) + .includeSampleId(true) + .sample(samples), + new QueryOptions(), entry.getValue()); + } + } + } + + public VariantQueryResult testQuery(Query query, QueryOptions options, Matcher matcher) throws StorageEngineException { + logger.info(""); + logger.info(""); + logger.info("####################################################"); + logger.info("########## TEST QUERY :" + query.toJson()); + logger.info("####################################################"); + logger.info("## Allowed VariantQueryExecutors:"); + for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { + if (variantQueryExecutor.canUseThisExecutor(query, options)) { + logger.info("## - " + variantQueryExecutor.getClass().getSimpleName()); + } + } + logger.info("## Using DBAdaptorVariantQueryExecutor for expected results"); + Assert.assertTrue(dbQueryExecutor.canUseThisExecutor(query, options)); + + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); + VariantQueryResult expected = dbQueryExecutor.get(variantQuery); + + VariantQueryResult unfilteredResult = null; + VariantQueryResult result = null; + if (matcher != null) { + logger.info("## Unfiltered query for comparison"); + Query emptyQuery = new Query(); + List fileNames = new LinkedList<>(); + List sampleNames = new LinkedList<>(); + List studyNames = new LinkedList<>(); + emptyQuery.put(VariantQueryParam.INCLUDE_FILE.key(), VariantQueryUtils.NONE); + emptyQuery.put(VariantQueryParam.INCLUDE_SAMPLE.key(), VariantQueryUtils.NONE); + emptyQuery.put(VariantQueryParam.INCLUDE_STUDY.key(), VariantQueryUtils.NONE); + emptyQuery.putIfNotNull(VariantQueryParam.INCLUDE_SAMPLE_ID.key(), query.get(VariantQueryParam.INCLUDE_SAMPLE_ID.key())); + emptyQuery.putIfNotNull(VariantQueryParam.INCLUDE_GENOTYPE.key(), query.get(VariantQueryParam.INCLUDE_GENOTYPE.key())); + emptyQuery.putIfNotNull(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), query.get(VariantQueryParam.INCLUDE_SAMPLE_DATA.key())); + for (VariantQueryProjection.StudyVariantQueryProjection study : variantQuery.getProjection().getStudies().values()) { + studyNames.add(study.getStudyMetadata().getName()); + for (Integer file : study.getFiles()) { + String fileName = metadataManager.getFileName(study.getStudyMetadata().getId(), file); + fileNames.add(fileName); + } + for (Integer sample : study.getSamples()) { + String sampleName = metadataManager.getSampleName(study.getStudyMetadata().getId(), sample); + sampleNames.add(sampleName); + } + } + if (!studyNames.isEmpty()) { + emptyQuery.put(VariantQueryParam.INCLUDE_STUDY.key(), studyNames); + } + if (!fileNames.isEmpty()) { + emptyQuery.put(VariantQueryParam.INCLUDE_FILE.key(), fileNames); + } + if (!sampleNames.isEmpty()) { + emptyQuery.put(VariantQueryParam.INCLUDE_SAMPLE.key(), sampleNames); + } + QueryOptions emptyOptions = new QueryOptions(); + emptyOptions.putIfNotEmpty(QueryOptions.INCLUDE, options.getString(QueryOptions.INCLUDE)); + emptyOptions.putIfNotEmpty(QueryOptions.EXCLUDE, options.getString(QueryOptions.EXCLUDE)); + unfilteredResult = dbQueryExecutor.get(variantStorageEngine.parseQuery(emptyQuery, emptyOptions)); + } + + for (VariantQueryExecutor variantQueryExecutor : variantQueryExecutors) { + if (variantQueryExecutor.canUseThisExecutor(query, options)) { + logger.info(""); + logger.info("###################"); + logger.info("### Testing " + variantQueryExecutor.getClass().getSimpleName()); + result = variantQueryExecutor.get(variantQuery); + logger.info("### Num results : " + result.getNumResults()); + logger.info("###################"); + expected.getResults().sort(Comparator.comparing(Variant::toString)); + result.getResults().sort(Comparator.comparing(Variant::toString)); + Assert.assertEquals(expected.getResults(), result.getResults()); + + assertThat(result, numResults(gt(0))); + + if (matcher != null) { + assertThat(result, everyResult(unfilteredResult, matcher)); + } + } + } + // Return any result. + return result; + } + +} diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/SearchIndexSamplesTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/SearchIndexSamplesTest.java index bd25023f6dc..12cb35f8714 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/SearchIndexSamplesTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/SearchIndexSamplesTest.java @@ -229,7 +229,8 @@ protected void checkLoadedData(String collection, List samples) Query query = new Query(SAMPLE.key(), samples); int expectedCount = variantStorageEngine.getDBAdaptor().count(query).first().intValue(); - assertEquals(expectedCount, variantSearchManager.query(collection, new Query(), new QueryOptions()).getNumTotalResults()); + assertEquals(expectedCount, variantSearchManager.query(collection, variantStorageEngine.parseQuery(new Query(), new QueryOptions())) + .getNumTotalResults()); SolrVariantDBIterator solrIterator = variantSearchManager.iterator(collection, new Query(), new QueryOptions(QueryOptions.SORT, true)); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchIndexTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchIndexTest.java index b83d1a9338e..4d5d8da6716 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchIndexTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchIndexTest.java @@ -215,7 +215,7 @@ public void checkVariantSearchIndex(VariantDBAdaptor dbAdaptor) throws IOExcepti TreeSet variantsFromSearch = new TreeSet<>(Comparator.comparing(Variant::toString)); TreeSet variantsFromDB = new TreeSet<>(Comparator.comparing(Variant::toString)); - variantsFromSearch.addAll(variantStorageEngine.getVariantSearchManager().query(DB_NAME, query, queryOptions).getResults()); + variantsFromSearch.addAll(variantStorageEngine.getVariantSearchManager().query(DB_NAME, variantStorageEngine.parseQuery(query, queryOptions)).getResults()); variantsFromDB.addAll(dbAdaptor.get(query, queryOptions).getResults()); assertEquals(variantsFromDB.size(), variantsFromSearch.size()); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java index a8fe6d9790a..17d7d073c51 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java @@ -21,7 +21,7 @@ import org.opencb.commons.datastore.solr.FacetQueryParser; import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.core.common.JacksonUtils; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -67,8 +67,8 @@ public void testTranscriptInfo() throws Exception { variantSearchManager.insert(collection, annotatedVariants); - VariantQueryResult results = variantSearchManager.query(collection, new Query(), - new QueryOptions(QueryOptions.LIMIT, limit)); + VariantQueryResult results = variantSearchManager.query(collection, variantStorageEngine.parseQuery(new Query(), + new QueryOptions(QueryOptions.LIMIT, limit))); for (int i = 0; i < limit; i++) { Variant expectedVariant = annotatedVariants.get(i); @@ -187,15 +187,16 @@ public void testSpecialCharacter() throws Exception { List variants = getVariants(limit); List annotatedVariants = annotatedVariants(variants); - String studyId = "abyu12"; - String fileId = "a.vcf"; + String study = "abyu12"; + String file = "a.vcf"; - variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(fileId); + variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(file); System.out.println(variants.get(0).getStudies().get(0).getFiles().get(0).getFileId()); //System.exit(-1); - scm.createStudy(studyId); - + int studyId = scm.createStudy(study).getId(); + int fileId = scm.registerFile(studyId, file, Arrays.asList("A-A", "B", "C", "D")); + scm.addIndexedFiles(studyId, Collections.singletonList(fileId)); String collection = solr.coreName; variantSearchManager.create(collection); @@ -204,17 +205,17 @@ public void testSpecialCharacter() throws Exception { samplePosition.put("B", 1); samplePosition.put("C", 2); samplePosition.put("D", 3); - annotatedVariants.get(0).getStudies().get(0).setStudyId(studyId).setSortedSamplesPosition(samplePosition); + annotatedVariants.get(0).getStudies().get(0).setStudyId(study).setSortedSamplesPosition(samplePosition); variantSearchManager.insert(collection, annotatedVariants); Query query = new Query(); - query.put(VariantQueryParam.STUDY.key(), studyId); + query.put(VariantQueryParam.STUDY.key(), study); // query.put(VariantQueryParam.SAMPLE.key(), samplePosition.keySet().toArray()[0]); - query.put(VariantQueryParam.FILE.key(), fileId); + query.put(VariantQueryParam.FILE.key(), file); query.put(VariantQueryParam.FILTER.key(), "PASS"); query.put(VariantQueryParam.ANNOT_CLINICAL_SIGNIFICANCE.key(), "benign"); - VariantQueryResult results = variantSearchManager.query(collection, query, - new QueryOptions(QueryOptions.LIMIT, limit)); + VariantQueryResult results = variantSearchManager.query(collection, variantStorageEngine.parseQuery(query, + new QueryOptions(QueryOptions.LIMIT, limit))); if (results.getResults().size() > 0) { System.out.println(results.getResults().get(0).toJson()); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java index 611265b0a0e..dc5a019baa1 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java @@ -118,6 +118,7 @@ public VariantSearchManager configure(VariantStorageEngine variantStorageEngine) VariantSearchManager variantSearchManager = variantStorageEngine.getVariantSearchManager(); variantSearchManager.setSolrManager(new SolrManager(solrClient, "localhost", "core", variantStorageEngine.getConfiguration().getSearch().getTimeout())); + variantSearchManager.setSolrClient(solrClient); return variantSearchManager; } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java index 5fb147d6d9e..e90b49cff4d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java @@ -18,6 +18,7 @@ import org.junit.*; import org.junit.rules.ExpectedException; +import org.mockito.Mockito; import org.opencb.biodata.models.variant.Genotype; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; @@ -27,18 +28,21 @@ import org.opencb.biodata.tools.variant.VariantNormalizer; import org.opencb.biodata.tools.variant.stats.VariantStatsCalculator; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageEngineTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import java.io.IOException; import java.net.URI; @@ -48,8 +52,7 @@ import java.util.*; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Created by hpccoll1 on 01/06/15. @@ -112,6 +115,88 @@ public void calculateStatsMultiCohortsTest() throws Exception { checkCohorts(dbAdaptor, studyMetadata); } + @Test + public void queryInvalidStats() throws Exception { + //Calculate stats for 2 cohorts at one time + checkCohorts(dbAdaptor, studyMetadata); + + QueryOptions options = new QueryOptions(); + options.put(VariantStorageOptions.LOAD_BATCH_SIZE.key(), 100); + Iterator iterator = metadataManager.sampleMetadataIterator(studyMetadata.getId()); + + /** Create cohorts **/ + HashSet cohort1Samples = new HashSet<>(); + cohort1Samples.add(iterator.next().getName()); + cohort1Samples.add(iterator.next().getName()); + + HashSet cohort2Samples = new HashSet<>(); + cohort2Samples.add(iterator.next().getName()); + cohort2Samples.add(iterator.next().getName()); + + Map> cohorts = new HashMap<>(); + cohorts.put("cohort1", cohort1Samples); + cohorts.put("cohort2", cohort2Samples); + + // Just cohort ALL is expected + VariantQueryResult result = variantStorageEngine.get(new Query(), new QueryOptions(QueryOptions.LIMIT, 1)); + assertEquals(1, result.first().getStudies().get(0).getStats().size()); + assertEquals(0, result.getEvents().size()); + + metadataManager.registerCohort(studyMetadata.getName(), "cohort1", cohort1Samples); + + // Still just cohort ALL is expected, as cohort1 is not ready nor partial + result = variantStorageEngine.get(new Query(), new QueryOptions(QueryOptions.LIMIT, 1)); + assertEquals(1, result.first().getStudies().get(0).getStats().size()); + assertEquals(0, result.getEvents().size()); + + //Calculate stats + stats(options, studyMetadata.getName(), cohorts, outputUri.resolve("cohort1.cohort2.stats")); + + checkCohorts(dbAdaptor, studyMetadata); + + // All 3 cohorts are ready and expected + result = variantStorageEngine.get(new Query(), new QueryOptions(QueryOptions.LIMIT, 1)); + assertEquals(3, result.first().getStudies().get(0).getStats().size()); + assertEquals(0, result.getEvents().size()); + + List cohort1SampleIds = metadataManager.getCohortMetadata(studyMetadata.getId(), "cohort1").getSamples(); + CohortMetadata cohort2 = metadataManager.addSamplesToCohort(studyMetadata.getId(), "cohort2", cohort1SampleIds); + assertTrue(cohort2.isInvalid()); + + // Cohort2 is invalid, but still all cohorts are expected, but with a warning event + result = variantStorageEngine.get(new Query(), new QueryOptions(QueryOptions.LIMIT, 1)); + assertEquals(3, result.first().getStudies().get(0).getStats().size()); + assertEquals(1, result.getEvents().size()); + assertEquals("Please note that the Cohort Stats for '1000g:cohort2' are currently outdated." + + " The statistics have been calculated with 2 samples, while the total number of samples in the cohort is 4." + + " To display updated statistics, please execute variant-stats-index.", result.getEvents().get(0).getMessage()); + + VariantStorageEngine engineMock = Mockito.spy(variantStorageEngine); + VariantStatisticsManager statsManagerMock = Mockito.spy(variantStorageEngine.newVariantStatisticsManager()); + Mockito.doReturn(statsManagerMock).when(engineMock).newVariantStatisticsManager(); + Mockito.doAnswer(invocation -> { + invocation.callRealMethod(); + throw new StorageEngineException("Mock error calculating stats"); + }).when(statsManagerMock).preCalculateStats(Mockito.any(), Mockito.any(), Mockito.anyList(), Mockito.anyBoolean(), Mockito.any()); + + options.put(DefaultVariantStatisticsManager.OUTPUT, outputUri.resolve("stats_mock_fail").toString()); + try { + engineMock.calculateStats(studyMetadata.getName(), Collections.singletonList(cohort2.getName()), options); + fail("Expected to fail mock"); + } catch (Exception e) { + assertEquals("Mock error calculating stats", e.getMessage()); + } + + cohort2 = metadataManager.getCohortMetadata(studyMetadata.getId(), cohort2.getName()); + assertEquals(TaskMetadata.Status.RUNNING, cohort2.getStatsStatus()); + + result = variantStorageEngine.get(new Query(), new QueryOptions(QueryOptions.LIMIT, 1)); + assertEquals(3, result.first().getStudies().get(0).getStats().size()); + assertEquals(1, result.getEvents().size()); + assertEquals("Please note that the Cohort Stats for '1000g:cohort2' are currently being calculated.", + result.getEvents().get(0).getMessage()); + } + @Test public void calculateStatsSeparatedCohortsTest() throws Exception { //Calculate stats for 2 cohorts separately diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv-large.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv-large.vcf new file mode 100644 index 00000000000..5e1de5845e2 --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv-large.vcf @@ -0,0 +1,52 @@ +##fileformat=VCFv4.2 +##FILTER= +##FORMAT= +##FORMAT= +##FORMAT= +##FORMAT= +##FORMAT= +##FORMAT= +##FORMAT= +##FORMAT= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +##ALT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT SAMPLE_3 SAMPLE_4 +20 500000 . C CCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG . PASS . GT 0/1 0/1 +20 600000 . CCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG G . PASS SVTYPE=DEL GT 0/1 0/1 +20 700010 . T C . PASS . GT 0/1 0/1 +20 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;RIGHT_SVINSSEQ=CCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG;LEFT_SVINSSEQ=ACCACACCCACACAACACACA GT 0/1 0/1 +20 850000 . A . PASS SVTYPE=INS;END=850000;SVINSSEQ=CCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG GT 0/1 0/1 +20 860000 . A . PASS SVTYPE=INS;END=860000;SVLEN=1000 GT 0/1 0/1 +20 900000 . C CACCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG . PASS SVTYPE=INS GT 0/1 0/1 +20 900000 . C CCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG . PASS SVTYPE=INS GT 0/1 0/1 +20 900000 . C CGCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG . PASS SVTYPE=INS GT 0/1 0/1 +20 900000 . C CTCCCGTTTATGAACTCAGCGCCGAACAGAAAAAATAGCCCCACTTTAAGTCCGCTTTAGCGACTCTAGGGTCCGAACGCGCTGTTTCGTACATGGCACGTCGGTAGGCAAGAACTCCCGTCCTCAGATGAAGATGCGTAATATCCTTACGTATTTTGGAACTCAGGCTGCCGAGCATCTTATTGGGAGACTCTTACCACTTTGCCCGTAAGCAGGGAAACGACATTTGATAAAGGATGGCAGGGAAGCTTTTATGCCCCTTTTCCTACTACAACGTCGAATGTTGACTTCCTGGTTAGCGTTGTGGCCTTGTACTGACCCTACTAGGGTTTCAGCTGCCTAGAGGACATTCGACCGACCCACGACCGCAGCTCGCGGTTCATACACGAGAAGCTACAATCCGCTTAAGTATTTGTTCTTTTCGTTCTAGGGCCCGCGGCACCAAGGAGGCTTCAGAAAGAGAATAATTACTCGATCGTCCCGTAAGTAGTGTATCGCTAACAGGGCCTGCGTCGTCCATACTAAGGCAAGGTGCTCCAGCAGGGCATAGGAATTGACCGGCGGGTAACATTGGAGACAACATGTGATTTGTTGCTTAATCTCGGTTAACCCGCCCCGCTGTAAAGGCGAGACGGCAATCATGAATTCTAGCACGCACCCCGCCCGTCTCTGTCTTCAAAATGTATTTTTGGCAACGAAGGATCAGCCTGTCCCGAATCGACACTGTGTTCCTATGGCGTAAAGAATTCTGTTCCTGAATCGCGGGCGCACCGTAATTCTTACTTTCACATCGAATTGTTAGATGCTGACGAGCAGAGCGCAGTGCGGCCGGGCGGAGATTGCAGGGCTGGCAGAGCCATTGGCCGTTGATGGTGTTTAGACGCTAAGCTAACCCTATGCCTCATGGATACTTGCTACACAACAGTGTCTGCGTAGGCTGAAATGGGGACGGCATGAGACCTCAGTGTCAACACAATTTGATGCACTGGGTTTTCTGCAACTACTTTACTACGGCTTTGTCCTTAACGCGTTAAGGGACTCCATCTCATCCGTATCATAAACTCCGTAGTGTATTGGGGCCAAATCTAGAATACGTACCGTCGACAATGCTCACGAGGGTTACTCAAAACTCGGCGGGGTGCCTCAATCCGCGTGCCGTGAAATGCCCGTATTCACGACGACTAAGCACTTAAGTCTCGGGAGCTCTGTTGCGTCTCGTCTAAGGGCAGTCTTGCTTCCTGTGTCTGCAAGTTCCTCTCTAGTGTTTAGGGTGCCTAATATACTCCACGTGTGTCTATGGACTCCATATGGTAAGTAATGGTGTTTGATAAACCCCTGGCCCTTTAGGTTCTACCTAGCCTAACTTCTTCCTTTGACTTTTACCCCCATTGATCCATGTTCGTCGAAGTGGCCGATTGAGGCTGCCGCATGGCCAAACCGCTAACCCGATCGAGAAGGTTGCAAGGGCGCATCCGCAAATAAACCATGGTTGCTAATTGGGGTGCAGGCGTAGAATTTGTCGGTTCAAAAGGCCCTCGACCTGCACAATGACTTGCGCTCGTAACTTATGATAGGGCCGGCATGGGATTATTCGAGGGACTCCTCACACTCAGAAGTTTACTCCCGGAGCACGACTTTACGAGGTGTGTTGCTTTATGATGGCATATAAAAAAGATGCACCAATCACCAAAACCCAACTATCCTTGACACGATCAGTGCGCGACCTACGACTACTCTAGTGGTACGACTATCGCGATGGAGGGGATATTCGTATATTTGAAGTTCATGTATTGATTCGGTTATGGCGTCTCCCTTAGTGTTTATGGACTTCATGCGTGCCCCTTATCCGCCCGCAGGCACCAATCACATTGTGGAGTTAGTTGGAGCGAAATTGGGTGGCTGTACGGTCAAACTAGAAGCATACCTTCACAAGGGCGTGGCGCTATGGGAAGTGGACCTGAGGGACACGATGGGTGAGATTCGGACACTCTGTCTACAAAATTAGGAGGTACCTACTGCTAGCGAGCTTCAAAATTCGGGAAGAGATCCTCGTACCACGTATTAATGAAGAGCCAGTGGAGCGAAGAAATTATCGCAAGACCGACCCTGTGCCGGCAGATGGACGTGTTAAACATGGTACATTGAGCAGTCCGACTCGTTGTATAGTTACCACGTGTGAGAAGACTTAGCTCGCAGCTAGTGGACAAGAACTCGGCCGAAGTTTCGTTCCTGTATTGCGCGTGTCCAGGATGTGAACAGTCCAACTTCATTATTTTATTCTGACTTGACACTGGACGTAAGTCAGCTGCTCAGAAACCGGAGTTCCCTGCCTTGGCCTGGGGGCCCCATTTGCGGATGAGCCCACAGGTCTCGTAGACTTAGGTGACGCCGGGATCTGCTGGCCTTATGTCCCTAGAAGCTTGGATGACCGGCCCACGGCAATGCGTAGTCAGGGAGGTATCCATGACGTGCGTGAGTCGGGCGAAGAGGGCCCCACATGCACCTAGAATAGCTATGTGTGTTTCTCGGGCGGAGCCCGACGTCGTCCCGCTCATATCCAAGTGCTTTTTCCGGGGGTTAAGCGGTGGCTTGGGCACAAACGGTCGCCTCCGCCCGCTTGGTCTCTCAGGTATTTCCGGGGGCACTCATAAACTACGAACGCGATCTCCGATAGCCTTCAGTTGTGGGGATGCGGCTAAATCCAAGCAATTGCGACGCTTTGGGTGTGGTGGGGGTTACTGATCCGGACTGCAAATTACTAACTCCGTCACCTATCAACACGTCTCCCTAATCTGTTGGAACCTCGCTCTATGACATCAGAATGCGAGCGCGGACATGGGACATGTCGGGACAATCTATTCTGCCTCTCCGGTACTGGGTACGATAAGCGCGACACAAGATTGTGCTCTTACTTCGCCGTGTTACCGGGTAAATCGCCGGGATGCTACTCTTATATGTGCCTTCTCACCTGAAGCCTTGCGACCTCGAATGATTTGGGTGTGAGTAAGCACCCGGGATCGAAGCTGGTAAGCACGTCACCATGTTAGAGATACTTGACCGACGATGCCCTTCTACGGCTCGGTTCACGGTGCCCAGCGGGATCCTCTTATCCCAGCCGGTGCCTAGCCGATAAAAGCTCACAAGCTTAAGGCTCTGTCAGGGGAGATGTGGGAGAGGCTCGATGAGAAGTCCTAACGACACCAGAACCCCAATTAGCACACATGGCTCTTCCGGGGACTTGGCTCTCGTAACTGCCTGCAATGTACTACGTCGCATGCTCATGTACCAGACATTATGTCCGTCCTTAACAGTCGTATTGGAGCAAGTTGTGAGTACAAACAGTACACAAGCTGTAGGTCACTCCCCCCACGTCTACCCTATATACTATAAGTCCTTGTAGCCAGGATGAGTAGGCCCCGTGTACCTTATTTGAATGCGCACCCGTGACTTCTGGTGGAGAACTTGGTCACGTCGGGTGGACCGTGTACGATACCCCACAGTGCCCCAGGAGGTATGGAACGATGCTGCCCTGAAAGGTGGAAGGCCGGAGTTTTTGTAGTTACAAACAGTACTCAGCGCTATCAGCAAGTGGACACGGAGAAGTAATTGAATGACCGCCTAGCAGTGCTCCGCCTCCGGGCGGCAGCCCGCGTGCCTCAGTCAGAGATCTCGAGCACGCACCCTTGCTGCTTCCGACTAATGGGAAGAAGTGGAGATGGCGACCCCCACGCGTTATAGTCCCAAGGTTGCTTCCAGGAGCTATACCCGAACCTTTAGGTCTAGACTTGTCGAGTTTGCGACCGTACTCGCTTTGTATGTCGACTTTTTTGAAAAGACAACCGTGGCCCGGTCACTGTGTCTCCAGTTTGGCGAGCCGTAGTCAGTGATCGCGCTGCTCGGGATCAATTATCTCCAACGGTCAGCAGATCTACGGAGTCGGTGGCCGGTATAATGCTGGAGGCCGGTGTATGCAGGAACCTAATAGTCCCCGAGTGTCGATCTGTACGGGAGTCACTAAGATAAGGCAGAACTGACTATCTCTCCACCGCTACGTAGTCCAGTGCCAAGGTGTGGTCATGGCCCACATAAATCAAGGATTTGTATGCTAGAATTACGCGATCGAGTGACACTCAGAGTGCACTGGCTAAGGACTCTAGCTTTGTAAAGACAGCATTACATCAACACTCGTTGCATTCCTAGACTGCGGTTCCTTCGGGATCTCCCCTTGGTACTTCCTATATTTCCTGAGATGTGGGCGCAAACAAATGCCAGATCCCGCCTGTAGGCATTCGGGACGGGTACTACGCGATGACCATATCGAGATTGAATCTTACGTATCTCCCTACAAGTGTGGCTTAAGGGGTTAGCACCGACAACTCATATAAAAACCAAGTAAAGAGAGTAAACACAGTGCGGAGTAGCCGTACTTGCGAAGCGCCATTTGGGCGCGACCTTCCGAATTTGCATCCGACGTCGGTCGTATAGCAAAGAGCGATTCTGAGCCTCCAGTCGCGGCCACGTCGCCGCTTGGCCAATGTATACCAGTTGTGAATAGGTCGCGGAACGCTAATGGAAGTAACCCCTCCTATTACCAACACGCCTCAATCGTACCACCGGTACCCACGGCATATAGAGACTATACGGCGTCGCGGCGGGCCTCCTCATGCTCGTATGGACTACACACCATGTCCTAATAACCTTGGTAACATTACGCCTTTGAGGGGTTCTTTATCTCCGATTTCGCGGTAGATAAGGCCACATGACATTTTCCGTTCGAGCGGACACGTGTCGCCTCAGTAGCCTATTGCCGTCGCCCAACCTACGCCACTGGCTCATGCCATGCCATCTCATACAATTAACTCCCGACGTTCGTACCAGGTGTACCAGGGGGTCCATACTAAGAGAGTGTGACCTCCAGATTCTAAAATACTATGTGAGGAAGTCATGATTTCCTCTAGTCATTGTGGCACCGCTCGTACCGTGAATCATATCAGATAAGAACGATCAGTTTGCGACTGTTATTCCGAGGGTACAGTCTTTAAGACACCATTCGGAAACGAAAAGGGACTAATGGATGCCGCGAAAAACCTAAAAAGGAAACAAGCTATCGATCTGTGGCGCCAAGGAGGGCGGGGCCCCGCATGACTCGTCAACTCCCATAGGCACCGCTCCGGGCCGGTAGCGACCGGAAGGCAATTGCTACCCTATCATATAAATGATCCTGGTCATTAGTTGTGCCCCGTATACTTGAGTCAAGTATTCCGTCGCAGAGACTACTGCTCATCCGAGACAACTTCTTCTCTAAGCGACCCGGGCGGACAAATTCACACGCAGATGGGAATTACACTAGGGGTATACTCACCCGGGCCTAAGGAAGGCACATGAGTGGGGCGGGCTGATTAGCAATTGTATCAAACTAGGGTCAGGCACACATAATGAAGGAGCAAATACATTGCACGGATATAGCCTAGTCGAGTTCGACGCGGAAGAGGTAGGCGAAATAAACTGTGAAAATGGGCAACACCTCACGAATGGTCATGAGTCAATGTAAAATCTTCGCGGCTTACTCGGCCTATTGCTATCGTGACTCCCAGGCCCTCAGTCCACCTCTCGTCAGATACTAGTTCGGGAGGCTCGAGATAATTGACCCTCCCTAATTCAAACTCAAACGAACTCACGTACGGGTAGGGCACCTAATTCGAGATGAAACATGCCTAAACGCCTCGAGAATATGTCTGTTTTGTACATAGTCTGTCGCTTATTCCACCACTTCAATCTGAAAATAGCGGACTAGTCCCGAGGGCCCCGATTTGACAGGTGGTTGAGAAGATCAAGTTGCACGCTAGCACCAGACCCTAACTCGCAACTGCTCTGCCTGATAAAAAACACATGAATCTGCGAACCATAACCTTCTTTGCCATTCTTTGTCTGCGGTTAGCGATGGGGGGTGTAAAACCACCTGCAGCGCGTCCCGATGATAAGCTATATGGGTTCACGTGCAATCCCAGAATCCTTTGGGGGGTGGATCAAATTGAAGCGAACGCAGCGTTGTATCGACGGTGAGCCCGCACATTTGAACTCGTTCCTCGGGTTCTTCGATGAATGTGGACTACGCTAGTTGACAATGCCTGCAAGTCTCCTCTCTCCCCCACGTGTAATATTGTACATAGTCAAGAAAGCTACATGCACGCCGACGTATCTTCTGTGCTTATGGATCGTTATAAAATTTTGTACTTACAGTGAAAATATGAAGGATGAAGCCTTAGAACTCGCTGCGTGAGAGAGAAACCCACCTATAACAAGACGCACGTGGCGCACATCCGTCCCCAGTGTTCAGCGACAGTTTGTCTAGCCAGTGAATCGGATCAGGCGGTCCTAGAGGTCTAGGCGACTTATGATCTTAGTCAGTAACCCCAGTAATTACCGTGATTACCATTTACAACGGTTAATTGGGTTGGTCTATGGTAACGCACAAATTACTCGTCATACCTGCTTAACATGCCTTCGGGAGTTAACGCTGTCGGGCGGGGTCAGCTTACATTACAGTAATCCGGCAAAAGACAAACCAATGCATCTCAAGAACCAACATAATAAAAAGCCTTATCAAATCTGGATCACAACTAAACGTATCACAGTAATATGTATCGCGAGTCTAGGAACTTGAGCACCCAATCATCATCCGTGGCGTCTCCGCGGGCTCTGAACGAGAAAACTGGGCACTCCCGGCTACTTTGAAATCAGGAAAAGGTCATCTGTTAAATGTCTGTTCACGATAATAGGCTATTCCACTTCAATACATGAATGACTTAGGATAGATATTCCCCTGGGCACGATGTTCTTATGGAGTGCGGGATTTAGTGGATATTCACGCTTGTATTCAGTCCCCTGTCCCATCACCTTTTTATTTGTCCTCCGCCCGGATTCGTTTCGTGCACGGCCAACGAGGAGCTGTTCATCAAATCTACCGACATTAATAAGTCTGGCCTATATGTCGTCCAACTAGGGACCCCTTTCAGAGTAGGTGGAGCTATCGTAGTCATCAAGTCACAAAATGAGTTCCTTTAAGGTGGCGATGTGTTGGGTGCAGACAGTATAGATAACACTCGGCATGGCTTTAGTCCCTCTATCTCGGATGCGCGCATCACAGATACCGCACAGGGTCCCACCTTACATGTTCGGTTGGGACTGAGTGGAAAAGTTGGACCCTAATGGTTGCTTCCGCGAGAACAGAGACTTGAAGCAGTCAGTCAGGGATTCAGTCCGTCAGATAGGGGTGGAAGTATGGCCGGCGGCCATTTGAAGTCGGCTAGCAGCAAATAAATTGCGTAAATATGTTGGTGTTTGGGACAAGGAGGCACGGATCCAGACGTTACTGAGTCCCCGAAAGCGTTACTCACGGATACACGCCATACGCTCTACTAAGAAGAACCATGCACGGACCATTCGATACAGTCTTTCAAACGCCAATGAACCAGGAGCCGGGGGGATGAAGGTGGGTTGGAGGCATCGGAGTGGTTGGGAACTGGTAAACGAGAAGGCCTAAGAGTACTCCCAACGCGGTGGGTAGTATGTAGGTTGGTTTAAGTCGCGTTATCTGGAGCCCGATACAGAAGACCATCAGCTAAGAGTGGTCGTAAGACAACAGATAATCCAGGCATACTAACACCGTTTTATTAACACCTAACCCGTGCGCGGGCAATGATCTGTGATTTCGGCGGAACCCTTTAGCAATCACTCGAATGCTCCCCTCAAATTTGAAAGGCATGACCTCCGTTGTACGCCCTGGATTTCTTAGGGATCGATTAAATTGGAGGCGAATCGTGTGTCTGATCTATTGGCCAATTTGATAAATCGGGACAGAAAACCCTGGGCAGCTGTATCCCGGCCAACTTGACTTTCACCTTTATGCTATCAGAAAAATCGGTGAAACTGAAAGTCTCTGAAGTTCTCTGCAGTCATGCGGGTACACGAATGTGAATATGAGCTGTCGCGAGTTAATAGGACAGGCAGAAGCGCAGGTTGCGTCCTAACAGGAATACTCTTAAAGTAGGTGCGAATCGAGCAACGTCATCGCGTCAAGCGCGACCGCTTCAGTTGGTTGGAGGCTGTCGGCAGGTTCATCTTGCCAAATCGAACTAGCAGTGGCTTCGTTACGGGCCAATACCACCTCGAGGAACCCGGGTAATGAGGTACAGCTTACACAGGCAACCTGACAGAATGGATGTTTGACCTTCTATGCTGTACGTGTTAGCCCAACACAATTAGCAAGATTGCACCGGCGCTAACAATGTTAATGAGGTGTACTAAAGAGGTCGTACAGACAGGTCATGGGGGTAGGCGTATTTTCGCGCAGAACACGGGTCTATCCCATGAACGTACAGGCAAAATATTCGTTCTAGGAGCTGTGGGTCGGTTCCGTGGAGAAAAGTCGCAAGGCTATGTAAGCATTCATGCCTAATCCTCGCAAAAAGTGTGTGGGATTAGAAGCGTGTCTGAGCAGATGTGGTGCGGGATCGAAATAAGTAGGTAACTACCACAGCACACTATTCCAAGCAATAGTTCGAAAAAGGAAGACAAACCGATTCGGTTAAACTTGTGGCGGTCCTTTATTGCGCTAGCGTGCACATACTTATAACCGTTCCTTGCTAGATGCCAATACACGATATTGGCACATCAGGTGCTTTCACCCGAGACAAACGGCCTCTGCTAGCAATCCCGGGTCGGTAAAAATAGAAGACGAGAGGACGGCGATCATATCGTGCGAATAGTATCAGGCTTCATATGGGCAGTTCGCCTCGAGGACTACCCTCGTCGAAAAGCTTTACCGGAAGATGCCAAACCCGGGCAGCGGCACCAGGCTGTTACCCGTACTCCTAATAGCTCGGCTAAAACACGGAGAATGAAATACCTGTACCAAACGCACGTTGAGCTTCATCCACCTCTGGCGTGTGTACTATCCTATAGCGGATTTCGCCGGAGTTGCCTTTTCTCCTTGAGTGTTGATGCTGCCTTCTAACAAGCGCCTTTGCACCCAACATCGACATTTAAACTTTATTATGGGGTTGGTCCTGCTCCGTGCAACTGTCCTCTCTGTCACCACTATATTAAACATAGTCTGACCTAATGCAAGTTAGTAAGGAACAAGAGCAGCCTCATAAAGACTTGGCGACGCGTTGAGACACATATTGGGTACTTTAGGGATAATAAGTGTGAGGTACGAGGTGCGCGGACATTATCTTGATGCCCTTCTTGGAAATGTCGTATTTCAGGCCCCTGGTGCGGAAGAACTTTGGCCGGGTTTACATAGTTTAGATAGTCCTAGGTACATCAAGACGCCAGTATGAGTCCGCGAGCCGGTATGAGTCATATACAAGCCAAAGGGCCGCCTCCAACTTCTGTATCCGTAATTTTCTCCATAGGGCTCTGACCAGTATTGACCCCAAAATGTGGGCTGCAGATCCAAAGAGTTAACGGTACTACCCGAGGGATCAGTCATTCACTACGGTGCGCCTTGGATGGGATCCTACCGTTGAATTCGTAGCCTCCTGCGCTCCGCTAACCCTGACAATATGTGCACTTCGCCTGGACGTATTGACACCTGAAACGAGCACTGAGCGTAAAGCTAGTTGTTATACGTCACTGCACCATGTGGCGGGTTATAGCATCCTCCCATTTAGGTAATGCGAATCTTCACAGACATGACGGGGCATGTGGACTTACCGGTTACTTCGGGAAAACTGGCAGACAAAGGTTGGTTATAACCTACCAGCAAGTCATAATCTGTTTCAGGAATTACGCAGTTCTGCCGCTCATCCTAAAAATCCACCGACCCTCGCACGTGCCGAGTTAATCTACGGAACGCATAAGAGAACGGAAGCCCAACTGGAAGTTGGACAATGCCATGAGCGCCCTCGGACCACCCTCAGAGAGGTAATTTAGGGGAGCTAGGTAATGATCACGCTTTTCAGTGGTCCTGTATCGTGCATAGGGAAACCACTCTGGATAGTGCTTCTAGCGAATCGTCTAAGTACCCCATCCTTATAGCCCTGCCAAAGTGCTGGTAGTAGCTATCCGGATCACCACGTTCGGGGGACGGGGCGTGGCTCGTCCGTATGCTGGCTTCGCCGTAGCGCAGTCCGAAAACTGACAGACATAGTGCATTAGCATTTAAACCACCTCGAATAAGATTTGGGCAGATAATAGCGCCATCGTGTGCACTTGTTGCTCGGTGCCACCTAACCCCGAAGCTCCTAAGGCGCGGGCCTATAATCGTTATCCCCTACGAAATGCAAGTGCGTCCTTGCTAGAATCATTTGACTCTCCAGGGCCGGGAGTGATAGACCTTTTCAGATGTGTACCCAACGATGGGTTCCTACTCACGTAGATACCAGTTCAACCGCATAACCGTGAATTTTGGTCCGTGTCGTGGACGCCCGGAATGATAAGCCTCAACAGATTGCCTGTTTAGCTGGTCGATTGTGCCGATCAAGTTGACGTGCTAGGCCGGCGCTAAGCGTAGAGGAGAAATCTCGCAGCTTCGATACACGCGAGAACTTGGGCGTCACTTACATAGTCGGAGGTGTCATGGTTGAGCCGGTATCTCCGATTGATCACTCGGGACCAAACTCTATCTTCCTTCTAGAATCATCACGCCGACTTGAACACTGTCGTCGTTTATCACTGATAACGCTTATCAGGTTACGTGGAATCGAAGGTTCAAGACGGTCATGGAGCTGTTTTCTATTATTTCACTGTTAGTTCTTGGATACTGGCGAAGGACGCGCCACTTAGACTGTGCAGTCCCCAATTGGCTCGAGATTCCATCGGCGCGTCTACCTGACTCAATGCGTTACAGTCTTGGCCTGGCGCTCCTACCCCTGCGGCCTATAACGAGGTGGCTGTCCCGCGTGCAACACTCGCGAGCAACATGCCAAGACTTAGGTATAATAGCTTAAAGGTGTTCGCTTCTTGGTAGATAATTCTGGATCCGCGTGCCCACCTCCCTGTGCTACGAAGTTCCGTGAACAATGTCTAGATGTATAGCAGTGTGCAAGCCACCAGGTTCTATGGTGAGCAGCAATACAAGCCGTAGGTGACGTTCGGAATGACCCGCAATTTGTCGCGCCCCAGAGTAAAGCGAAGGCTTGACACTCCCGACTAGTAATCCCATGTGTAGGCCCCAGCACTAAAACATCTCGTTCCTCCTCACTTCCTGTGCTCAAGCCGCCTGGGGTCGAGGCGTAAGTACGGTCTTGATTGGTTGATTTTTGGACAATATGATGTCTCTACCAGCAACGCCGCCACCGGTTATGTCAGTCTCTGAGTTATAGCTACAATCTCAAGAATGTCGTGCGGCTCCCGTGGATGGCAGACGAGTCAAGACCAATAACAGACGTGTGGCACCTCTGGTCGTATCGTCACCCCCAATTGATGAAAATGTGGCTAGAGTTCCGAGGAGGACTTGCAAAACGAGTATCATTGATCCAGCCCATTACTCCAGTGTCCAAGCCCGAAACTGTAAGAGGGCCATTGAAATAATACCGGCTAAGGAATGTCTCGGGCGTGCGGGAATCATAAGGATCCCTCAGTTGACGATTGGGCGCCTCTTTTGCGAGCGGCATCCGCTGCATCGAGGTAAATATGTGTTGGGGACACAAGCCGGATTGGCATTGCCGGAACTTATGTGCGGCAATTGCAAAAGTCGAGTACTAGAAGGATGATCAATTGCCGATGCGCCAATTGAAGATGATCGCCTAAATTGGCTTGTTCATCCTTCGCTCCTGTACGAAGAACTCTATCCAACAGTTGATGGTTGGCAGTAGTTCCGCACGAACAACAACGGGATGGAATTCACGGACATGTTGGTGCCTAAACTAGCGCCGGTTTACCGGACGTGTCCGTACTCATTCCCGCCGCGCCAGCCAGAAAGGGTCGATAATTGCAACCTCTCTCGCAGTATCCTTTGGTCGGCAGAACGCCATGGAGCGATCTCCTATCTCTCAAGGTCAGTTTAAGAATTATGAAGTACTTCTCGACGTGTTGAACTTCCGGTCTCGGCCGTTAGGCTCGCTCGATTGAAGTACAACAGAATTCAGTTTAACCTACTACTCGCCCCAGAGTACTTCCTATGAATGGCTGAACTCTGACACCGAGTAGACAAGTGGTGGATATACTCTTTACGGGAGTCACGTATATCGTACTATTATGTTTCAACGGTGCTTGCCATCCACCACAACACCGGTAAGATTATCAACGCTTACTCTCAATATTGCCCGCGAGGAGGTCTTATCCAAGGTGTAACAATAGTAGTCTGTAGCGTTGAGATTAGTCTTAGAAACGGTGAAGGGCGTTCTCGTGCGCTCGATACCGCTCCCCCCCAATCGTGTTCGGCATCACTAGGACATACAGCGGTCGCCAAAGCCTTTCACCGGTATCGCTTCTCGGGACTGATGCTGCAATACACGCGTAGTCGAAGCAAATTTATGGTGCCCAGCATCTGAAGCAACTCCATAACTTTTACCGTCGATAACAGGATTTGGTAGTCCGCTCGGTACTAATGCCGCAAAATGAAGCCCTATAATGAGATATCCTAAGGACGTAGGGCGCCATTGTTGGTTAATTCCCCCTACGGATTGACGGGCCGCGCCGTCGGGCTGGGGTTCGTATGAACCTAGCTCCACGCCAAGACCTGTCCCTTGTCATAGCCAACAGGACACCTTTCATTATGGCGCTACGCATATGCGACTGTTCCCACAGTACAATGATGTGCTTAACCATCGTCGCTCGGCGATAGGGTATCTTTTAACTATCTTAGACTGATAACGTGTGGGCCGATAGTATCTAAGAGCTGTGAGGCTTGTGGACCACTTAAGCTATCGACTCCCATTGGGGCAAGTGATTACCCGGGTGCTATCCACAAGACCGCACTTGTGTCCTTGCGTCGAATTAAGAAAGCTATAGGGACCAGCTGATATTAGGTGCTGAGGAGTGCTACCGGCCTGTTTCACATATATCAGAACGCGTGCCCTGAACTAAGACTCAGCCAATCAATACTGCGTTGTGGGTGAGGCAAGTATTCTGTCCGGGGATGCTGTGGGGGACAGTATTGATCTATCATACTAACCATAGTCGACATCTCAGGTTTTCGAGCGCTGACGGTACAGACAAATTTAGGGCGCTACGAGAACAGCATGTCGCTTGAACCGACTCAGGCGCCGTAATTAAAGGGAGTGTGCCATGAACTCGAACGCACTCCGTATCAACGCAGAAACCGCCTTACAACACTGAGCGGCGTAAGAATACTCACTCGATATTGATTACCGTAAGGCTATATTTGTATGGGAAGCAAACGAATGACATTTCCTGGATATACAACAGTTAGCTGAAGGCATATGTGATCTGCCAGATTCCAGAAGCTGATGGGGGATCAGGATGAACCTTGTCAAGGATTCGTATCTGCGCTATACAGGGAGAACTGTGCGATTGTTGAGGCTACGCCGCGTAGCCGAGCCGTTGGAATAAGCAGCCCCGTATGTGGACATTGGTAGGAGCATATTCAACGGGCCACTAAAGGTCGCAATTGCCTACATCAGATGAAAGGTTGTTACCGGGCTAATATCATATAGAGCCCTCTAATATCCACCTGTTGTATGAATACTGTCGCAGTTTCATCACGGTTGCCCCTAAAACACTGACAATTAGGTAGCGGTATCGTTTCTCATCTACTCAAGGCGTATCTTGGGTCGCAGGGGCACGCCGTGGTTGCTCGCCAGTTACTTACTGACGGTACCTCTCATTCTTTGCAATGCGGGTACTTGTGTCGTTGAAAGTAGTGGTACTCGCACCAACTCAATCTATCGCCGCTGCGAGACAGTCCCCGAAAGCCTCCGCAGGAGCAGCGCTACCTCATATCCTGCACGATTAGAATTAACTGCCTCTCTAGAAACGAGTCTTACTAGTAGTTGACCTTGATTCTACGGCCACTCGCATCGCGATGATTGAGAGAGTTCCCCCACTCACGAATTAGGAATCATGTTGTAACAGCGGTCCCACTTGAAGGTGAGTCTCCATACAACGAGTACATACTTGAGCAAGCAACTAGATTATCAAGAACACGCATTTGTCCTAGCCTGAACCACTTTTGGTCTCACATAAGACAGCGGGAGAAGCGATTCGGGCGACGAGGCTGTTAAGTCGCACGGGTTGCTATTCATATAACACCTCGCCGTCGTGTTAATGATGTTTATGAGCCCTGGCGGTATGCAGAGCTAATTGGTTTGCATGGGTGAGGTAACTTGATTCCGTGTTTGGTAACGAGCGGCGGTCTACACCTAGTCTCGGCCATTGGAGACTGTTCGCGGGGGTAAAAAATCAATTATACAGACACTCTAGCCTGATATTGCCCAATTTGAAAGAGTTCAACCTTTCGCAGTTTAGTGAAAACTGAGCGGTTTGTGAATATTACCGGCCTTATATATCAAATGGTCCGGTACCAAGCCTAGACATGTTCGGACTTTGTTCCGTTAAAACCGAAGATTACGCATTTCACGACTCTAATTTTTAGAGACTGCTAGACAGGATATCAGCCACGGTGCGTCCTTCCTGTATCGCTACGTACTTGCTCCTCGCCTTCTATATAACGGTACTGGCGAATTCGACAATACCCTAATAGGAGCTGCGACTCTCTAGACGACCAGCATGCAGCTTATTTAAGTCACACAGATCGTAAGAGAGCCTCGCAATGCTCAGGATATCGGCTGCCCTAGAGGATTAGCGGGAGTTGGCTGGCGCTATTGAATGTGTCAGCAATGGCAGCTCCAGGATCGTCCAAGGAGCCGCTGGCCCGTCCCATATGGTCTTCGTATCAGTACTGATCAAAAGGCGCAGCGTGAATACGTTTTCCTCCATAACATCCAGTCAGTAGCTGATAACGTTCTCACCCACTGACTTCCAACAGGGGCGATGGACCTCCAGGCGAGCCATGGAGTTCGACCCCCTTGAGGTATAGTGTAGGAACGGGATAGGGATTAAAAGCGTTGTGGTTTACTCTAAGCAGGCTAAGAGGCATAATGTCAGCCCATCAGCGCGCGGTTAGACCTAGCCGATTGGCGCGTACCAGGAAGCCGTCCGTCAATTTACAATTGGTCGTAGTCGGGATACCATCTTGAAATGCGTAGTCCGTCGCGCTTCGGGGCGGGCTTGAAGTGATCATGTGCACCCTTTAAAGCCATAGCCATACAACGTACCTCGGCTGAGTAGGTTCATTCTCTTGGAAAAATACCGCTAAGATAGAGACACCTAAAGTAGGCTTCTTAATAGGGCATGTGTTTCCGCCTAGCGTTGACAGCATGTGATTAACCACTTGACCATCTGTCACGCGTCATGCTGAAGCTTACCCTTGATACACGGGCCGTTATCCAGACGCCTGGAAACTCGAATTCATCAAAATTCCTCCATTTCTCTGGCCCGACAATCCACTGTAGAGTGGGATCGGTGTAATACCCTGAGTTTTCTAAACGGCGACACCGATGCTATTTAGTATGCCACGCGCTTCGTACTGTTGAAAAGCAATTGGAACTGGATGTCTATCCCGTACTTTTAAGTCGTAGATCCGCATCATCCATGCTAATCGTCGTCATTAACCAATCACAGTCTTGCAAAACTCACACTGGCTGATTCTGTCCAGCGTTACCGCGGTCCTGTCCGATTAAGAGCTCTCTCTGCCTATTTTCAGTTCCCTCCAGGCGTAAGGGACTTGATGTACACGCTCGACGAGTGCGGCTCTCTGTGACTAGAAGTCACTCGGGCTAAAGACGACTAGAGGTTTTTTCTGCGATCTAATATTAACCATACCCTACACAAACGCCCACCGGTTTTTATCACAACCACAGTAGCGGAGACGTAAATCGTTGTGCACAGAAAGTTAGGTAAGTTATCAAAGGGCAAAAGAATTACAGGCCATATTCTTTCAGTGTGACCTTCGAGTGCGGATTAAGTCCGCGGCAAAAGCCTAGGGACGACATCTTAGTAGGCTATCTCCACCCAGTATCTGGCGCGACGTCAGGTGTCAGTACCATGCTCTTATGCCCCCCTTTAGGTCCACCTAAACCTCGAGGTCAGCGATTGACCCATGTAGATACTGGTAGACGTCGGTGCCACTATGATCGGTTTTGTCGTAATTCACGAGCATTTGTCATACTGTTAAAAACAAACTTTGACGACAGTTTGGTGCTAATTTGAGAACCCACGTGGGGCCTGCCGCCGGATGTACGGTATGATCTCACCGGGTCTCCATGATACCCCCATCAGCCTCATGTTCTGGCCCGTCCTCAGAAGCGAGCCCTCAGTCCTTTCTTAATCTCCAGTTACTTATCAGGGAGTCCAGAAGCTCCGGGTCTTCCAAATCCTCCGCGATGTCTTAGTCGACCCTAGGGACACCTAGAGGATAGTCGCATTTGTAGTGTAGCTGTCCGAGAGCAGAACGCTGCGGCAGGCGGACATTGAAGTTACGGCTATGTTATCAGTCATCCTCGGCTTATAAGAGGCTTTGGGAAAGTTGGGTTAGTGGCCCAAACGATGATATGCACCAACCAAACTCTTTCACAGCAATCCCGCAGAATAATTGCGGCGCGGACGGGCGATGGGCCAGCTGCTAGGGTAGCGACAAAAGGACCACCAGCTTCTGTCTGAAGACAGGGGCCACCAAACCACCCTTGTCCAACTCGTATGCCCGAACAAGAATCGAGATCGAATTGTCACTTATCATCCCGTACACAGTAGGTCCCGTCTACGTATTCCATGCGTTGCCGCAGGATTTACACCAAGTTTGGTCGCCGGTCCTCACTCGCTAACTACAGCTGTGTAAAGCGCGGGCGAGGGCGATGGATACGTACCCGAGGTTATCAGGCTGAATTGCTGGTGAACACTTAGGTACGACTCATACGCGGGAACCCGGCAGAACTGTACATTCGCGGTTTTTGTCCTCTAGCTATTTCCAAGAGCTGATCACGAGAAGTCTCTCCACGCACATCGCTGAAGCGAGACCCTCGCCAAGTGTTACTTATAGACTTCTATGCGGCGCCGCAGTTTATACAAGTCCTCGCTCATTTTAGTTCGTGCCTACACTGTCACTCCTACGGTACGCCACGGCGCGTGGTGATGAAAAGGTTTCAAGTTCGTATGCGAAAATACAGGCCATTTAACACGGACTCCGCGTCCGGGCGGGGCCGTAACGTTGCCCGTACCCGGACAGTTCCATTGTGAAGCACTACAGCTTCAATGTTACCAGTGTAACGCCAGCATCAAGAGTTGCAATGGTCTAAGCGTCACCTGAATTTGCTATATCCTGACGAGACAGGCGTGCCCCATCGGAATGGCAGAGGTAATAGGCAGCATCCCCGCGGATCTTCATCTGCCGTCAACAGCGCCGGATTTGTAGGCGGGGCGATAAGCACTCGGGGGTCATCGTTTAACCACCAAATGGGCAGTACAGGTGTTCTCCTGTCGGGCCGTGTAGTTAGTCCGACAGGTGAAACAATCTGTCAACGAATTGGCGCCTACTTGGTCCGCCAAGTGAGATCTTGAAATTACAAGTCTTTCCGGCGCAATCAGGCCACGATACTGGGGCAACACCCCGTTGGGACGACTGGTAGAAGCTGGACTCCGGGACGGGGCACAAGGCGGTGGCGGCATTGAACAAGCACGCACGCTCCTCCCTATACCTACGACAACTTAATCTTGTACCAGACAACCTAGGAACCCTCACTTGACTGCTCAAGCCGAAGGGAACAACCCCCTCCTTGGACAAAGCTTACTGCCAAGGCTTGGTACGTTCGCGTGTCCGTTCTGTATCGGGACCTGAATTGCGTGTGCAAGCGTCATTCGCCATGGCCACTACGGCCTGGTCGGGCACCCAGTGTGATAGTCTTTCATGCGAATGGCCCAGTGCTATCTTGTTCGCAACGTTCTGTAGATAGAAGGCCGTGGAAACTGCGACTATCCGCCGCACAACCTTCACGGGCCCCGCCTCTCTATGGAATACGTGTCTCATGTGCGATATGAAGGTTGCCCTCCATAAAGCAGATAGTGCTGATGTGCACTACTTGGTACTTTGAAACGTATTAGTCTCACCTGCTGATGATTCCTTTAAAGCGGTGCAAACAGCCTTCGGCAACCGGCCACTTAGGCTCGACGGACACCATGCCTTTCTGACTTACAAAATGACCACGCCCCTACGACAGCTGTCGTTACCAGGCATGCCTTAGCTGTCAATGGCCTCATCCTGCCTCCTAAGGGGGTATGTTCTCCTAGTAGCCATGTCATAAAACCCCTGCGCGAACCGGGTTATCTACGAACAGTACGAACCCGGCGCGTTTTGGCTAAAGTGGGGTTTATTCAGGGAGCCCGACATACCACAACAGTAACAATAACACACTAGCCGTTAGCAGGGTCGACGTGCTCCCGTTCGCGGAATTCTGGCTGAACGTAAATACGCACCTGAGCGAATTTTCCACGAACACTTTTACATTTAACCTCGCAAGAACCCTCCCCATTAACGAGCTGAAACGCTTATAAGAGAGACGTAATGGATCAGAGACCATCTCCTCCGAGAAAAGTCTGGGTGAATCTACGATGGGTATCTGGAACCATGCTCCCTCTTCCGGAGGACAAGCCCTTAATATGGAATCCAGACTAAAGCCATAGCGATAGCAGCTTAAAACTAAGGCACGTCGTCCAGCCAATTCGGCCGATTCCGTTTGGAACAAATAGCTGTCTGTCGCAAACACGACCCTGGGAAATGATCCAGTTCTACCGCACCAGAAGCGACCCACCCCGTATTCGCGTGGTGTTCCTTCAAAAACGCCCCCTTGGGACTCAAGACGTCGAACTTCTTTACCCCAACCGTTGCGGGGTGTGTGAGAAACCGCAGACTTGAGAGCCCGAGTATCTAATGCAATCTTCTCGGGTAACAAGTGTGTGTACATCGTGACAAGTCCTTGGGGTCTGCAACAGTATGTCCGACCAATTGACCGACCGCAGCATAGTCACAGTCGGAACTTTCGGAGTAAATCCAGGAGGGACGCCTATAAACTGTTCATAGGTGTCAAAACGGGCAAGGAACCAGAGAATGATTCAGTGCATTCGAAGAGTGGACGCGGTAACGACGTATAAATTTCGCGCTTCCAAGTAACACCGGACCGCGATGTGTGTCGCATACCGAGTTCTATGACGATGCTCAGAACCTTTGTCCACTAGCTTGCAGTTGCTCGACCCTAAGCGTTGATCCGATCTGTGTACAGAGAATCCACTTCTTATCTCCGTTGGCCATTAAAGCGTTTGTGTCCAGATCGGACTTGGTGTGTATGTAAGGCCTGATACCGCACAATAGGAAATATATCGGCTAAATTAAATTGGCTCGGGATAGCCGATCTCGAATACGTCCATTACTGAAGCAATTGGTTACGCGCGAGCCCCCGATATTGACTTTGATAGTATGTCCAAACGAGTACAAACTCGTTGCACTTAAGTTCGACTTGCTATGAAACATAGCCGTAGGAGGATTTAACCGAGCCACAGGCGGCTCCCTCTATCTTAGCGGATCGAAGTAATATTCTGTTGTCATGGCTTTAATTGGGAAATGGCTGCCGAAATTTGGCGAAAGATAACAAGTTAATAAGGACAATTCCTACTTACCAAGGGTACTGTTTCGCAGCGGCCGGTACCCCTTCGAGACTCTGTTGATTTTCCGTGTACACAGGCAAAATGCATTGTATATCCCGGATTGCTTGCTGCTCATTAGGACTTTACGACATTAAAAGCAGACCAATTTGACCCTGCTAACCATTAATCAAGGTAAGCTGCTCTTTCACGGAGAAAGCGCGTGTCTGACTGAGTATATCTAATAGAGTTAGCGTACAGTAGTGATTGGCATGCCATCGAGAGGCGACGTCCGTAGTCAACCTACACCGTTAGAGAACCCTTGTCCGTTGCGCTTCTTAATCTTTTGTCGAAGAGGAAGTACATGAAACAATCAGATTAGTCATTATGATACACATCGGTTGGGACACATAATCTTTGCGCTGCAATACCCTACAGTGGCTGAAAGAGTGGCGTTGCACGCGGACTAGGATTACAGCGTATGCCCAGGTTCTTCGTACGTTCCTATGGGGCTGTAACTTAGTATCAGACAAGCTTGCAGACCCGCGCCCGGGCACCGTGCTATCTGAGTGCATGCCGTCGGCGTAAATCGTGCAGTGATTCTCCTCCGCCAACGGCCCTGAACAGGGTTCGGTGCTACTACTGTGGCAGGGGGGCGGGGTGTATGTTACCACAGAAACTCCTCACGCGGCAGGGTTGACGGCCAGTATACGCTATGAAAGCCAACAATTGGAGTATGAATCCCTCACGAGGCTTAGCGATGCAGAAGCTCGCCTGTAGACTAGTCTTACTAAGTGAAGAAACGGCGGACGAAATCGAACGTGGGTTATGCTTCGATAGTCAACCATGAGTGCATTCAGTAGAAGGTCGGAGAATCCACTAGTGTGCGGAATTATGGGTTCCATCTAAGGCGAGAAGGGTCCGGGTTAAGTCCAGCTTCTACACTCAGCAGGCAAACATACGTGGCCAGGCTCCTTTTAATCCCCGGAACAACCCACTCCGGGGGGTGATCCCGAAAGATAGATTTATGTTGAACAGGCAGTAGCGATTAAGGCAGCGGGAGTGGACATATATAGCAGCAAATTGTCTCCCTCGGGCCTTAGCTCTAGCTCCGGATAACATAACATACCTGTGCATTTGGCAGCAATGATGATCTTGTGCTAGGTTGACACTGGTTGGCGGACAACGCCCAGTAATGGGACTAGCCTGTGGCAGTTTGAGTTTATGCGTGAGCACAGAAGCGCGCCCGTCTTAGGTTGCTTCACACGGCCGATGTCACGGCAAATTGGTTTCCAGGGAGAGATTGACGTGTCTTGAGTCTACGCTCCGCGTTAAATACACCCCCCCTGACCACGTTATTGTGCGTTTAGACCTATTCAGTTCAAGGCGTATGGGAAGACAGCCCGGTAACGTCGCGGTCGATGTCGAGATCCGCTGCGCCCGTTCATAGACGACAGCATGAGAATCGACGGTCTTGAACGTTAACAGATCTCAAAGTCGTTAGCATAAAGCTCCTAGACCAGTCCGTATTGCCGAATGTCCTTAGACTGATAGGCTATGGCTGCAGACTCACGCATCAGCCCGGCGGGGCTTCAGCGATCTAAAGCTCGCTCCGAAGAAGCTCGGGATACTCATGAAGGGAACTTGAAGAGTTATATCGAGCACGGAGAGACCACACATAGCCCTCTATAGATTTGTAAAATTAATGGGATTGTCATACATGCCAATGGTATGGGCTAGAGGCGCCTGTCAGTCAAGGAGATAATTCGTACTCCCTTACTTCAGGGTAGATCATGACAGTTCAGGTGAGGTCATCTAAGGAAGGTCGCTCGGCTGTAGGGGTCTCGTACTATTAGCCCACCTCAGCCCATCCGCCTTCACTCACATATTGTCCCCCTGGCCTAATGGAATGGAAGGTGATCACTCTGGTCGGGTACTACCACCATATAAACACTGTCGCTGTAATTGTGTTCACTAAATACGCAGTCGAACGTCCCCAGTCCACGTCCGGGTGATGCCGCGCTCCCAAGTGGAAGGTATCGATTTAGGAACCATCGATGCTAAAGACTGTATCCTCGTACGCAAGCTACCGGGAGACACCTCCGGGAGCAGTGTCGGTACGCTTGTATAACATCCTGACAAACGCGTTAGTTGAACATTAGGCGTGGTTCGCAAGGTCGGCACTCGCTGAGACAGTGTAAAGTTTGGCGGCTCGTTACCATGAGGCCTTTTCTTCTTGGGAGGTGTTAACCCAGGGCGTGTGCGTATACGCCATAGGACTTACTAAGTTGTGGACACCCGCCCGCCTTTTTAGTTTTAATAATGCCCGGCACCTTTGCTTATGCGATTGCAACATCGCTGCAATAAGCTCGCTGGACGTATCAGGCCGGAGAGATGCTATCGCGAAAGGATTCTTCCCTCGCAGCGCGTTCGCATTCTCATTTCGATAGAGCTCGATTAAATCACGGAACGCTATGGAACAAGTGTGCGGGACCGAGAGCAAAACGCACGTGTTAGCACCTTGACCTAAGAATTGGGGATAAGGATAGCCGAATGAATGCACTTAGATGCGTTTCGCCGCCTGCTACCACATGAGCTGAGACAATTAAGCGCAGAGATCATTGGTGCTGTCAGTGCAATACCTATGTGGGCCGCCTTACACTAATTAGACATCCGCAGCCGCCCATTTGCGGCACGAAGGGAAATCCGTTAAACACGGGCCGTCCGAAGTCCACAAGCACAGGTCCGTGACCCGTGATGGTTCTGTTTTAAGCCATCTTTTGGCGTTCTGTCGATGAAATGCTGAGGTTAAGTGCGCGAAGTTGCTTTCCGACGAGACTGCTCCCTGCCGGCGGGTACGCGTACAGACCCTGCGGTCCGCGGGCGTGATCTAGATTACAATCGACTTCCACGGCTATCATCCGCTCGTTTGCCATGGAGTCGTGACAACCAGTCTAATTGGGCTATATCCCCTGCGCGAGTACTCCGCGGCCATAGGTATGCGTTACAGCGTTCCGGAGCAAAGCGTGCACTAGTATCTTCATCCATGAGGTAGACGATAGGGGTGGGATCGACTCCGATGCTAATGTCTAAAGTCCCCACGCATCTATGTCCCATAACACATGACCTACAGACCTTCTGTTGTTAAGATGGTGTTTGTCCGATGAATCGTAAAGACTAGCTGACTGTACGCAAAACTAGGTAGTTTGCTATCTGACCATGGGGTTTGTCTCTGTCGCTTTCGAGGGAATAATTAAAATCCGGGATGCGGGTTTCAGGCCCACGTAAGGCATCCGACTATCTACCAGCATACACGAAATTCTAATGAGTTCAGATGCCGTCTAAACAGCTGATAGTGACTGCCATCCTAGAGGACTCGCTCTCTTCTTACCCATGAGGTACTGCGTCGGAGTTTACTGACCACCCCCGATCCAGCTAGAGCGATGAAAATTAAGTTTTACGGACCATAGATGGGGTGTATACATTTACAGCAAACAATCGCCCTTGCTCCTAGAGTTACATACACGCCTCTACGACCTGCCAGCCAGATCTCGTTATCGCCCAAGGCAAAACTTACCGCAACGCGAGGCGCCTTTGGGACGGCTGCCTCATATAGCCGCCATACACATGACTACCCAGCGACGTTTGTAGAAGCAATGGGCGAAGAAGGACGATTCGAACACTATCAGGGGTGCTTCAGTTTACCTGGAAACGACTTTCTCTGCGATCCTATTCACGTCCTAAAGCATCTCCGCTAACTCGACGTTGAGATGAGCCATTGGGCTGGAACAACAGTCAGCCCCTAATGAGGCTACCTCGACCAGTTTTGGTTGCAAAAATCCGCCCCCGGCTCTCCTTACAATTTGCGGTCGCTTAGAGACATCATTATTCACGGTCGCCAGGATACTATCCGTACGAGACCTATGCGTAGATTTAAGACCCGGCATTCGTTTTTCCGGTGTATAATGATTCAGTTATTACCATCCACATCTAACATCAAGAAAACCAGCAGGCGTGATCATGCTTTAGGTGGGTCCAGAGGATGACGACAGGCTCTTGCTTATATCCGTTCACTACTGAGGTCCCGGGGACCCACATAAGGAGCATTGGGCAAAATACGATTGCAGAAGGCCGCAGCTGTGCTACCAATTCTTGCCTTCCGTGTTTCCCTATCACTAAGACACCGCAAAGTCCATCAGGTCGAGTCATCCCCAGCCGGCAAGCAGATTCAGGTAACTACACAACCTCACTGAGCAGCGACGGCTTATAAAAGTGCGTTTGTTAACTTAGGTCTTTAGCTATAACACGTGGCTGATCACACTCATTCATGTATCATCGGCGCTAATGCGGCAAGTGGGAAGACATCTAACCGAGGGTAAGAGTTCTATCTTAGATCCGGTATCGTCAATACTGACGCGAAGTTGCGGACCATTGGAGTGCGCGCCCTCTGGCTGTTTGGGGTATTTCATCTCTGAACATTTAAACATGTAAGAGGTTGAACGATCATGACTCCAACCACCTTCGCCCAATTCGAACAAGTTGACGGCTGTGTGAGACCCGCTCATAATACCAATAAAGAAGGTCACAAAGCTTAGTGCACGTTTGATGTGAGCCCAGCCCGCTAGAGGCCTGCTAAAACCTGCAGCTGACAGGGGCGCCAAGACGCGAATCTGTTGTATTACAAATCAAAGAAACCGACACATTTTAGACTACGAGCCAATTACGATATCGGGCCCCCTTCCCGCCGAAGAGTTTGTCAGTCACATAATGATCGTATGTCACCGATACCCCTGGAGGGTCCAAATATAAAGAACAGTCATTTGTAATCTGGGAAGTCACAACATTTATGACAGGAGATAAGGTTATACAGCCTTGTAAGACTCGGATGATAATATTCGCTCTATTGACTGGCAAGTATTGCACTATTTGCGAATTAAGAAACACCCGCGTCCCAGCTCGACATTCCCGAGTGTTCATTCCTGGTCCCATCTTGACCGGGTATGGACTGGGTCCCTCGATGAGCACCTGTTCGCTATCATGTGGGCCGATACATCACACACCCTGATCAGGGGAGTGCCGAATTTGCACCGTCCCTGTGGCATGAAGTCTGGTGCACCGGTTACAGGGATAGCCATTGAATTCAGGGCCTCAATGTAACCAGTAAGAGCAGTGGGAAAATGCTGATGAGTTCGGATGTGGTTGTAGTCGACAGAGTACCGCGCATACTGCTTCTTGAAACGTCCGCGAGGGGATATTCGGCTTGTCGGATTATGGAATGGGATCTCATAAATCTTGAACAACGAAACACTACATGCGTCCCGACCGTGCCAAGTCATCTATTTAGAGTGCGCTTAATATATCGTAAGTAAGTCTGCAACTGTTCCGGCCGTACTTCTTTGTGGTTCCTCACAAGTATGAACCTTGTGCTCTATTCCACCGGAAGCTGGTGACTGCAATCAGAAATCGACCTGTTCTCGTTGACTGCACCGGCTGACGCTTTCATACTCTTATACGGACGAATGAATTTCTAGTTAAAGCCGCGACTTTTTAGGAACAGAGGCACCGGTTGTATCTTCGCCCGGAAATATTGGCAGCTTTTGTTGCGTAGTTATGGCCATACTAATACCGCCTCATTAGTCTATATTGTTTGGGCACATATATCATCACGCGTACCCGCCCAGTACTAACCATCAATGGCCTCTACGAGACAGTAACATTTTAAGACCCTTAGGCGACTCAAGAACTTCTCAGCAGGGTTCTCGCGAGATCATAATCTATTGCGGTTTCGTTAGCCATAGCAGCATCTTGTATTGTTATCATGTCAGGTATGCGCTCTTGTGAGTCTGACGAGTGGTCTTGTTAGTCGTTTCCACCGCTCGATGCGAGTGAGTGCAGACCGCTTCATGTTCTTCGGTTGCTCGGAATGGAATGGGGCACGCAGGCGACAGCCTACCGTCCTGCCTGGCGAGGGAAGATTTCAAAGTCGCTACCCGATCTGGGTTGTTTCTTAGTCGGCATATTGGAGGCGACTCATAGAACGTTACTTCAAACATGGGATTTCATAGTTGTTTTAGAGCCGCAAGTCTTACCACACGATTGCATAACTTGATCGTATTGCGAAGTTTGAGTCCGGGAAAAACAAACCTCTAATGATAACACATGTCTGCCGGACCGCCTTGCAGTTATCACCGATCCGGTTCATAGGGTAGCCAGCTTCGTATGACCTTAGCCGATTACTGTAGAGGCAATTGCAAAGTCAAAGCGAAATTCAGGCGTACCTCAGACATAAACGGGAACCTCGTCAGCAAGATCGGTAGACCTAGGCTACTCACGCAACGCGGTATATGCCCGGTTACTACGCGACCACGTGATGTAACCTGCTGTGTTTTACCTGGATTGCCATCGCTGGCAGATAAGTATACTAGTGATCGTGTATTCAGCATTAGGTTATTTAGAGAGAACCATCCACGTCAGGTTATTGTTGTTAACATCGTGTCGGGCTCCATCAGAGATATCCGATCTACCCTGTCAAACAACGCGAACACGAGACAACTTTTTGCGGCGCCATTTGGCCGCCAGCCGCAGGCCCCCAACGTGGTCTTACCATGAAGGGGAAGCACTTGGACGTGAAAACTGAGGTCGATTTAAGTTTGACATACAGGACTCTACGATGACCGTCGGGTGACCAATAAGGATCTACATCTCTATTCTTCGTAAGCACTGATGGTAGTATTCCAGTAATGGGAAGCTCGCAGCCACTTTGATGCTCGTTTGATAAACCCATATCGACCAAGAGCCGGCGGAGTGGCGGTTTGCTTATCAAATTCATACAGCCTGAATTTGGTTATCAGATGCCTAGCCCCTGAAACTGCAGCTCAAGCCACTGACTACAGAACTGCCCGTTTCGCCGACATCAACCCAATCGAAAGCACGATGAATCTATCGGTAAACTCATCTGTGGATCCAAACAGCATCGTTGGCCAGTACGATTTTTGCATCCCCTAGGACAGGCGGGTCCCTAGTCCGCTTGGGGCCACAGCTTGAAGGATTAGAACAAGTCGTGAGGATAGTGTTCTGTACTTTACTATAAGTATTTCGAAACTATCCATCCGTTGAGTTGACTCACGTTCGCCATGGGTTGAAGATGGACCTTCAATGGTCCAGTGTGCGCAAACAACCAGTAAACGTATTATCGTATCGAGGCAAGCGACTCATCACGAGCGCCTCGTTGGGATCGCAGAAGGGACGTAGCTGGTCACCCCAGCTATTAGGCCTGCATTGACTGCCCGCGTCATAATTCGTGCGACAGTCTATCCCGGTACTCAGCTCTACGTTATGTCTACTAGATACTCGGCTCTTGAACACCTTCCGGGCAGGACCGGAATCCCTGTTCCAGTGCACGATAAGAAGAGTTTTCGCTTCACTGCAATAAGCCGTCGATTACACAGTGAATGTAAAAAGAATGCTGGCCATGAAAGCATTGTACAACCATGCCCAGGAGGGTCTCGGACTCTCTTTCGGACAACCGCACTGGCCTTTAGAGCTGGAGCGCGGGTAAATTTCGAGGGGGTCGACTTTTTCGGGTGATTCCATATAGGCGATGTGAGACACGGAACATAATCTGTCACGATGGATGTGCCCCATGACGACCGTTGACGTACGACAACTCGAATTCAAGGAGAGTAACGAAGCGTCCGATATAAAGTCCCAACGTCTTCCGAGATCAGACCCAACTAGAGGATAATATTGTGCCAATAAAGACAGGCAAGCACTGTGGGGTCGTCAGTTTTGAGCCTAGCGTATCGGCAGAAGCTCGCAAATAAAGTCACTGACGCACATCGGGTAACGTGGTGTCCCACTGGATTTAGCATTGCAAACGGCTCAGAGCGCGGGATATCGGTGTTCTCGACTCCGAATGTATTGAGTGCGCCGAGTCGCCTCGTTATTCCGGCTTGAGTCGCTCTCTTCTGGGATATGAGAGAATGCTTTCAGCCTCTGACATCCGCCGCGCATGAGAGAGAGATGGAGATAATAGGGTGTTGGCATTCGGGAAGGGCGCTCAATGCCTACCTCGTTCCCCTCATGGTTCCCGCCTTAACTTGAGGAAAGATGGCCAGTGTACTCGGACCCCCATGGAAGATTGGTTTCCATAGATTCCAGCAGACCTAAACGCTATCTGTTCTAGCACGTAGGTCGGAAAAATAACGGCCGTAATTGGTTTTTTTCCCAATATTACAGCCCTTCCAGAATTACATATCGTCACCAACTAGGAATACAAAATCGAGCCAGATTCCCAAGCGCTCCCATATCTTTATGGTGCATTGATATAGCCAAACCCTGGTAGAGAAAGGGTGGGCGTACATCGTCAACTCTTCTGAGTAAAGTCTCAATTGAAGTGTCTTACAAAACGATCCGATATCGACTAGCTGTCTCTGTTCCTTCGGTGCCCAATAGACATTGACAAGCTGTGAAATGTTGGTTGCACTAACTTTGGGAAGCTGCTACAAATGGCATAACGATTACGACCGACGGGCTCTATTCCCTTTTGCCCCATCCTACTCTGTTATCTCGCAAATCCCAGATGTGCGACCTCCTAAACAGACAGAGTGGTGTTCCCGCGATCTGCTAGAAGCAGAGTGGTCTGGGTACGACGTACCTTCCTCGCGGAAAGTTAACGGGAGGGTTACCCTCCTGATTAAGCTCCGCGCCCTCCATCACGACTCCGATGGCCCGTTCAAACGACCCACTCGATATAACAGGACTAAGCCACCGCGCAGGGACCGCTTGACCTATCGCCAGCCCGTTGCTGGGGGAGCCTTGCTTTGCAAAATTAAGCCCCGAACCGGACATTTGCGGTCATTGCAAGGGCAGATCTACTCGTAAAGTTTCCCAGTATCGAATAGCTACGAGTAAACGGAAGCATAAACGCATCAGTTATTCCGGGAAGCTCTCTTTAACGTTGCTATCTCGGTATTAAACTCATTTTTGCCTCCGTCACTTGACCCCACCGGAGACAAAAGGAAGCGCGCTCTAGGCGAGGTATCTACTCGTAACCGCATCCACCCGAGCGTGGGTATTTGGCCTTGGTAAGGAATCTATCGATCATCTGACACGCTACTCCGGCTCTAAATAGCTTCGTTACGGGGACTATTCACAAATCACTGGAACCCATCTTTGTAAAATTGGGGGGCTGGGGCCATACTCAGTAACTAGGCGGTTTCGTTATCCACAGTAAGCTAGCTTGCCCCTTCAGTACAAGATTCAGCACTCTATGTCTCATTGCGGGTGCGGTCCTGAATGACTGTATTTCTCCAAAAGTCCTCTGAAGCGTCATCATCGTCAAGCTCCTTATCCTCTCTAATTGTCAATATTCAGATGTTGCGTCCATCGGAGCTCGGTATGGCGTGATAATACCAGAAACGTGTTAAATGAATGCTGACGGAAGCCGTTCGACCATTCCCCGAGAGTGCATGGTCGTGTGGCGACAGATCCTTCATTTACGCTACACTTTTGGCGGTTAGACCTCACCTTCCAGGTGTGTCGTGCGCATCATTCGGCGCAAATGACAGGTTTTGCCGACTTGACGCCCTATCCGTGGCACCCCCCTACCTTCGTGAGCGTTGGCCCTGCGGCACTTCCCCAAACCCTGTACATCGTGGGAGATCAGAGACACTCATAAGTACTAGCGTTGGAAGAACCGGTGTTGGCGGGTGTCAGCTCTCTCTGTATCACATACTCTGAAGTCCTACCAAGAGGGACGCTGCCTACGCTACGCCCAGGTAAAGGCATTGGACTGCTTGTTTTGTTCGGCGTCGCCCATTCACTACATCGTACCCAACGGTCTAAATTGTTGGACCAATTTGTTCCACGGATGGGCGAGCTGCATCACCTCACAGCGGAACCCCTTCATAATTCGCGACCTTCCCGCAAGGTGAGGTATAAGGAAAAAACGGCATCCCGTGCAGTCGCGGACCGCCACTGGACAGGTTCTGAGTACTAGATGGGTGTGGCCGAGAAGATCCGGACTAAAAGTCGCCTCAATCATCCGTTACCAGTTTCTAAGTGTATACGTGAGCGACACATTAGCTCTGGGTTTCACCACCAGTCGAATGCGTCAATTCAAAATTGGCGTCCTCGAACACGCTTTACGAGCGATGCTCATCGCGACACTCCAGTCACTGTTAAAGGTGTTCCGTTAAGGCAAGAGCAGCTCCACATATAACGTCCATGACATGTTGCAAGCCTGCACCATACGTCCTTTTGGAGCCGTAACCTTTTCCGGAAAGAGGATTCAGGTCAGCATTTTAGGTCTCTATTAGTGACATTGCGGATTCGCTCCGTTAACTCAAGGCCATCATTTTGGGCACTCCTCGGAGGGACCTAATTTTACTCCTACTTGCGCGATATCGATTGACAAAAGGAATTGCGTCGTATTTTCCATTGATATAAAGTGTACTTACGGCCCTGACATATCCTCACGGACACGCCAACACCACCCCGCCGTTCACGTGCCCCCCCTCGGGCTACCAACTGCACAAGGGCACCACAGCTTGACATCCATCAGCATGTTTTTTCGACTATTCGGCCGGCACCAAGTTGACACCGCCGACTGACGCGTGGTCAGCCGATCAGAGGCACCAAAACGGGGTCGCATGCCTTAGACGAATAAGGGTGCCATCGATGTTGGGTATTATTACCGAAACATTCGGATTAATAGTTGAAATAAACCTCCTATTCCAGAGTACTTACCCTAGACCTCAAATAAGACCTGCGAGTGGATGGTCTTAACAATCGGGTGGTCGTGGTCCCGGTTTGGGATAGCGAAGGATAGGGCAAGAGGACGTCCAAGTCCAGTAAACTCCAAACAGCACGCTCACTCTAACAGGGCGGTGATAATGGGGTAGTTAGACGAGCACTCATCGAGCAGTACTTGCAACTGTCTTTCTCCAAGCGACGCTTGTCCCAATGGCATCCGTAGACGACGATGTGTCGGTCCGCCGTCAGGGGAATTATTCGTATGATGCCTCGAGTCGGTCTCGGGAACTTTTCTCGGTTTCCGGTTCCTAGGGTTGCATCCCTAGGTCCAATAATCATCTGTCGTGAAGGGGCGAGTCCTTCGGGGAGATGCTAATTTCTATTGGCCCCAACCAATTTTATGAAGTGTCGGGCGGCGATGTAGTAAAATTTATTTCTATACCATGAAGAGTGCTCAAAGACTGATCCAGGTTCTGTCAAGCTTTTTCTACTATCTATGAGACCCTAGCCCACTATGCACTGGATACGATAACGATGCTAAGGACTACGATGATGCGTGCGGGTATTTACGCTTTGTTGGTTACCATAACACCCACAACGGATCTCTTATGGTTCTTTGATACTTTAAGATCCTTACAATATATCAGACATGTCTACAAGCCCATTCGGTGAATTCTTTTCTCTCTGAAGAGGGTTTTGGCGTTCAACCGGGTATGCTGAAAAGCGACTAAAGTTAGCGCGAGAAACATTATAACACAAGCTCGCTGTCTTAGCAGGTCGGGCTATGCCCAGGAGGGGAACGATGATGGACACGTGTACTTGTGCGACCGGTCATGGACATATCTCTCCGTTGGAGCGTCCGTTCCCAAATGGAGAGAGACTGTGACAGTTATCTACAACTGCCGGTAGCCGTGCCCACTCCTACGGTACCGCTAGTCACAGGGATAGCAGGAAGTTAGTCCCAGTTAGCCATCACGCGGAAGTTATTGACCGTCTGAGTTATTGTTCCCATTATGAGCCTAGCTGAGATGAGTCTCAGCGCGGCTCCGCCTGTTGATTAAAATGTTTCCAGATTAGGTACTTCCATGAACTGATTTGCTCATACATTGACGGCGGGCGAGATGACTACGCTTGCCGACTACGTGGGCTCGGCTCACAAGCTGCGCGGAGTGATCGAAATCAAGTCAGTTGCACATAGCCTCACCCAGCACCCTTGACCGGAGCAAAAGTGCTGAATGACTGCCCGCGCAACAGCTCATGTCTAACTATAGGTCCAAGGAGACAACTTGGAGAACGTTCCTGCGCAATGCTCCCAAGGTAGCCATGTGCCAGGTAAACGCCTGCTAATCTAGTTAAGGTTACACACTAGAGGGGTCCCATTATTGCTCACGTGGGCCACGTGCTACACTTCGCCCATAGCGTACGGTCTTTCACTAGTTCCGGGTACCCACATTACGTACGTTCGTTCACTACTCGCTCAGTAGCTAAGATCGGGCTCTGGGGAGTTCCAATAGAGCCAGGTCCGAGCCATCAATTGTCTGACATATTTTAACTCTAGAACTAAAGCAGCCCAGGTGGGAAGGCCACAAAGGAGCAGCCGGAGACTATCAGATAAATACATACGCACCACTAGTCGTCATAAATAAAGGAGTTGTCCCCATGCTACTTAGGATTCAACGGCTGGTAACGGGACGACAAATAGGATTACGGTTCTGTCTTAGTAAGGCTTATTCTATGGAATGGGGACGTTGGGCCTTCAAGAACGTAAGGGAATGTCAAGTCCGGCTTGGTTTTTTCCTGATAGGCGTGATACGCGAGCTTTTGAGTGTAATAGCGGGAGTGTCTGTTGTTAGATTACTTTTTCCGTAGTATCTCACTCAAACTAAATTAACACCAGTAGGTATTATACGCGGAATCTTCCGCTTTTGACGTAGAGCATCCCGTGTCCAAACCGAATTGTCCTTTTTGGATCGCATGACATAAGGTTAAGAATTTACCACCACTCGTAGGGAAAGACCAAAGCGGGACAGACAACTGCCAGCGGGGCATAGCCTACTTCCTGTTATATCAAGCTCCAGCTGACTCAGAACCAGAGTCAGTAACGCCTATCTCTGACCTTTGGGTACTCCCACGCGGTATCATTGGCGACCAGCTTGTGGAGGATCCATTTAGCCACTCAACTTGTTTCTAGTAGAATTGAATAGACACTGGAGAGATGGCCAGCGACTGATCTTGTCATACACTTGTAGGTACTGTACCTAAGGTGGTTCAATCCTGGCTACGGGTAACAGTTGGTGAGGTGGGCCCTTCCTTGCGTTTGATGGGGGCAGCCTCGTTGGGACCGACTACCTAACCAGGTATGGTTTCCTCGCAAAGCATGGGCCGCCAGTATCAACTTGAATTCCCGGATTACGTAGCAGATTACTCCTGTAGTTCTTACACGCCCTCCTCTAGAGAGGAGCCGCCACATAGGGTACGCTCGTCCTGGGGATATTCACTATACGACTGTGTACTCCCTGGCACTGCGCAATAACCGGAAATAGGAACATGATAGCAAATCACAGGCATTGACCCCAGTGAACAATACCAACCTCAGAAAATGGGGGAACACCCTGCACCTCCGTGCTGCCTATAATACCTCATATCGTCGGCTCTCCATATGAGGGATAAAGATTCTTGTGCTTCGAATTTCAGACAGTCGACCAGTAGAGCAGAATAATAATCGTCGACCTGGTCAGTAAGGGGGCCGGCTAACGTAGACGTTCCCTCACGACCGCTCAACGTGTCTAGACAAGCACACAGCATATTCCGTCCGGATCCACCAGTGTATATTGGTAAGTTGCTCCCAACTGGTCAGGATGATCCTCGAAATTATTTTGGATAAATAGATACAATGCCTATCCACCCAGGTAACACCACTGGTACGCTATTTAACGCCTTCTCCCGGGTCGCTTAACTAAGTATGCTACACCCACATGCTTCAAATATGGTCGTTTCACCCTGTCGGTAGACTCGTCAGACCTTGTCTCATACCCAGTGATTTCAACCGACCAGTGGTGATATAGTAGACCCTGGCGGTAACGATGTATCCTTATTGACTCACCTCAACCCCCTGTTCACACACATTACGCCCCGTCCGGGGCGAGTAGTGCTGCCAGGATTTTGGGGATACAAAAGGTCTCTTCCTTAGCGGTGTAGGGGCGGATTTACCTGTTTCTCAGGTTAGAGTCACATAAGCTCTGAGATAGATATGAGGGCGTCATAGGTTCGCACCGGACATACCTCGCATGTCCCCCTGGCGTAGCCACAAGGTGACTAGAGCCCACCCTGTCCGCGACCTTATGGCCCACATCTCGCTACTCACACCATTGATGTAATAGGGGAGTTATCCTTCGTTCAAGTCCGTTACCAGGTTCATCAAACAAGCTTTACGGATTGAAGCATCCCGGTAAAGACAGTAGCATGACTCCAAGGGCATTTTATAGCCTTAAAGGGCGTCCATGCGGGCCGGCAAGCCACTAAACCTTCATCTCGGACTGTTGGTCCTCTTTGCAAATTCATGAATGCTTTATGCTGGGAGACTAAGAACTTTTGAGGTTTCTATAGTTCAGCGGTGCGACGAAGTGGTCAGGCGCTGTAAATGAATGGAATACTCCTAGCGGGTTACCCCAGGCTTGAGGTTTTCCTAATAAACCCACAGCGTGGATCTCACCCAAGGCGCTAAGCCATAAATCAAGTCCCTAAATGTCCTTTTTAGAGCAAATGATCAGATCTCTGCGCGAAATTTGATCAATGTAGGACCGCAAAACCGCGAAGTCCCGCTGCAATCAAAAGGCGTTATACCGCCACCATTCCCGTGTGCAAATATATAGGCGACACCGCTGCAAAGCTCGGCTCATGCGATCATAACCCCACGCATAGCTTCCTCAATGTTATTTGCACTTCCCCCATCACACTGATATGCCCGGATGAACACCATTCGGGGTTTAATAGCCAGAAGATCCGCCTGCCTAAGATAGATTGTGGTTTCACCGAAGTAATGCCAAGCCAGTAGGTGACAAGACTGTTATCCATTCACGGGTGTAATATTTGGCGGTTCTCCTACAGGGTCGTTCCATGTGCAATGGGCCCTCTTACGACCCCGAGCAGCCTGAAGTCTGTCGAATTAATCTTATTCCTCAGCCCGCGGTCAGGAGGGCCGTAGGTCATACAATCAAGTGAACTCTGGCAGCGTGACGGCAGAAATGCGTAAGAACAGGGCTGTAACGATCCATGCCGGGTCAAGAGAAGGCAAACGGGGCTCTAACGTCCGATCTCGACGAAAATCGGAGGAACCGTCGCTAAATCGCTGTGCGCATTATTTACTCGGCTCTCTCTTGCCCATAAGTTTCTAGGTACGTACGCACCAATAGACAGGGGTATGTACTTTCGGGTAAGCACTGATCGTGGTGTTGTCAATCGGCTTCACTAGTGCTAGTGCTGAGAGTTCACTGTCCTTCTTCCGTGCTAGTTAATGAACCGCTTTCTATCCGGAGCGGTCTTCTTTCGCTCACTTGTAACATGCGCTAGTGGCACTACCGACAAGCAAAGCTAAGGTGCCTCCTCATCGACCGGAGGTCCCTCCGAGTTTAGACGAGCTTTGTTCACTCAAAACGAACACGCTGCGCATAGAGCAGGAATCAGTAAAGGGAACAACCTAACTGCAAACACGTGGCGGCTTGTCGTGTCTGACTACCGGGCAGTTCGGGTCCTAGGCGGTTAGTGAGCGGAACGCGTCGCCGGGCGTATCCGTAAGGATTGAAATAATTCTCTAAACGCCCCGCGTCGAATCTATGTCCCTTAGGCTGTGCCCGTCATTTCCGAAGCGCCCACAGGTAAGAAAAGATGGGTTTTGCAAGGCAAGGTTGCCGATTGGCGTTCGCAGCTGTTTAACAGCACATGCCGCGTGCTATACGGCAAGGAGAGCCTCTACTCATGACCGTCATCACACGCCATATGCCGTGAACCCCCCCGAGGAGTAAAGCGATGTTCTGCTGTACTTACTTCACAATTGTTAGCCGTGGAATTCGCATTCATTCCAACCGTTTCAATGATTCGAGCAGGCGAGGCTCCTGGGTGTTTCGTAGCAGGGCCCAACCACGAACATCTCCTTAAGCATCCACCCACGTGTAGTATGCACACCATAGATGGCATATGATTGTTCGAATGCTACCGTGATGCGCCTGTCTGACCAAATACCCTGAAGTTGCGGGCGCTTGTCCAAAATATGTAGGCGGGACACAGGGCCAACGTATTCCCTACGTCCGTGTACCTAGCTCAGGGCAAGTTTCTTCAGATTCCATTGGGACCCTGTAATAAGCAGCTTTTAAATTACGCTCCCTTCAACGAGACGGAAGCGATTCCGAAGCACCGAACCCTCAGAAATGGACATAGCTGGTGTTGTGGTGAAGCCTGAAATCCGACTCGGTACTATTTGTCATGGGGCTCCGGATATTTGTTGATCTTCGCTGCATTTGCGTCAAACTACGACACAAGTAGGATGGGCGTACCGCGAATTCATGACATGCGCCCCGACTATCATGAGCCAGAATGTAGAATCCAGGCAGTACCATTGGGAATGATTCCGTTGATTTGCAGACGGCGCATTCACTCTAAAGACACATATTGCAAGATTAACCTTCACTTTAACTCATGTCTCATACGGGTTGTACCAGCTGCTAACAACTTGACGTGGATGGCCGGGAAAAGACAGTAGTGGGTAAGAGGCATCTATCAGCGATACCACTTTGAATATGAATTATCCTATTAGAACCGTTCGCGTCGGCTTCTTAAAGTTAAAGGTCAAGAAGCGCCGGCCATCGTTAGGGCACATAGTGGCTATGAGTTGCGAACGTCATAAAACTCGTATTTAGCGAGGTCTCGGGACAGGATAGGGTTGCTGCGTAATTGTACGCGGCAGACAGTACAAACGTGCGCGCGGCGACTCCTATCTCTCCCGCTTAAGCTTTATATCAGCCTGCGCCGGGTGTGCGCGCGAAGCGAGTCTGAGTCCCGGTCGCTAGTGATAAGAAGGCACGTGCTCCCGAATTGCCCCCCTCTAAGGTGTGGTCCTGACGAGACTTTGGATCACGCCTCTGCACAACCAGGGGTTAGCAAGAAAAGCCCAACCTCCTTGACGGCCGTTCGCCCAATTGTAATGTCCGATATTAGGCGAGCTCAAAGTTATCGGGTGCTAAAGCTTCCACTTATCTTTAGAGGGCGCACAGTTGAGGTACGGGCCCGAGAATCGGTTCTGCCACGTAGACTCTTATGTGGTGTCGTATGTGGCGTAGTTCAACCGCGTCACGACGAATCGTCATAAAGGGTAAACTATCCCTAAGGGTTTTGGAATCTTGCGCAATAGTCGCCTTTGAACGGAAGAAGCCTTACCCCTCTGCGTTCACAGGGTAGCACAAGCTGCGACGAGAGTTACTGGTTGTTTGAGGGGTGCCCTAGTAGGACCTCCGCCGTATTATGTACTATCATCCGAGTGGGCCACCGCGCCCGCCTTAAACACATGCTACCAGTCTGTGCGAGTTGCAGGTGGATCGGCAAGGGGGGGTCCGCTACGCATGTAACCATAGGGGAGGGGTAGCAAAGGTGCGAGGACCACATGCCGGCGTAAAAACGAACTCTCACATGTCAGGCTTTGCATTCTGCGCCCCATAATCATCTAGGCTTTGCCCCACCGTAGCGAAACAGTTAATGCTTTGTCGCTCGTGGCTTAACTTCCTACGCGTTCTAGTTGCTACCCGAAGCGGCGTGTGGAGTCTGCGTTTTTAAGACCGGTGTAGTTCGCATGCAGACGTGAGACCCGAGTTGTGCTTGTGCCTTGTGGTACAACTCCACATATAGTCGCAGTGCGCACGCAAGGCGCACTTCGTCGCTGGCCGGGTCGGGTCGTACGTGCGTTACGAAGCAAAGAGTTGTTTATTAGAACGCTGTGATCCGCACGTAAGACCCGTTACTAGCTGCTCCTCTGTTCTTGCAAGGAGTGGTCAGCACGTATACCAGAACAATATCCTGTTAGAACTGTTAGACATACGACACTTCTCCACCTGGGATACGTAGGAAGGTGCCCCCTCACAACTGATGTCTTCCGCATCGCAAGACAACGTTGCAAAATATTACCCAGCGCCGCGCGTAGTGTGCACGTGCTGGTCCCTCGCAGTACCGACGGTAGATTTAACTCCTAAGTCGTTAAAAAAATTGTGATTCTACGAGGAGAGGCCTACGGGTTCTTTCACCAATTCGATAACTTTCCGACCGTCATATTCCCAATGAATGGAATTGAACTGGACAACATATTACTGACATAGAATCATGCGAGTGATCAAGAGTTCAGGAGTAACAGGTGATAACCTCGGTCGTGGGCCCGCGATTAAACAGACCTTAATGCACTGGATCAACGTTATAACTTCTCTGTAATTGGACATATCTTGTATCCTACCCAGACACTGAGGCACGCGGCTGGCAAGCCCCCAGGACAAGTTCATGCTGCGATGGGCTTTATTCGGACTGCCGGGCCTGACCAAGGCCGATCTATGAGTAGGGCGCGGTAAGAGCGAGCCCCGTTGTCCCAGCCGTGCTGAGGGCCATCTGCCTGGTAAGATACACAAGAAATACGCCTAAATGTTCGAAGAAGGCCTTAGATGATGAAACGTTCTAAACATGCGCGCTAACAGGCCACACTGTATGACATTGCCAGTCTCAATGGGGGATACACTTGAATTAGCCAATGTCTTGCCTCGCCTGACACCACTCGACTTCGCTGCACGGAGCAGCGCTTTTCCTTCAGTAAGCCTATAAGGGGATCTCTCTAGTCACGAACTGCGGTTTAACCCGAGAAGACGTTCCTAGTGAGAGCGCGAGCTCCGAGTACGTTCGGCTAGACTCCGTGAATGGGAGACTTAGGAACTAAGCAGCGCGGGGTTTCGTTTATTGGTACTTGTGCTTGAGCGGACGGGAGCACTTCCTGAGGCACATGCCGGGCGGGGTCGCAGGTAGGCTTAGCAGAGACCAGTAGCTCGTCCGCCTTTCCTCCCGAGCGTGAAAGCGGTAGCCATACACGGGGCTAATAGGGAGCGCACCATGCCCACAGTAGGCCAGCCCGGGCCTCACTCCCCCCATTTTCAGGCTACCTGCAGAGCCATTCGGGTTCCGTGGCGTTTCAGAGAGACCTGACTATCGGTTTGTGTCAGCCGATAGTCTTGCGATGTAACACACGAAGGGATAAAAAAGCGGCCGCAGTGTAGTACCAGAGGTATTTAAACGTTAGACCGAAATTGAGTTTTCGAGCCTATCGAATAATTTCGCTAGGGCTAGGTGATCCATTGGTTGAATTTGCACGCTTGCAAATACAGGCAGTGTGTCATCCAGCTCGCCATAAATGTTCAGGCGACTGACAAGAATTCGAAACTCACACGCACTAGTATTAAGCTTGGGTCACCCTGAGGCACGGGCGATTGTTCCGGGATTGATATCGGACGCCGCTTTTTAAACGATGTACAATTATCATATTTAGCCCATAGGCTCGATCATTCAGCCTTATGCGAACCTCGAAGGCAGGCGGAGTTAGTCGCAGTCCTCAAGGTGCCTCGTAACAAATAATGGGCAGCGCCTGACAAGGGTTGATGTACACTCTCACAACACTCTTAATTCAAAGGAGCCCGCTTATGGATAAAACTAATCCCAGCATTATCTCTCTAGGCTTTTTTTGGGAAGCTTAATCTCACACCTTAATTCCTCGTTCGCGGCCCCGTTGACAAACCTCGACGCCCAGCCTTATAGGAACATTAGATACTCGAAATATGGCAGCGAGAAAGTCCGAAGTGGCTACAAAATGACAATTGTCTACATCGGGAGTATTGCCTCGGTCTTTGGTTGATTGGAACGGGGTTGCATAAAGTACGCGTACATATCGTTGCCGGATAGATAATCTCAGTATTATTCTATTTCATGTGTGTGTAACGGCTACTCCGAAGTCCTGTCAATCAAGACGGGGGCAAGCACAACAGTTCACTCACGGATAGCAGATTATTTACGGCTGAGTTCAATATCGGCAGGAGGAATTAATTTGAACGAAAATACACGCCCCCGGGCTACTGTTTAGCAAGACTATCATACGCAAATACTATGCCATTGCTCGGTGGAATTTCGTTTCCGGACCATGATAAACCGTGTTGTTATTCCAAAAGCGAAAATTTTGATTTAGATCGGACCCTAGCATCCAATGCAGGATCTAAAATAATAATGCGGATAAAATGATAAATTCCCTGCGATCGACGGACGCTAAGTATTGAAGAGCATATAAGAATTGAACGGGTGGCTCCTTCGGTTATGTGGCACTGTTTGCTTGCCTGGCAGAGTATATCAACAGTGATCAACGCGGGAAACTCGGCCGCGGTCCGATGCTGTCTGCTTGGTTACCTCCGGTTTACCATCTGGTTCTACGTCCACTTCGTGCATTTTTGTTGGGTAGGCGCACCTGATGAATTGAGCTGCCCCCGATTTCGCACGTCTGACATCCGAATTTTTTGTCTCAACTTGTCAATTCCGGATCGTCCTAGCTCAGAACCCACGGATCAGCTCGGATAATTTGTTTGGTTGTTTTTACACGTTTATCATACGAGCACAATGGATGGGGCTCATACCCCGGCCCTGCCGGCGTGTGACGAGTGGAGATATTCCTTCTCTAGCTGTTCCTTGTGTAGGAGCACGCTTGGTCCTAACCGTAGCCACTTCGCGATATTGAATGGTTATGTTGCAGCTACAACCAGAACTGGTTGAGGGAGCCCCGAGCCTCGATCGTCTTCACGTCCATACGCATTCGCCACCCTAGCTGGGCTTAGACGTTAAACTGCGACCGGGTAATTACCTAATTCAGTCAAGCGTCCCGTTCCACATTAATATTCCAGATTTGCTTAATGGCCCAGTTAAATCTATCATTACGTAATAGCTGTTACTCCCTAAAGCGCCAAACTTTTGACCCCTTGTTACTTACAAATAAACGGGTTGCGTCTTTCTTTCCACGGACCTCCATGCTCGAATATTGTCTCTTTCCGTATTGTACTGTACAACTCGGTGACAGTTCGTAACTACCTTCAATAGCAACCAACCCCTGTTACACCCGTCGGGCTACTAGTTTCTCTTTACAGCCAGCGTCTGCAGGTAGCTCAGCCCTCAGATTCAGACACAATATGAGGCTCAACTTTGCACTGAGTCCGTGGCCGGAGGGCTCTGGCTAACGGTTACGTTAGCATGAAGTACTGGCTTCATAGAGCCGGGCCTAGTCACACTGGTTGGACACTACCCGGATAGAACGCAAGTTATAAGATATAGGGCAGGTTCTGGGGTGTGCCTGGATATTTTTTGTCCCAGGTTCCTTGCGGCATTGCGTTCAAGGTCTCTCTTACCCAGTCGTCATATACAAGATAATACTATCGACAATCTTGACAACGTCTGTTGAGAGCTCGCGTCATAGGGACGGCAGCGCTAGGGAGGTATGCCCAACCGGGAATGTGGGGTTCTCCACACCGTATGTCCGATGGGCACAACTAGACCGTGTCATTATAAGAAGCTACAACCGAGATGCCGCACCTCTTTAATGTTGATCCTGTATTGTGAACTATCACTCAGCTAATTCGGGGCCATAGGATATAGGCTTCTAAGTCGACTTTTCCGATGAAAGATCACTATCCAGCGTGTCAAGGGTAAATGACATCAGGACGCTGACAATTTGGCTGAATCCTTGGCGACGCGGCAAGAATAAAGGCATAGCACGGGCGTTTGAGGGGACGCTGATTGGCTCGGGTCCAGGTCTGACCTCCACGGAGTCCGGATTCGAACTCGGTCAGTCACACGGGAGGCTTAGGAGCCTTCGAACGGACGATAATATCCCGTTACTCGCCACTGTGCCTGCCTATAATGGAAAAGACAACTTTTTGGAAGGAGCTCGCATTGTCCGCGGCCGAGCGATTTCATGTGGCTTTGTGAAAGAACTCGAGTTGGACATAATGTCTGTATATGTATCGCCGACGCGACATATCCTGTACGTGTTGTCGATGATATGAAATTGACCTACTTACATGATACCAAAAGGAGGCGGCCCCCGGAAGGTTGGCCCGAAACACGAGCTCCTCGCAAATGATGTCATGACGATGGCAGAAGCCGAGCTCACGTCTCATTCGGTCGCAAATCCTTGATTCGTGCCGTAAGGAACTTGGGAGGGTAATAGCAGGTGCCACGTTCTCCTATTGTCGGACGCAGTGCTTATGTAACTGATTCCCTGAAACGTAAGCTATTTCGTCCGGAGATAGAACATTATGTCCACAAGGCAAGATATGTTTCCTATGAGAAGAAAACGACGCTTAATGACTTGGTGTTCTTTCTCCAGGCAAGATACGCGAGCGCCTACGGAACAAATGACGTCGTCTACGACGTCCAGATCCCAGCTTTTCGTTCGCTGCGGCCCCCAACTTTGCCAGTAGTGCCGATTGTGCGTTGCGGCAGGGGTCCTTCATACTAGTGCTTCATGCATTTGCCAGGTACCATGGATCTGCACTACTATCAACCGGGGACCGCGCCATCGCTTATGGGGTAGATGATCGCGCCACATCGTTAAGCGCGGATTATCGGAAGCAATCGTCCCACGTATCCTAGGTATAAGATATAAGTTGCAACCGCCACTAATCCCTCACACTTTCCACGGTGGAATCAGATGTCAAGCACTACAAACTTGGTCCACAGATGGGCAAATATATGTGCTCATCTCATAAAAGGTGTGACCGCTCGTTCCGGAACCTCAACTTTTGCTCGTAGCGGGTATTCCTGACGCGATTCACCAATCGGGGCCAACGGCTGACTTAGACCCGGCAAATAACACAGTCCAGCGTAGCTTGCGCATTTACGGCTTTTTGAGCTGGCGTTCCCGCCAGATTTGCACAATAACATCGATGAACCTGTGCTGCGGCTAAATGATAACGGAATGGACAGTTCGCAGTGCTTCTGATTGAGTTCGTGACCAAAGTCATCCGTAGGCAGGGCACTCTATACCTCACGTATGTAGCCTGTGTATCCTTGCCCAAAGGCGCGCGCAGCTCGAGGAGGCGGGATTGACCGGGACACATGTATTCGCCTCCGAGCGAACGCAGTAAATATTCATACGCAACGAAAGAGCTTCTTACGGTATTAACAATTTCATCCGGGGGTTATCGAGGATCCGCGCCACGGGTCTGCGCGAGACTCACGTGGCGGCGGATGAACCCGGCAACAGTCGGTGCCCCTGCTCAGCAGAGCGTTATTGCGAAGTCCATGAAAACCTTATTCACTAACCCGGAACAGCCCCTTTAATACCAAGTGTGAGTAGAATATTCACGCATGGGCAGGGTGTGGCGAAGAGGCGGCCACACATCGACCGGAAATCTCTGTCCTGCGTTCCATACCCCGACGAATTTCATTTCCCCACGGACTAACTTTCAGGGGGAATAAGCGAGGTACCCCTGGCAGAAAAGGCGCAATGTCTGATGAAATTTCTCGTGATGCTCGTGTGGACGATGGCAAAGCTCATGGAATTAGCTAGCAGCAGTGAAGTAAAAGCCGCTTCGGCGGCGACGGAGTGTATCACATCGGGTCGGATGTGGTCGCGGAGTGACGAGATCGAGGATGTGAGAGATAAGGCCCGTAGATCGGCAGGTGTGTGCCCTACGGCTCACTCGTGGTCAGCCAACTGAACTCGTGACCCTCGTTATGGATCTTAAGCTCTAGTAGGACAGCCCAACTACCACGTCTTAGAAGCCAACGATAAGCCCATAGGGCAATGTTTTTAAAGGAGCCCCCGTTCGTTTCGGCCACAATTAAGAATTGAGTTGATCATGAGGACTATGTGATTTAAAAAGTACACGAAAGGCGCTCAACGGAGTGTACTAGGCATTAATCATCCGAGTACATGCTATAGCTTGTCCAACACTGTGCGATGCGGCGAGCGCGGGTCTGAAACGCACCCGATATTGGTTATGTGTTGGGGTGCCCATCGCGCTGTGTCCCCGGTGGGAAAGTGATGTGCCATCATACAAACGATCGTAAATCTGCTGATAAGGACAGACGGAAGGCTAAAATGGAAAGATTGAGAATTGGGCGACTTTCTTTATTCTCAGAAGCAGGAATAGTCATCCGTAGTGCCGTCAGTTAGTAACAAATAGTTGCACGCTGCGGGGAAATCCCCCCACTCCCGGAGTACCCAGCAAGTTAGGAGGGTGAATCCCGGTTGGGCTCAGCGGACACATGGTGCCGACCAGTATTTCCGCATTTTACGGGACCCATACAACGCGTAACGCTTGACACTGGAAACAATAATAGATGTGTCATGAACTTCTAAAGGTCAGTTCTGTCGTCGATCAGTGCTCTCCGCGATACCCGAATGCGATTCTAAGAGTGCAGCTTTCCCTAATTTGCTACGCGACGTATAAATTGGCCTAGATTGATGGGTGGCGTAATGAAAAGTTATGCTGTTGGTTCGTATCCTGTGTATTTCGGTTCTTCAATTCGCACAGTGATTAACGGCCCATGACTGATGGACGTGGGCGTGTAGTATGGGTATACCCCCCTCAATCACCCCGAGTGCACCTTCGTGCGAGAAGCACATCGCCTACATGATTGGGTACCCCCTAGCTAATGACACCTCCGGCCGGTGCGGCGGAGTCGTAGAATGGGCCAATGATTCATCCCCTTGTTTCGTCAGTTCAGGTCCTGTGATGCACCGTACCTGCCTTATCATTCTCGAAGCCTGTTTAAGTTTCGTTTGCGAATTCTGCGGCCTGCTCTGATCCGGGGAGACCTTCGAACATGTGGTACCGAGACTGAGGTGAGTAGTTCAACCATTTAACTGGAACAGCTAAATGTCGCTTTCGCCCCTTACTTCTTGTATCTTTCGTGGCGTTACCATGAGGAGGGCATTTCTGACTGTCGCTTGGATGGAGGAGTAGAATAACCAGTACCCTATTCTGGCCCTGTCAGCGGCACCGGAAACCGGGCCGCACGTGTGTTCGCAGTCATTTGTTGACGAAGCATGCGTCCAGTGAAACGTATCAGGCATAACTTGCAGGGTAGGAAAATAGGCACTCGTAGCCTAGGTGCAAGGCGTTTTATGGCGCTAGAACTTAGAAACGAAAGATCCGGAGCTGTCCGATAACTAGTTTAAAGAATCACCATCGGGGGATTGCCAAACGATACTACCATTCCTTACACCAGTGAGGGGGCGCTTAGTGGGCATGGAGTTGACGCGGGATGCGCGGGCCAACAGATTCCAAAAAGTGACGCTATTTAAAGAGGTGGGCTATGGAATCGGCATGCTTACAGCTGCTCTTCCTCCTGTAGTAAGTATATGCGACCGATGGAGGTCAAAAGGTCTTGATTCGTCGGTGGCCGATGTTATGCGGTGAGATACCTCCCAACGGCGTTGATAAAGGAGGAGGCTTACCATTACGCCCATGTTCGAGCCCCCAGCCAGACATCGGTACCAAGAAGGGTAGTTGGATACACCCCGTTGCCACCAACGAGTGTCTTGTGATTGGCCGATTTCACCAAGGGCAGATCGACCTGGCGTGGAATTCCCTTATGCAACAGAGTCGACAGTTTGGGGTGTAACAATTGTGCGTAGGCCCAGCCGTCGAGTGGTCCAGGGCTGGTACTATACAGCTAGCCGTACAGAATACTCGAACATTGATCACCATGTGACACATCCGAATAAAGGCTAGCTACTTCCGCACAAGCAGATATATTGCCCCGAGTGGATTCCCTGGGGTCTCTGAGGTATGGGGATAACATACTTTTTAGGACTACAGATAAAAGTCTTACATGGAAACCAGCTACGCGTGGTAAACGGATAACGCCACCGGATGTTAATTGGCCTCGCGAGAAGTTTCATATGCACTTAGGGGTCTTAAGCCGTAGCGCTCGGTTTATGCGGCATTGCGAAGTCTGACGAGGCCATTCGACAAGTCCAACAACTTATCACACTTGGTTTGTGCTGGGCTGTGAAGTCTGCAATTGCAGGTAACTTAGCACTCCGTGATCTACACCCAACCGGCCGTACCTCTTGGGTAAAGAGACTGACCCGGGTGTGATATCTGTGGTAGCCAGGTGTTCATTCAAATAAAAATTGTCCTAGCATTCCGACTCTAATTTTCAAACCGTTGCAAAATTCATCGTTCTAAAAGCAATAGCTTGAGAGGTCCTTTTTTACGGGTCACTGCATCAATATTCTTTCCTCTACCACTCAGTTGATCTTAGTCACGGGTATAAAATATCCCTACTGTACTGTTTGCCGGGCGCGACGGTATATTACCCGTGGGGCGGCGCCATCCGATAGACTCGTATATATCAATCACTTCCGCGGAAATTTTCAGCGTGAGCCAATCGATTGACACAGAGTCCGGCACCGGGGTGACGCATTGAGGTCCATGACACTCGAAACTTGGGCTACGACAGGCGCGGGACGGCCGGCTCGTTACTGAACTGTCAATGGGGATCGCGTGCTTACGGTGCGCGAAGGATTTATTTGAACGAAGGCATCTTCTTAATTGTCTTCATTAAAATGGCCTGGTATATAGTTCGCTAGGCGTTTCTACAGTACGGGTAACTTTCTCTCAAGCAGGAACCGCAAACGTCGATGCTGCAGTGGCATTACGGGGTGATCTTAACATCAAGGAATATCTCCGGTGATTCGACACGTCTCCCCACTTAGCGCGTGACCTCTGACAGGTACCGACTGTAAAAGATCTACGTGTTATATTCAACATTATCCAGCAAGTCGTGGAGTGCGTACATGTTATTGTTTGTGTACTAGAACACTACTATGACGGGTGGAGCCATCATGGTTGACACGGAAGGATAAGGATTGCGCACCCATCAGGAACATGTACTAGACACTGCGCGTTACCACCATAAGCCAGTGCCAAGGATATTGGAACGCTAGGCTTACAGTAAGGCCACAAAGATGAGACTCCTAGATCGTCTGCCGTAATATAACAACTGTACTGCTACAACGACCGATATGCGCATGGCATGGATGCGTAGTTTTCAGGGCATTCGGTATACCGGAGCTGTGCCGAAATCACCGTTTCGGCCACGACGCTCACACGATGGTCTGTGAGACAGGGGGGCTCGCAATCTTCCCCATGAACATGGCGTGATTCAAGAGGAAGCGGTGGCGTGTTCGTAGGCGGGGCGCTTTTCATTTCACGAGAGTAAGACAATAGCAAAGTGACGGCCACGATAAGGGAACTAGAGAAACGAAATTATATTACACGTGCAGTCCGGATGGCCTTGCTGATGTAGCTACATGTCCCACTATTCGCGTTGCTACAGGGGGTCTAGGCGTCTGTCGGAGATTCTATTCAGAACATATGTACAACAATCGGCTAGGACCCACTGGCTAAACGCCTGTGGCAGAGTCTCGCTTTCTGTCAAGTCTACACTAGTAATAATGGTCGCGACTGTGTGCTTAAGCGATCCCGGGCGTATCCGATATGAGAGGTAAAGGTATATTGCTCGTCCCGCCCCCATCTAGGTGCGCGTTTTTCTAGATGATTCAGACCCTCTAAGCACCGACGATACTAGTACAAGAACTCTCCATGGTACCCTCATGCAGCTGTGACGTACCGTGTAAGTATCACTGTATAACTACGATTGGGTAATGCTCGTGTACCTGGTCTATAGTTTACGTATGGCACGTGCCAACCGCGCACCTTTTAGGATTCGAAAAGGCTGCGATCGCGGATGTGCTGGCTCGCTTCACAAAGAATAGGGCGCCGTATTAGCTCGCGGCCGGGTTGCGTTCCCGCTAAGTTACTCTATGCACGCTGGTATAAACTCTAGAGATATAAAATCATGAGCCCAGGGCGGGTTGTAAATATCAGCAACGTAACAATTGTGTGCGTTCCTTGGTATCTGCGCAATCGCGATTCTGGGTTACCGCGAGAGCCCCAGGTTCCGGTTTTACAAGTACAGGGCACAAGAGAGATACAGATGAGTGCACGCAAGATGCATTATCGCCATAACTGAAATGGAAGTTCCTTCCGCTAAAGTTGTACCGCACCGAGAGAAACCATTACTCCAAGGACGCTACAACGGGCTCATCTTCCCGTCAGCGGGTATGCAGTATACTTTACGCGGCCGTCCAACATCCCAGTCTATACCTAACCTTAGACATCGTCTTACAACAGCCGGTATCCCTGTGACCGCCCTACGTACGTTGTGGGTGGAAATCAAAAGACTAAATCAATGGATTTCGATGTCGGAGTGACTTTACTCCATGTGAATAAGACCGGACGAACGCCAACTCGAATTGTACCCGGCTCTCCTTTATCGACAGATGCGTCAATACACAGCGAGATCTATGCCTCGAGCAAATCCGGACACTTGACGTCTGCGATCGTAGAAGGACGGTTTTGTTAATCGTACAAACTACCTCCAATATAGCCCCTCGTACTGTTGACAGCGATACGGTGCTATGGCACTAATCTTCACTGGCGTTTATGGGTATGACAGCTTGGATCATTTACCCCTGACCACTCTTACATACGGAAGACAAATTCCAACCAAGCGCTGGATCTTCGCCAGTGACACCAGAATAGTAACCGGGACCACACATTCCTCAGCTGAAGGAAAAGGTAGCAAGTCACTGGAAGGCCTGGGCACAGGAACGATTGCCAAAATTCTTAAACGAGGCACGAGTAGTGATCCTAAGGCACGGAAACAGCGCGTGGCCGATCTGTCGTGCACCTTGTGGTACTTTTCTGGACGAAGAGCAACGTGTATTAACTCTTAACGCTCTTCTGACATGGCTCACAACGCAACAACGTACCGTTTAAGACTCACGGACACCTAACGCTTCCAACCCCCAAGTATTCACGATGCTCCCTCCGTCGTATACCACTCGTTAACTTGATAGGCTTGCTGAAAACAGTGCTCAGGCCCGGTCGGTCGCCTGTGGTAGCTACCTTACTTTACAGCGAATCTAGCGATAAGTTACGTTCAAACAAGACTGACTTTAACGCCGACGGAGAGTTAACCAATGGACCTTCCAGCGGCGCTGACTGTTAGGTGTGCAACTCCCCATGCCAATTAAAACCAATGGGCCTTACAAAATATGATCTGAATGTCCACATTAGTCTGGGAAGAGTCATATTGTGTCCATAGACCAACTTATCAATAGCCCCAGATACTCCCTTGCGGTATGGAGCTAGCATAGTTGCTTTAGTTATTTGAGAACGTGAACTTCCTATTTGTTGTTTACAATTCTAATCGTCTCCTATGCGAACACATAGAGTCCTGACCTAATTCGTTACGTGGCTTGGAAGGTTGCACAGGTCTATAAAGCCGTAGACAGTGGTCCAGATCGTCTTTGCGTTAATTCGGGACAAATCCCTTCCAGTTAACGTTACCACACGGGAGCTCCTACACGAAAATGCTCTTGGGTGTCCTATAACCTGTACATCTACCCAGGCTCGTCGCACCCCCAGGGAAAGGCCACATCTGCGAGGGCTCTGGCGGACCGTGAGTCTTTATTCTTACTCCAACTCCCCCGAAGTAGGCACTGAAGGCCGTTCGTGTGACAGTTCGACCCAGCAGCTCCGCCGTAAAATGCTACAGAATGAGTGACCCCCCCAGTACGGACACAATAGATTGTCCCTCGTCCACCGATATGCGACGCAATTATGCTCGAGCACAGTTTGCATACGATGCCTACAGATGGGTACCACTTGGGCCGGGGCGACTTTCGCCGGGATTGCCGAATATACGTAATGCCGTCATCGGGCCATTTAAAACGGGGAGACTGGGTGGGGAACAGTGAACCCTGAGGCTAGGCGCAAACATTGAGGGGATCGGACCGTTCAAGCCACGAATCCCACGGCGAAGATATCTAGGAACATGTAATCTGACGGATCTCTTACGAAAATCAGAGAACTGGTGCATCAGTGTCATGCCTTTTTAGACATCATATACATGTCTTAGCACATGGCAATTGGGTGCACAACATCCTTGAGTCGAATTCCCTACCGGTTAAGGGGACGTACAGAACACAACATAGGAAGTCTAGTTCGATGACGATCGAAAGCTCGTGCAACATTTACATTTCACGACGGACTCATTAACGTTGCCCCGGCGTGATATACGTCTGCCACTCGGCCCGTCGTACTAAGGTAATCCACTTGAAAGAGCAAGGTCTCTTGCAGGTGACAACACGTGCCTTTTCCGGTACTCAGGTGAATATATCTCGTCCAATGGGCCTCGCCGCAACTATCACCCTCGTTATCGACGCTAGGACGGTTGGGGATTTCTGCATTGGCTCGTAACTACCATATAAACTTTGCAAACCGGATTCCTTGCGTAGGGACCTTCATTGATATGGAAAGCTGCTCATCGTAGGGTATTTGCGCCGCGCAGTTTAAATGTCTTAATGCTGCGCCTCTAACGCTGTCATAAAACGTTTTTGTGAATAATAGAGACTAACTTCCGCTATTTCAAGTGAGTCTTGCTGGTCAGAAACAG . PASS SVTYPE=INS GT 0/1 0/1 \ No newline at end of file diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf index bce755b9486..9fc4fe9c325 100644 --- a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv.vcf @@ -46,7 +46,7 @@ 1 700000 . C . PASS SVTYPE=DEL;END=700297;SVLEN=-297;CIPOS=-22,18;CIEND=-12,32 GT 0/1 0/1 1 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;RIGHT_SVINSSEQ=ACCACACCCACACAACACACA;LEFT_SVINSSEQ=TGTGGTGTGTGTGGTGTG GT 0/1 0/1 1 850000 . A . PASS SVTYPE=INS;END=850000;SVINSSEQ=ACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACAACCACACCCACACAACACACA GT 0/1 0/1 -1 860000 . A . PASS SVTYPE=INS;END=860000;SVLEN=1000 GT 0/1 0/1 +1 860000 . A . PASS SVTYPE=INVERSION;END=870000 GT 0/1 0/1 1 900000 . G . PASS SVTYPE=INS;END=900000;SVLEN=6027;CIPOS=-16,22 GT 0/1 0/1 1 1000000 . A . PASS SVTYPE=DUP;END=1021100;SVLEN=21100;CIPOS=-500,500;CIEND=-500,500 GT 0/1 0/1 1 1100000 . T . PASS SVTYPE=DUP;END=1100076;SVLEN=76;CIPOS=-10,10;CIEND=-10,10 GT 0/1 0/1 diff --git a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf index c6e2b9176e4..32ad6f4c4f0 100644 --- a/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf +++ b/opencga-storage/opencga-storage-core/src/test/resources/variant-test-sv_2.vcf @@ -46,7 +46,7 @@ 1 700010 . T C . PASS . GT 0/1 0/1 1 800000 . A . PASS SVTYPE=INS;END=800000;SVLEN=6027;CIPOS=-16,22;RIGHT_SVINSSEQ=TGTGGTGTGTGTGGTGTG;LEFT_SVINSSEQ=ACCACACCCACACAACACACA GT 0/1 0/1 1 850000 . A . PASS SVTYPE=INS;END=850000;SVINSSEQ=TGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTGTGTGGTGTG GT 0/1 0/1 -1 860000 . A . PASS SVTYPE=INS;END=860000;SVLEN=1000 GT 0/1 0/1 +1 860000 . A . PASS SVTYPE=INVERSION;END=870000 GT 0/1 0/1 1 900000 . G . PASS SVTYPE=INS;END=900000;SVLEN=6027;CIPOS=-16,22 GT 0/1 0/1 1 1000000 . A . PASS SVTYPE=DUP;END=1021100;SVLEN=21100;CIPOS=-500,500;CIEND=-500,500 GT 0/1 0/1 1 1100000 . T . PASS SVTYPE=DUP;END=1100076;SVLEN=76;CIPOS=-10,10;CIEND=-10,10 GT 0/1 0/1 diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml index ea3dbc32689..740b4bb3101 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-api/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml index 927d616c88c..d8dd4f61336 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.0/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml index a84279ad4de..c24d2e7072b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.2/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml index ea5b63a7b6f..2bf4db839f0 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/opencga-storage-hadoop-compat-hbase2.4/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-compat - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml index 705f7b4104c..0dd166e460a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-compat/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 066630724a2..876fb2c1054 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml @@ -259,10 +259,6 @@ org.apache.parquet parquet-hadoop - - - - org.apache.ant ant @@ -428,6 +424,9 @@ org.apache.logging.log4j:log4j-api + + commons-lang:commons-lang + org.opencb.opencga:opencga-storage-hadoop-compat-api @@ -439,6 +438,9 @@ org.apache.zookeeper:zookeeper + + commons-lang:commons-lang + diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java index e16371af737..509a758f268 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/HBaseMain.java @@ -258,9 +258,9 @@ private void exec(String tool, List args) throws Exception { engine.setConfiguration(storageConfiguration, HadoopVariantStorageEngine.STORAGE_ENGINE_ID, ""); MRExecutor mrExecutor = engine.getMRExecutor(); - int exitError = mrExecutor.run(tool, args.toArray(new String[0])); - if (exitError != 0) { - throw new Exception("Exec failed with exit number '" + exitError + "'"); + MRExecutor.Result result = mrExecutor.run(tool, args.toArray(new String[0])); + if (result.getExitValue() != 0) { + throw new Exception("Exec failed with exit number '" + result.getExitValue() + "'"); } } @@ -318,7 +318,7 @@ private void exportSnapshot(String storageConfigurationPath, String snapshot, St engine.setConfiguration(storageConfiguration, HadoopVariantStorageEngine.STORAGE_ENGINE_ID, ""); MRExecutor mrExecutor = engine.getMRExecutor(); - int exitError = mrExecutor.run("hbase", args.toArray(new String[0])); + int exitError = mrExecutor.run("hbase", args.toArray(new String[0])).getExitValue(); if (exitError != 0) { throw new Exception("ExportSnapshot failed with exit number '" + exitError + "'"); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java index 9bcc56abb56..49fdbf02237 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/utils/AbstractHBaseDriver.java @@ -11,16 +11,17 @@ import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.hadoop.io.HDFSIOConnector; @@ -52,6 +53,8 @@ public abstract class AbstractHBaseDriver extends Configured implements Tool { public static final String COUNTER_GROUP_NAME = "OPENCGA.HBASE"; public static final String COLUMNS_TO_COUNT = "columns_to_count"; + public static final String MR_APPLICATION_ID = "MR_APPLICATION_ID"; + public static final String ERROR_MESSAGE = "ERROR_MESSAGE"; private static final Logger LOGGER = LoggerFactory.getLogger(AbstractHBaseDriver.class); protected String table; @@ -187,11 +190,19 @@ public final int run(String[] args) throws Exception { boolean succeed = executeJob(job); if (!succeed) { LOGGER.error("error with job!"); + if (!"NA".equals(job.getStatus().getFailureInfo())) { + LOGGER.error("Failure info: " + job.getStatus().getFailureInfo()); + printKeyValue(ERROR_MESSAGE, job.getStatus().getFailureInfo()); + } + } LOGGER.info("================================================="); LOGGER.info("Finish job " + getJobName()); LOGGER.info("Total time : " + TimeUtils.durationToString(stopWatch)); LOGGER.info("================================================="); + for (Counter counter : job.getCounters().getGroup(COUNTER_GROUP_NAME)) { + printKeyValue(counter.getName(), counter.getValue()); + } postExecution(job); close(); @@ -228,23 +239,50 @@ protected void parseFixedParams(String[] args) { private boolean executeJob(Job job) throws IOException, InterruptedException, ClassNotFoundException { Thread hook = new Thread(() -> { + LOGGER.info("Shutdown hook called!"); + LOGGER.info("Gracefully stopping the job '" + job.getJobID() + "' ..."); try { - if (!job.isComplete()) { - job.killJob(); + if (job.getJobState() == JobStatus.State.RUNNING) { + if (!job.isComplete()) { + job.killJob(); + } + LOGGER.info("Job '" + job.getJobID() + "' stopped!"); + } else { + LOGGER.info("Job '" + job.getJobID() + "' is not running. Nothing to do."); } // onError(); - } catch (IOException e) { + } catch (Exception e) { LOGGER.error("Error", e); } }); try { Runtime.getRuntime().addShutdownHook(hook); - return job.waitForCompletion(true); - } finally { + job.submit(); + JobID jobID = job.getJobID(); + String applicationId = jobID.appendTo(new StringBuilder(ApplicationId.appIdStrPrefix)).toString(); + printKeyValue(MR_APPLICATION_ID, applicationId); + boolean completion = job.waitForCompletion(true); Runtime.getRuntime().removeShutdownHook(hook); + return completion; + } catch (Exception e) { + // Do not use a finally block to remove shutdownHook, as finally blocks will be executed even if the JVM is killed, + // and this would throw IllegalStateException("Shutdown in progress"); + try { + Runtime.getRuntime().removeShutdownHook(hook); + } catch (Exception e1) { + e.addSuppressed(e1); + } + throw e; } } + protected static void printKeyValue(String key, Object value) { + // Print key value using System.err directly so it can be read by the MRExecutor + // Do not use logger, as it may be redirected to a file or stdout + // Do not use stdout, as it won't be read by the MRExecutor + System.err.println(key + "=" + value); + } + protected boolean isLocal(Path path) { return HDFSIOConnector.isLocal(path.toUri(), getConf()); } @@ -326,6 +364,7 @@ protected void deleteTemporaryFile(Path outdir) throws IOException { FileSystem fileSystem = outdir.getFileSystem(getConf()); fileSystem.delete(outdir, true); fileSystem.cancelDeleteOnExit(outdir); + LOGGER.info("Temporary file deleted!"); } public class MapReduceOutputFile { @@ -502,6 +541,7 @@ protected static void main(String[] args, Class aClass) { System.exit(code); } catch (Exception e) { LoggerFactory.getLogger(aClass).error("Error executing " + aClass, e); + printKeyValue(ERROR_MESSAGE, ExceptionUtils.prettyExceptionMessage(e, false, true)); System.exit(1); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java index 0619d56fd30..35305dd4fdc 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/AbstractVariantsTableDriver.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.filter.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.util.Tool; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -34,7 +33,6 @@ import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.gaps.FillMissingFromArchiveTask; import org.opencb.opencga.storage.hadoop.variant.metadata.HBaseVariantStorageMetadataDBAdaptorFactory; @@ -54,7 +52,7 @@ /** * Created by mh719 on 21/11/2016. */ -public abstract class AbstractVariantsTableDriver extends AbstractHBaseDriver implements Tool { +public abstract class AbstractVariantsTableDriver extends AbstractHBaseDriver { public static final String CONFIG_VARIANT_TABLE_NAME = "opencga.variant.table.name"; public static final String TIMESTAMP = "opencga.variant.table.timestamp"; @@ -266,7 +264,7 @@ protected String getVariantsTable() { } protected String getArchiveTable() { - return getConf().get(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME, StringUtils.EMPTY); + return getConf().get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, StringUtils.EMPTY); } protected HBaseVariantTableNameGenerator getTableNameGenerator() { @@ -343,7 +341,7 @@ public static void setNoneTimestamp(Job job) { public static String[] buildArgs(String archiveTable, String variantsTable, int studyId, Collection fileIds, ObjectMap other) { - other.put(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); + other.put(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); other.put(AbstractVariantsTableDriver.CONFIG_VARIANT_TABLE_NAME, variantsTable); other.put(STUDY_ID, studyId); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java index 462aa6858b0..ada36f36d49 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java @@ -70,6 +70,7 @@ import static org.opencb.opencga.storage.core.metadata.models.TaskMetadata.Type; import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.*; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine.TARGET_VARIANT_TYPE_SET; +import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine.UNSUPPORTED_VARIANT_TYPE_SET; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.*; /** @@ -80,7 +81,7 @@ public class HadoopLocalLoadVariantStoragePipeline extends HadoopVariantStoragePipeline { private final Logger logger = LoggerFactory.getLogger(HadoopLocalLoadVariantStoragePipeline.class); - private static final String OPERATION_NAME = "Load"; + public static final String OPERATION_NAME = "Load"; private int taskId; private HashSet loadedGenotypes; private int sampleIndexVersion; @@ -182,19 +183,33 @@ protected void securePreLoad(StudyMetadata studyMetadata, VariantFileMetadata fi final AtomicInteger ongoingLoads = new AtomicInteger(1); // this boolean resume = options.getBoolean(VariantStorageOptions.RESUME.key(), VariantStorageOptions.RESUME.defaultValue()); - List fileIds = Collections.singletonList(getFileId()); - taskId = getMetadataManager() - .addRunningTask(studyId, OPERATION_NAME, fileIds, resume, Type.LOAD, + VariantStorageMetadataManager metadataManager = getMetadataManager(); + LinkedHashSet sampleIdsFromFileId = metadataManager.getSampleIdsFromFileId(studyId, getFileId()); + VariantStorageEngine.SplitData splitData = VariantStorageEngine.SplitData.from(options); + + taskId = metadataManager + .addRunningTask(studyId, OPERATION_NAME, Collections.singletonList(getFileId()), resume, Type.LOAD, operation -> { if (operation.getName().equals(OPERATION_NAME)) { - if (operation.currentStatus().equals(Status.ERROR)) { + if (operation.currentStatus() == Status.ERROR) { Integer fileId = operation.getFileIds().get(0); - String fileName = getMetadataManager().getFileName(studyMetadata.getId(), fileId); + String fileName = metadataManager.getFileName(studyMetadata.getId(), fileId); logger.warn("Pending load operation for file " + fileName + " (" + fileId + ')'); } else { ongoingLoads.incrementAndGet(); } + if (splitData != VariantStorageEngine.SplitData.CHROMOSOME && splitData != VariantStorageEngine.SplitData.REGION) { + // Do not allow any concurrent load operation on files sharing samples + for (Integer fileId : operation.getFileIds()) { + Set samples = metadataManager.getSampleIdsFromFileId(studyId, fileId); + for (Integer sample : samples) { + if (sampleIdsFromFileId.contains(sample)) { + return false; + } + } + } + } return true; } else { return false; @@ -490,7 +505,7 @@ private void logLoadResults(VariantFileMetadata variantFileMetadata, int duplica if (skipped > 0) { logger.info("There were " + skipped + " skipped variants"); for (VariantType type : VariantType.values()) { - if (!TARGET_VARIANT_TYPE_SET.contains(type)) { + if (UNSUPPORTED_VARIANT_TYPE_SET.contains(type)) { Long countByType = variantFileMetadata.getStats().getTypeCount().get(type.toString()); if (countByType != null && countByType > 0) { logger.info(" * Of which " + countByType + " are " + type.toString() + " variants."); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java new file mode 100644 index 00000000000..6aa472ee735 --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantQueryParser.java @@ -0,0 +1,37 @@ +package org.opencb.opencga.storage.hadoop.variant; + +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; +import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; +import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; + +import java.util.List; + +import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.STUDY; +import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*; + +public class HadoopVariantQueryParser extends VariantQueryParser { + public HadoopVariantQueryParser(CellBaseUtils cellBaseUtils, VariantStorageMetadataManager metadataManager) { + super(cellBaseUtils, metadataManager); + } + + @Override + protected Query preProcessQuery(Query originalQuery, QueryOptions options, VariantQueryProjection projection) { + Query query = super.preProcessQuery(originalQuery, options, projection); + List studyNames = metadataManager.getStudyNames(); + + if (isValidParam(query, STUDY) && studyNames.size() == 1) { + String study = query.getString(STUDY.key()); + if (!isNegated(study)) { + // Check that study exists + metadataManager.getStudyId(study); + query.remove(STUDY.key()); + } + } + + convertGenesToRegionsQuery(query, cellBaseUtils); + return query; + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index ff93b19fe54..e0cbbfb3963 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.StopWatch; -import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; @@ -47,7 +46,6 @@ import org.opencb.opencga.storage.core.metadata.VariantMetadataFactory; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.*; -import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.VariantStoragePipeline; @@ -55,10 +53,12 @@ import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.adaptors.sample.VariantSampleDataManager; import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotationManager; import org.opencb.opencga.storage.core.variant.annotation.annotators.VariantAnnotator; import org.opencb.opencga.storage.core.variant.io.VariantExporter; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.executors.*; import org.opencb.opencga.storage.core.variant.score.VariantScoreFormatDescriptor; import org.opencb.opencga.storage.core.variant.search.SamplesSearchIndexVariantQueryExecutor; @@ -115,25 +115,16 @@ import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.*; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.REGION; -import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.STUDY; -import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.*; import static org.opencb.opencga.storage.hadoop.variant.gaps.FillGapsDriver.*; public class HadoopVariantStorageEngine extends VariantStorageEngine implements Configurable { public static final String STORAGE_ENGINE_ID = "hadoop"; - public static final EnumSet TARGET_VARIANT_TYPE_SET = EnumSet.of( - VariantType.SNV, VariantType.SNP, - VariantType.INDEL, - VariantType.MNV, VariantType.MNP, - VariantType.INSERTION, VariantType.DELETION, - VariantType.CNV, - VariantType.COPY_NUMBER, VariantType.COPY_NUMBER_LOSS, VariantType.COPY_NUMBER_GAIN, - VariantType.DUPLICATION, VariantType.TANDEM_DUPLICATION, VariantType.TRANSLOCATION, - VariantType.BREAKEND, - VariantType.SV, VariantType.SYMBOLIC + public static final EnumSet UNSUPPORTED_VARIANT_TYPE_SET = EnumSet.of( + VariantType.NO_VARIATION, VariantType.MIXED ); + public static final EnumSet TARGET_VARIANT_TYPE_SET = EnumSet.complementOf(UNSUPPORTED_VARIANT_TYPE_SET); public static final String FILE_ID = "fileId"; public static final String STUDY_ID = "studyId"; @@ -698,6 +689,11 @@ public void removeSamples(String study, List samples, URI outdir) throws remove(study, fullyDeletedFiles, samples, outdir); } + @Override + protected TaskMetadata preRemove(String study, List files, List samples) throws StorageEngineException { + return super.preRemove(study, files, samples); + } + @Override public void removeFiles(String study, List files, URI outdir) throws StorageEngineException { remove(study, files, Collections.emptyList(), outdir); @@ -1031,46 +1027,48 @@ private synchronized HBaseManager getHBaseManager(Configuration configuration) { @Override public ParsedVariantQuery parseQuery(Query originalQuery, QueryOptions options) { try { - Query query = preProcessQuery(originalQuery, options); - ParsedVariantQuery parsedVariantQuery = getVariantQueryParser().parseQuery(query, options, true); - parsedVariantQuery.setInputQuery(originalQuery); - return parsedVariantQuery; + return getVariantQueryParser().parseQuery(originalQuery, options); } catch (StorageEngineException e) { throw VariantQueryException.internalException(e).setQuery(originalQuery); } } @Override - public Query preProcessQuery(Query originalQuery, QueryOptions options) { - Query query = super.preProcessQuery(originalQuery, options); - - VariantStorageMetadataManager metadataManager; - CellBaseUtils cellBaseUtils; - try { - metadataManager = getMetadataManager(); - cellBaseUtils = getCellBaseUtils(); - } catch (StorageEngineException e) { - throw VariantQueryException.internalException(e); - } - List studyNames = metadataManager.getStudyNames(); - - if (isValidParam(query, STUDY) && studyNames.size() == 1) { - String study = query.getString(STUDY.key()); - if (!isNegated(study)) { - try { - // Check that study exists - getMetadataManager().getStudyId(study); - } catch (StorageEngineException e) { - throw VariantQueryException.internalException(e); - } - query.remove(STUDY.key()); - } - } - - convertGenesToRegionsQuery(query, cellBaseUtils); - return query; + protected VariantQueryParser getVariantQueryParser() throws StorageEngineException { + return new HadoopVariantQueryParser(getCellBaseUtils(), getMetadataManager()); } +// @Override +// public Query preProcessQuery(Query originalQuery, QueryOptions options) { +// Query query = super.preProcessQuery(originalQuery, options); +// +// VariantStorageMetadataManager metadataManager; +// CellBaseUtils cellBaseUtils; +// try { +// metadataManager = getMetadataManager(); +// cellBaseUtils = getCellBaseUtils(); +// } catch (StorageEngineException e) { +// throw VariantQueryException.internalException(e); +// } +// List studyNames = metadataManager.getStudyNames(); +// +// if (isValidParam(query, STUDY) && studyNames.size() == 1) { +// String study = query.getString(STUDY.key()); +// if (!isNegated(study)) { +// try { +// // Check that study exists +// getMetadataManager().getStudyId(study); +// } catch (StorageEngineException e) { +// throw VariantQueryException.internalException(e); +// } +// query.remove(STUDY.key()); +// } +// } +// +// convertGenesToRegionsQuery(query, cellBaseUtils); +// return query; +// } + @Override protected List initVariantAggregationExecutors() { List executors = new ArrayList<>(3); @@ -1164,8 +1162,8 @@ public VariantStorageMetadataManager getMetadataManager() throws StorageEngineEx } @Override - public DataResult getSampleData(String variant, String study, QueryOptions options) throws StorageEngineException { - return new HBaseVariantSampleDataManager(getDBAdaptor(), getCellBaseUtils()).getSampleData(variant, study, options); + protected VariantSampleDataManager getVariantSampleDataManager() throws StorageEngineException { + return new HBaseVariantSampleDataManager(getDBAdaptor()); } @Override @@ -1177,7 +1175,7 @@ protected List initVariantQueryExecutors() throws StorageE executors.add(new SampleIndexCompoundHeterozygousQueryExecutor( getMetadataManager(), getStorageEngineId(), getOptions(), this, getSampleIndexDBAdaptor(), getDBAdaptor())); executors.add(new BreakendVariantQueryExecutor( - getMetadataManager(), getStorageEngineId(), getOptions(), new SampleIndexVariantQueryExecutor( + getStorageEngineId(), getOptions(), new SampleIndexVariantQueryExecutor( getDBAdaptor(), getSampleIndexDBAdaptor(), getStorageEngineId(), getOptions()), getDBAdaptor())); executors.add(new SamplesSearchIndexVariantQueryExecutor( getDBAdaptor(), getVariantSearchManager(), getStorageEngineId(), dbName, getConfiguration(), getOptions())); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java index ab6dfaf87ef..9c1f22ed349 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageOptions.java @@ -58,6 +58,7 @@ public enum HadoopVariantStorageOptions implements ConfigurationOption { MR_EXECUTOR_SSH_REMOTE_OPENCGA_HOME("storage.hadoop.mr.executor.ssh.remoteOpenCgaHome"), MR_EXECUTOR_SSH_HADOOP_SSH_BIN("storage.hadoop.mr.executor.ssh.hadoop-ssh.bin", "misc/scripts/hadoop-ssh.sh"), MR_EXECUTOR_SSH_HADOOP_SCP_BIN("storage.hadoop.mr.executor.ssh.hadoop-scp.bin", "misc/scripts/hadoop-scp.sh"), + MR_EXECUTOR_SSH_HADOOP_TERMINATION_GRACE_PERIOD_SECONDS("storage.hadoop.mr.executor.ssh.terminationGracePeriodSeconds", 120), ///////////////////////// // Variant table configuration diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java index 40d45a70bee..c672e94fb58 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HBaseColumnIntersectVariantQueryExecutor.java @@ -2,14 +2,14 @@ import com.google.common.collect.Iterators; import org.opencb.biodata.models.variant.Variant; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.query.ParsedQuery; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; import org.slf4j.Logger; @@ -76,23 +76,19 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) { return false; } - @Override - public DataResult count(Query query) { - throw new UnsupportedOperationException("Count not implemented in " + getClass()); - } - /** * Intersect result of column hbase scan and full phoenix query. * Use {@link org.opencb.opencga.storage.core.variant.adaptors.iterators.MultiVariantDBIterator}. * - * @param query Query - * @param options Options + * @param variantQuery Parsed query * @param iterator Shall the resulting object be an iterator instead of a DataResult * @return DataResult or Iterator with the variants that matches the query */ @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator) { + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { logger.info("HBase column intersect"); + Query query = variantQuery.getQuery(); + QueryOptions options = variantQuery.getInputOptions(); // Build the query with only one query filter -> Single HBase column filter // diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHBaseQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHBaseQueryParser.java index 1de83e96382..9c838765146 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHBaseQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHBaseQueryParser.java @@ -577,9 +577,12 @@ public Scan parseQuery(VariantQueryProjection selectElements, Query query, Query //// filters.addFilter(keyOnlyFilter); // scan.addColumn(genomeHelper.getColumnFamily(), VariantPhoenixHelper.VariantColumn.TYPE.bytes()); // } - if (selectElements.getFields().contains(VariantField.TYPE) || !scan.hasFamilies()) { - scan.addColumn(family, VariantColumn.TYPE.bytes()); - } + + // Alleles must always be included. + scan.addColumn(family, VariantColumn.ALLELES.bytes()); + // Because alleles column may be empty, we must still ensure that we get, at least, one result per row. + // Include "type" column, which is never empty. + scan.addColumn(family, VariantColumn.TYPE.bytes()); // if (!columnPrefixes.isEmpty()) { // MultipleColumnPrefixFilter columnPrefixFilter = new MultipleColumnPrefixFilter( diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java index 3d6fe440da1..d3c608c8b42 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/VariantHadoopDBAdaptor.java @@ -33,7 +33,6 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.*; import org.opencb.opencga.core.config.storage.StorageConfiguration; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; @@ -45,6 +44,7 @@ import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotationManager; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; @@ -52,6 +52,7 @@ import org.opencb.opencga.storage.hadoop.auth.HBaseCredentials; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantQueryParser; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.adaptors.iterators.VariantHBaseResultSetIterator; @@ -221,33 +222,35 @@ public void close() throws IOException { } @Override - public VariantQueryResult get(ParsedVariantQuery query, QueryOptions options) { + public VariantQueryResult get(ParsedVariantQuery query) { List variants = new LinkedList<>(); - VariantDBIterator iterator = iterator(query, options); + VariantDBIterator iterator = iterator(query); + QueryOptions options = new QueryOptions(query.getInputOptions()); iterator.forEachRemaining(variants::add); long numTotalResults; - if (options == null) { - numTotalResults = variants.size(); - } else { - if (options.getInt(QueryOptions.LIMIT, -1) >= 0) { - if (options.getBoolean(QueryOptions.COUNT, false)) { - numTotalResults = count(query).first(); - } else { - numTotalResults = -1; - } + if (options.getInt(QueryOptions.LIMIT, -1) >= 0) { + if (options.getBoolean(QueryOptions.COUNT, false)) { + numTotalResults = count(query).first(); } else { - // There are no limit. Do not count. - numTotalResults = variants.size(); + numTotalResults = -1; } + } else { + // There are no limit. Do not count. + numTotalResults = variants.size(); } - VariantQueryResult result = new VariantQueryResult<>(iterator.getTime(TimeUnit.MILLISECONDS), variants.size(), - numTotalResults, null, variants, null, HadoopVariantStorageEngine.STORAGE_ENGINE_ID) + return new VariantQueryResult<>(iterator.getTime(TimeUnit.MILLISECONDS), variants.size(), + numTotalResults, null, variants, HadoopVariantStorageEngine.STORAGE_ENGINE_ID, query) .setFetchTime(iterator.getTimeFetching(TimeUnit.MILLISECONDS)) .setConvertTime(iterator.getTimeConverting(TimeUnit.MILLISECONDS)); - return addSamplesMetadataIfRequested(result, query.getQuery(), options, getMetadataManager()); + } + + @Override + @Deprecated + public VariantQueryResult get(Query query, QueryOptions options) { + return get(new HadoopVariantQueryParser(null, getMetadataManager()).parseQuery(query, options)); } @Override @@ -334,7 +337,8 @@ public DataResult distinct(Query query, String field) { } @Override - public VariantDBIterator iterator(ParsedVariantQuery variantQuery, QueryOptions options) { + public VariantDBIterator iterator(ParsedVariantQuery variantQuery) { + QueryOptions options = variantQuery.getInputOptions(); if (options == null) { options = new QueryOptions(); } else { @@ -385,7 +389,7 @@ public VariantDBIterator iterator(ParsedVariantQuery variantQuery, QueryOptions private VariantHBaseResultSetIterator phoenixIterator(ParsedVariantQuery variantQuery, QueryOptions options, HBaseVariantConverterConfiguration converterConfiguration) { VariantStorageMetadataManager metadataManager = getMetadataManager(); - new VariantQueryParser(null, metadataManager).optimize(variantQuery); + new HadoopVariantQueryParser(null, metadataManager).optimize(variantQuery); logger.debug("Table name = " + variantTable); logger.info("Query : " + VariantQueryUtils.printQuery(variantQuery.getQuery())); @@ -679,7 +683,7 @@ private class VariantQueryIteratorCustomSplit extends MultiVariantDBIterator.Var VariantQueryIteratorCustomSplit(Iterator variants, Query query, int batchSize, QueryOptions options) { super(variants, query, batchSize); - parser = new VariantQueryParser(null, getMetadataManager()); + parser = new HadoopVariantQueryParser(null, getMetadataManager()); variantQuery = parser.parseQuery(query, options); cts = sizeOrOne(variantQuery.getConsequenceTypes()); bts = sizeOrOne(variantQuery.getBiotypes()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java index eedeeb3c9ed..9e06ebc4b51 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactory.java @@ -19,6 +19,8 @@ import htsjdk.variant.variantcontext.Allele; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.query.QueryConstants; @@ -28,6 +30,7 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.VariantBuilder; import org.opencb.biodata.models.variant.avro.*; +import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import java.sql.ResultSet; import java.sql.SQLException; @@ -43,12 +46,15 @@ */ public class VariantPhoenixKeyFactory { + public static final Integer UINT_SIZE = PUnsignedInt.INSTANCE.getByteSize(); protected static final String SV_ALTERNATE_SEPARATOR = "|"; protected static final String SV_ALTERNATE_SEPARATOR_SPLIT = "\\" + SV_ALTERNATE_SEPARATOR; public static final Comparator HBASE_KEY_CHROMOSOME_COMPARATOR = (c1, c2) -> Bytes.compareTo( VariantPhoenixKeyFactory.generateSimpleVariantRowKey(c1, 1, "N", "N"), VariantPhoenixKeyFactory.generateSimpleVariantRowKey(c2, 1, "N", "N")); + public static final String HASH_PREFIX = "#"; + public static final byte[] HASH_PREFIX_BYTES = Bytes.toBytes(HASH_PREFIX); public static byte[] generateVariantRowKey(String chrom, int position) { return generateSimpleVariantRowKey(chrom, position, "", ""); @@ -59,6 +65,26 @@ public static byte[] generateVariantRowKey(Variant var) { var.getSv()); } + public static byte[] generateVariantRowKey(ResultSet resultSet) { + String chromosome = null; + Integer start = null; + String reference = null; + String alternate = null; + try { + chromosome = resultSet.getString(VariantPhoenixSchema.VariantColumn.CHROMOSOME.column()); + start = resultSet.getInt(VariantPhoenixSchema.VariantColumn.POSITION.column()); + reference = resultSet.getString(VariantPhoenixSchema.VariantColumn.REFERENCE.column()); + alternate = resultSet.getString(VariantPhoenixSchema.VariantColumn.ALTERNATE.column()); + + return generateVariantRowKey(chromosome, start, null, reference, alternate, null); + } catch (RuntimeException | SQLException e) { + throw new IllegalStateException("Fail to generate row key from Phoenix result set: " + chromosome + + ':' + start + + ':' + (reference == null ? "-" : reference) + + ':' + (alternate == null ? "-" : alternate), e); + } + } + public static byte[] generateVariantRowKey(VariantAnnotation variantAnnotation) { byte[] bytesRowKey = null; if (variantAnnotation.getAdditionalAttributes() != null) { @@ -97,6 +123,12 @@ public static byte[] generateSimpleVariantRowKey(String chrom, int position, Str return generateVariantRowKey(chrom, position, null, ref, alt, null); } + public static boolean mightHashAlleles(Variant variant) { + int size = getSize(variant); + return size > HConstants.MAX_ROW_LENGTH; + } + + /** * Generates a Row key based on Chromosome, start, end (optional), ref and alt.
*

    @@ -114,16 +146,16 @@ public static byte[] generateSimpleVariantRowKey(String chrom, int position, Str */ public static byte[] generateVariantRowKey(String chrom, int start, Integer end, String ref, String alt, StructuralVariation sv) { chrom = Region.normalizeChromosome(chrom); - int size = PVarchar.INSTANCE.estimateByteSizeFromLength(chrom.length()) - + QueryConstants.SEPARATOR_BYTE_ARRAY.length - + PUnsignedInt.INSTANCE.getByteSize() - + PVarchar.INSTANCE.estimateByteSizeFromLength(ref.length()); alt = buildSymbolicAlternate(ref, alt, end, sv); - if (!alt.isEmpty()) { - size += QueryConstants.SEPARATOR_BYTE_ARRAY.length - + PVarchar.INSTANCE.estimateByteSizeFromLength(alt.length()); + int size = getSize(chrom, ref, alt); + + if (size > HConstants.MAX_ROW_LENGTH) { + // This is a problem. The row key is too long. + // Use hashCode for reference/alternate/SV fields + ref = hashAllele(ref); + alt = hashAllele(alt); + size = getSize(chrom, ref, alt); } - byte[] rk = new byte[size]; int offset = 0; @@ -143,6 +175,31 @@ public static byte[] generateVariantRowKey(String chrom, int start, Integer end, return rk; } + private static int getSize(Variant variant) { + String symbolicAlternate = buildSymbolicAlternate(variant); + return getSize(variant.getChromosome(), variant.getReference(), symbolicAlternate); + } + + private static int getSize(String chrom, String ref, String alt) { + int size = PVarchar.INSTANCE.estimateByteSizeFromLength(chrom.length()) + + QueryConstants.SEPARATOR_BYTE_ARRAY.length + + PUnsignedInt.INSTANCE.getByteSize() + + PVarchar.INSTANCE.estimateByteSizeFromLength(ref.length()); + if (!alt.isEmpty()) { + size += QueryConstants.SEPARATOR_BYTE_ARRAY.length + + PVarchar.INSTANCE.estimateByteSizeFromLength(alt.length()); + } + return size; + } + + public static String hashAllele(String ref) { + return HASH_PREFIX + Integer.toString(ref.hashCode()); + } + + public static String buildAlleles(Variant v) { + return v.getReference() + SV_ALTERNATE_SEPARATOR + buildSymbolicAlternate(v); + } + public static String buildSymbolicAlternate(Variant v) { return buildSymbolicAlternate(v.getReference(), v.getAlternate(), v.getEnd(), v.getSv()); } @@ -215,6 +272,26 @@ public static String extractChrFromVariantRowKey(byte[] variantRowKey, int offse return (String) PVarchar.INSTANCE.toObject(variantRowKey, offset, chrPosSeparator, PVarchar.INSTANCE); } + public static Variant extractVariantFromResult(Result result) { + byte[] variantRowKey = result.getRow(); + + int chrPosSeparator = ArrayUtils.indexOf(variantRowKey, (byte) 0); + int referenceOffset = chrPosSeparator + 1 + UINT_SIZE; + if (variantRowKey.length > (referenceOffset + HASH_PREFIX_BYTES.length) + && Bytes.equals(variantRowKey, referenceOffset, HASH_PREFIX_BYTES.length, + HASH_PREFIX_BYTES, 0, HASH_PREFIX_BYTES.length)) { + // The reference and alternate are hashed. + // The type and alleles are stored in the result + byte[] type = result.getValue(GenomeHelper.COLUMN_FAMILY_BYTES, + VariantPhoenixSchema.VariantColumn.TYPE.bytes()); + byte[] alleles = result.getValue(GenomeHelper.COLUMN_FAMILY_BYTES, + VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + return extractVariantFromVariantRowKey(variantRowKey, type, alleles); + } else { + return extractVariantFromVariantRowKey(variantRowKey, null, null); + } + } + public static Variant extractVariantFromResultSet(ResultSet resultSet) { String chromosome = null; Integer start = null; @@ -226,9 +303,10 @@ public static Variant extractVariantFromResultSet(ResultSet resultSet) { reference = resultSet.getString(VariantPhoenixSchema.VariantColumn.REFERENCE.column()); alternate = resultSet.getString(VariantPhoenixSchema.VariantColumn.ALTERNATE.column()); + String alleles = resultSet.getString(VariantPhoenixSchema.VariantColumn.ALLELES.column()); String type = resultSet.getString(VariantPhoenixSchema.VariantColumn.TYPE.column()); - return buildVariant(chromosome, start, reference, alternate, type); + return buildVariant(chromosome, start, reference, alternate, type, alleles); } catch (RuntimeException | SQLException e) { throw new IllegalStateException("Fail to parse variant: " + chromosome + ':' + start @@ -237,13 +315,12 @@ public static Variant extractVariantFromResultSet(ResultSet resultSet) { } } - public static Variant extractVariantFromVariantRowKey(byte[] variantRowKey) { + public static Variant extractVariantFromVariantRowKey(byte[] variantRowKey, byte[] type, byte[] alleles) { int chrPosSeparator = ArrayUtils.indexOf(variantRowKey, (byte) 0); String chromosome = (String) PVarchar.INSTANCE.toObject(variantRowKey, 0, chrPosSeparator, PVarchar.INSTANCE); - Integer intSize = PUnsignedInt.INSTANCE.getByteSize(); - int position = (Integer) PUnsignedInt.INSTANCE.toObject(variantRowKey, chrPosSeparator + 1, intSize, PUnsignedInt.INSTANCE); - int referenceOffset = chrPosSeparator + 1 + intSize; + int position = (Integer) PUnsignedInt.INSTANCE.toObject(variantRowKey, chrPosSeparator + 1, UINT_SIZE, PUnsignedInt.INSTANCE); + int referenceOffset = chrPosSeparator + 1 + UINT_SIZE; int refAltSeparator = ArrayUtils.indexOf(variantRowKey, (byte) 0, referenceOffset); String reference; String alternate; @@ -257,8 +334,16 @@ public static Variant extractVariantFromVariantRowKey(byte[] variantRowKey) { alternate = (String) PVarchar.INSTANCE.toObject(variantRowKey, refAltSeparator + 1, variantRowKey.length - (refAltSeparator + 1), PVarchar.INSTANCE); } + String typeStr = null; + String alleleStr = null; + if (type != null) { + typeStr = (String) PVarchar.INSTANCE.toObject(type); + } + if (alleles != null) { + alleleStr = (String) PVarchar.INSTANCE.toObject(alleles); + } try { - return buildVariant(chromosome, position, reference, alternate, null); + return buildVariant(chromosome, position, reference, alternate, typeStr, alleleStr); } catch (RuntimeException e) { throw new IllegalStateException("Fail to parse variant: " + chromosome + ':' + position @@ -268,7 +353,17 @@ public static Variant extractVariantFromVariantRowKey(byte[] variantRowKey) { } } - public static Variant buildVariant(String chromosome, int start, String reference, String alternate, String type) { + public static Variant buildVariant(String chromosome, int start, String reference, String alternate, String type, String alleles) { + if ((reference != null && reference.startsWith(HASH_PREFIX)) || (alternate != null && alternate.startsWith(HASH_PREFIX))) { + if (StringUtils.isNotEmpty(alleles)) { + int i1 = alleles.indexOf(SV_ALTERNATE_SEPARATOR); + reference = alleles.substring(0, i1); + alternate = alleles.substring(i1 + SV_ALTERNATE_SEPARATOR.length()); + } else { + throw new IllegalStateException("Reference and alternate are hashed, but alleles is empty!" + + " '" + chromosome + "' '" + start + "' '" + reference + "' '" + alternate + "'"); + } + } if (alternate != null && alternate.length() > 5 && alternate.contains(SV_ALTERNATE_SEPARATOR)) { Integer end = null; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchema.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchema.java index 1d408da09b4..eb627dcaeda 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchema.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchema.java @@ -105,6 +105,7 @@ public enum VariantColumn implements Column { CI_END_R("CI_END_R", PUnsignedInt.INSTANCE), TYPE("TYPE", PVarchar.INSTANCE), + ALLELES("ALLELES", PVarchar.INSTANCE), ANNOTATION_ID(ANNOTATION_PREFIX + "ID", PInteger.INSTANCE), diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java index ae8ce82ee84..2cb6bbe5403 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java @@ -79,7 +79,7 @@ public void registerStudyColumns(int studyId) throws StorageEngineException { registerNewFiles(studyId, new ArrayList<>(metadataManager.getIndexedFiles(studyId))); List cohortIds = new LinkedList<>(); - for (CohortMetadata cohort : metadataManager.getCalculatedCohorts(studyId)) { + for (CohortMetadata cohort : metadataManager.getCalculatedOrPartialCohorts(studyId)) { cohortIds.add(cohort.getId()); } registerNewCohorts(studyId, cohortIds); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantSqlQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantSqlQueryParser.java index bb9737144d2..6a7d9fe5518 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantSqlQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantSqlQueryParser.java @@ -44,6 +44,7 @@ import org.opencb.opencga.storage.core.variant.query.*; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjectionParser; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantQueryParser; import org.opencb.opencga.storage.hadoop.variant.converters.HBaseToVariantConverter; import org.opencb.opencga.storage.hadoop.variant.converters.annotation.VariantAnnotationToPhoenixConverter; import org.opencb.opencga.storage.hadoop.variant.converters.study.HBaseToStudyEntryConverter; @@ -116,7 +117,7 @@ public VariantSqlQueryParser(String variantTable, @Deprecated public String parse(Query query, QueryOptions options) { - VariantQueryParser parser = new VariantQueryParser(null, metadataManager); + VariantQueryParser parser = new HadoopVariantQueryParser(null, metadataManager); ParsedVariantQuery variantQuery = parser.parseQuery(query, options, true); return parse(variantQuery, options); } @@ -124,8 +125,6 @@ public String parse(Query query, QueryOptions options) { public String parse(ParsedVariantQuery variantQuery, QueryOptions options) { Query query = variantQuery.getQuery(); - StringBuilder sb = new StringBuilder("SELECT "); - try { Set dynamicColumns = new HashSet<>(); @@ -144,21 +143,22 @@ public String parse(ParsedVariantQuery variantQuery, QueryOptions options) { hints.add(HintNode.Hint.valueOf(hint)); } } + + StringBuilder sb = new StringBuilder("SELECT "); if (!hints.isEmpty()) { sb.append("/*+ ").append(hints.stream().map(Object::toString).collect(Collectors.joining(","))).append(" */ "); } - appendProjectedColumns(sb, variantQuery.getProjection(), options); appendFromStatement(sb, dynamicColumns); appendWhereStatement(sb, regionFilters, filters); appendOrderby(options, sb); appendLimitSkip(options, sb); + return sb.toString(); } catch (VariantQueryException e) { e.setQuery(query); throw e; } - return sb.toString(); } private void appendOrderby(QueryOptions options, StringBuilder sb) { @@ -216,13 +216,13 @@ protected StringBuilder appendProjectedColumns(StringBuilder sb, VariantQueryPro return sb.append(" COUNT(*) "); } else { Set returnedFields = projection.getFields(); - Collection studyIds = projection.getStudyIds(); sb.append(VariantColumn.CHROMOSOME).append(',') .append(VariantColumn.POSITION).append(',') .append(VariantColumn.REFERENCE).append(',') .append(VariantColumn.ALTERNATE).append(',') - .append(VariantColumn.TYPE); + .append(VariantColumn.TYPE).append(',') + .append(VariantColumn.ALLELES); for (VariantQueryProjection.StudyVariantQueryProjection study : projection.getStudies().values()) { int studyId = study.getId(); @@ -600,10 +600,17 @@ private String getVariantFilter(List variants) { Iterator iterator = variants.iterator(); while (iterator.hasNext()) { Variant variant = iterator.next(); + + String reference = variant.getReference(); + String alternate = VariantPhoenixKeyFactory.buildSymbolicAlternate(variant); + if (VariantPhoenixKeyFactory.mightHashAlleles(variant)) { + reference = VariantPhoenixKeyFactory.hashAllele(reference); + alternate = VariantPhoenixKeyFactory.hashAllele(alternate); + } sb.append("('").append(checkStringValue(variant.getChromosome())).append("', ") .append(variant.getStart()).append(", ") - .append('\'').append(checkStringValue(variant.getReference())).append("', ") - .append('\'').append(checkStringValue(VariantPhoenixKeyFactory.buildSymbolicAlternate(variant))).append("') "); + .append('\'').append(checkStringValue(reference)).append("', ") + .append('\'').append(checkStringValue(alternate)).append("') "); if (iterator.hasNext()) { sb.append(','); } @@ -710,9 +717,9 @@ protected List getOtherFilters(ParsedVariantQuery variantQuery, QueryOpt protected void addVariantFilters(ParsedVariantQuery variantQuery, QueryOptions options, List filters) { Query query = variantQuery.getQuery(); - addQueryFilter(query, REFERENCE, VariantColumn.REFERENCE, filters); + addQueryFilter(query, REFERENCE, VariantColumn.REFERENCE, filters, VariantSqlQueryParser::hashAlleles); - addQueryFilter(query, ALTERNATE, VariantColumn.ALTERNATE, filters); + addQueryFilter(query, ALTERNATE, VariantColumn.ALTERNATE, filters, VariantSqlQueryParser::hashAlleles); addQueryFilter(query, TYPE, VariantColumn.TYPE, filters); @@ -966,36 +973,39 @@ protected void addVariantFilters(ParsedVariantQuery variantQuery, QueryOptions o } boolean multiFileSample = VariantStorageEngine.SplitData.MULTI.equals(sampleMetadata.getSplitData()); List sampleFiles = new ArrayList<>(); + // First file does not have the fileID in the column name + Integer firstSampleFile = sampleMetadata.getFiles().get(0); + if (multiFileSample) { if (fileIds.isEmpty()) { - sampleFiles.add(null); // First file does not have the fileID in the column name List fileIdsFromSampleId = sampleMetadata.getFiles(); - sampleFiles.addAll(fileIdsFromSampleId.subList(1, fileIdsFromSampleId.size())); + sampleFiles.addAll(fileIdsFromSampleId); } else { for (Pair fileIdPair : fileIds) { - if (fileIdPair.getKey().equals(studyId)) { - Integer fileId = fileIdPair.getValue(); - int idx = sampleMetadata.getFiles().indexOf(fileId); - if (idx == 0) { - sampleFiles.add(null); // First file does not have the fileID in the column name - } else if (idx > 0) { - sampleFiles.add(fileId); // First file does not have the fileID in the column name - } + if (fileIdPair.getKey().equals(studyId) + && sampleMetadata.getFiles().contains(fileIdPair.getValue())) { + sampleFiles.add(fileIdPair.getValue()); } } } } else { - sampleFiles.add(null); // First file does not have the fileID in the column name + // Non multi file sample + sampleFiles.add(firstSampleFile); } for (Integer sampleFile : sampleFiles) { + if (!metadataManager.isFileIndexed(studyId, sampleFile)) { + // Skip non indexed files + continue; + } List sampleFileGtFilters = new ArrayList<>(genotypes.size()); for (String genotype : genotypes) { if (negated) { genotype = removeNegation(genotype); } String key; - if (sampleFile == null) { + if (Objects.equals(sampleFile, firstSampleFile)) { + // Special scenario for the first file. Column name does not contain the fileId key = buildSampleColumnKey(studyId, sampleId, new StringBuilder()).toString(); } else { key = buildSampleColumnKey(studyId, sampleId, sampleFile, new StringBuilder()).toString(); @@ -1079,6 +1089,16 @@ protected void addVariantFilters(ParsedVariantQuery variantQuery, QueryOptions o } } + private static List hashAlleles(String r) { + String[] split = r.split(","); + List list = new ArrayList<>(split.length * 2); + for (String s : split) { + list.add(s); + list.add(VariantPhoenixKeyFactory.hashAllele(s)); + } + return list; + } + private void addFileFilterFieldFilter(QueryOperation filtersOperation, List filterValues, StringBuilder sb, Pair fileIdPair) { sb.append(" ( "); @@ -1230,27 +1250,24 @@ private void addSampleDataFilter(ParsedVariantQuery query, List filters, SampleMetadata sampleMetadata = sampleDataFilter.getKey(); boolean multiFileSample = VariantStorageEngine.SplitData.MULTI.equals(sampleMetadata.getSplitData()); + // First file does not have the fileID in the column name + Integer firstSampleFile = sampleMetadata.getFiles().get(0); List sampleFiles = new ArrayList<>(); if (multiFileSample) { if (fileIds.isEmpty()) { - sampleFiles.add(null); // First file does not have the fileID in the column name List fileIdsFromSampleId = sampleMetadata.getFiles(); - sampleFiles.addAll(fileIdsFromSampleId.subList(1, fileIdsFromSampleId.size())); + sampleFiles.addAll(fileIdsFromSampleId); } else { for (Pair fileIdPair : fileIds) { - if (fileIdPair.getKey().equals(sampleMetadata.getStudyId())) { - Integer fileId = fileIdPair.getValue(); - int idx = sampleMetadata.getFiles().indexOf(fileId); - if (idx == 0) { - sampleFiles.add(null); // First file does not have the fileID in the column name - } else if (idx > 0) { - sampleFiles.add(fileId); // First file does not have the fileID in the column name - } + if (fileIdPair.getKey().equals(sampleMetadata.getStudyId()) + && sampleMetadata.getFiles().contains(fileIdPair.getValue())) { + sampleFiles.add(fileIdPair.getValue()); } } } } else { - sampleFiles.add(null); // First file does not have the fileID in the column name + // Non multi file sample + sampleFiles.add(firstSampleFile); } for (Integer sampleFile : sampleFiles) { List sampleFileFilters = new LinkedList<>(); @@ -1270,10 +1287,11 @@ private void addSampleDataFilter(ParsedVariantQuery query, List filters, sb.append("TO_NUMBER("); } sb.append('"'); - if (sampleFile != null) { - buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb); - } else { + if (Objects.equals(sampleFile, firstSampleFile)) { + // Special scenario for the first file. Column name does not contain the fileId buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sb); + } else { + buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb); } sb.append('"'); @@ -1287,10 +1305,11 @@ private void addSampleDataFilter(ParsedVariantQuery query, List filters, if (op.startsWith(">>") || op.startsWith("<<")) { sb.append(" OR \""); - if (sampleFile != null) { - buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb); - } else { + if (Objects.equals(sampleFile, firstSampleFile)) { + // Special scenario for the first file. Column name does not contain the fileId buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sb); + } else { + buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb); } sb.append('"'); @@ -1307,8 +1326,8 @@ private void addSampleDataFilter(ParsedVariantQuery query, List filters, if (multiFileSample) { // The first file is null. Get the actual fileId Integer actualFileId; - if (sampleFile == null) { - actualFileId = sampleMetadata.getFiles().get(0); + if (Objects.equals(sampleFile, firstSampleFile)) { + actualFileId = firstSampleFile; } else { actualFileId = sampleFile; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/sample/HBaseVariantSampleDataManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/sample/HBaseVariantSampleDataManager.java index 588833cfb6e..31850f1f486 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/sample/HBaseVariantSampleDataManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/sample/HBaseVariantSampleDataManager.java @@ -15,7 +15,6 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; @@ -26,8 +25,8 @@ import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper; -import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.opencb.opencga.storage.hadoop.variant.converters.HBaseVariantConverterConfiguration; import org.opencb.opencga.storage.hadoop.variant.converters.VariantRow; import org.opencb.opencga.storage.hadoop.variant.converters.annotation.HBaseToVariantAnnotationConverter; @@ -45,31 +44,20 @@ public class HBaseVariantSampleDataManager extends VariantSampleDataManager { private final VariantHadoopDBAdaptor dbAdaptor; private final VariantStorageMetadataManager metadataManager; - private final CellBaseUtils cellBaseUtils; - - public HBaseVariantSampleDataManager(VariantHadoopDBAdaptor dbAdaptor, CellBaseUtils cellBaseUtils) { + public HBaseVariantSampleDataManager(VariantHadoopDBAdaptor dbAdaptor) { super(dbAdaptor); this.dbAdaptor = dbAdaptor; metadataManager = dbAdaptor.getMetadataManager(); - this.cellBaseUtils = cellBaseUtils; } @Override - protected DataResult getSampleData(String variantStr, String study, QueryOptions options, + protected DataResult getSampleData(Variant inputVariant, String study, QueryOptions options, List includeSamples, Set genotypes, int sampleLimit) { StopWatch stopWatch = StopWatch.createStarted(); Set includeFields = VariantField.getIncludeFields(options); - final Variant variant; - if (VariantQueryUtils.isVariantId(variantStr)) { - variant = new Variant(variantStr); - } else { - variant = cellBaseUtils.getVariant(variantStr); - } - variant.setId(variant.toString()); - int studyId = metadataManager.getStudyId(study); boolean includeNoneSamples = VariantQueryUtils.isNoneOrEmpty(includeSamples); @@ -90,6 +78,7 @@ protected DataResult getSampleData(String variantStr, String study, Que int skip = Math.max(0, options.getInt(QueryOptions.SKIP, 0)); int limit = Math.max(0, options.getInt(QueryOptions.LIMIT, 10)); + byte[] rowKey = VariantPhoenixKeyFactory.generateVariantRowKey(inputVariant); try { List samples = new ArrayList<>(limit); @@ -97,7 +86,7 @@ protected DataResult getSampleData(String variantStr, String study, Que dbAdaptor.getHBaseManager().act(dbAdaptor.getVariantTable(), table -> { // Create one GET for samples - Get get = new Get(VariantPhoenixKeyFactory.generateVariantRowKey(variant)); + Get get = new Get(rowKey); LinkedList filters = new LinkedList<>(); filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, @@ -139,7 +128,7 @@ protected DataResult getSampleData(String variantStr, String study, Que samples.add(sampleColumn.getSampleId()); sampleDataMap.add(sampleColumn); } - }).walk(); + }).walk(inputVariant); } }); @@ -148,8 +137,10 @@ protected DataResult getSampleData(String variantStr, String study, Que Set fileIdsFromSampleIds = metadataManager.getFileIdsFromSampleIds(studyId, samples); HBaseToVariantStatsConverter statsConverter = new HBaseToVariantStatsConverter(); List stats = new LinkedList<>(); + Variant variantResult = new Variant(inputVariant.toString()); dbAdaptor.getHBaseManager().act(dbAdaptor.getVariantTable(), table -> { - Get get = new Get(VariantPhoenixKeyFactory.generateVariantRowKey(variant)); + Get get = new Get(rowKey); + // Add file columns for (Integer fileId : fileIdsFromSampleIds) { get.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.buildFileColumnKey(studyId, fileId)); @@ -172,7 +163,7 @@ protected DataResult getSampleData(String variantStr, String study, Que Result result = table.get(get); if (result == null || result.isEmpty()) { - throw VariantQueryException.variantNotFound(variantStr); + throw VariantQueryException.variantNotFound(variantResult.toString()); } // Walk row VariantRow.walker(result) @@ -186,9 +177,9 @@ protected DataResult getSampleData(String variantStr, String study, Que }) .onVariantAnnotation(column -> { ImmutableBytesWritable b = column.toBytesWritable(); - variant.setAnnotation(new HBaseToVariantAnnotationConverter().convert(b.get(), b.getOffset(), b.getLength())); + variantResult.setAnnotation(new HBaseToVariantAnnotationConverter().convert(b)); }) - .walk(); + .walk(variantResult); }); // Convert to VariantSampleData @@ -204,13 +195,14 @@ protected DataResult getSampleData(String variantStr, String study, Que new ArrayList<>(fileIdsFromSampleIds))) .build()); - StudyEntry studyEntry = converter.convert(sampleDataMap, filesMap, variant, studyId); + variantResult.setId(inputVariant.toString()); + StudyEntry studyEntry = converter.convert(sampleDataMap, filesMap, variantResult, studyId); - variant.addStudyEntry(studyEntry); + variantResult.addStudyEntry(studyEntry); studyEntry.setStats(stats); -// String msg = "Queries : " + queries + " , readSamples : " + readSamples; + return new DataResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), Collections.emptyList(), 1, - Collections.singletonList(variant), 1); + Collections.singletonList(variantResult), 1); } catch (IOException e) { throw VariantQueryException.internalException(e); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/julie/JulieToolDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/julie/JulieToolDriver.java index 2321e25f6e1..5192e7356f9 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/julie/JulieToolDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/analysis/julie/JulieToolDriver.java @@ -68,6 +68,7 @@ protected void parseAndValidateParameters() throws IOException { for (Integer studyId : metadataManager.getStudies().values()) { List studyCohorts = new LinkedList<>(); cohorts.put(studyId, studyCohorts); + // Only READY cohort stats are used for JulieTool for (CohortMetadata c : metadataManager.getCalculatedCohorts(studyId)) { studyCohorts.add(c.getId()); } @@ -78,6 +79,8 @@ protected void parseAndValidateParameters() throws IOException { @Override protected Job setupJob(Job job, String archiveTable, String variantTable) throws IOException { Scan scan = new Scan(); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.FULL_ANNOTATION.bytes()); for (Map.Entry> entry : cohorts.entrySet()) { Integer studyId = entry.getKey(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/mr/VariantAnnotationRebuilderDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/mr/VariantAnnotationRebuilderDriver.java index 5690b374981..eb4bdc5b4eb 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/mr/VariantAnnotationRebuilderDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/mr/VariantAnnotationRebuilderDriver.java @@ -68,6 +68,8 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws logger.info("Regenerate annotations for region " + region); VariantHBaseQueryParser.addRegionFilter(scan, new Region(region)); } + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.FULL_ANNOTATION.bytes()); VariantMapReduceUtil.configureMapReduceScan(scan, getConf()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/pending/AnnotationPendingVariantsDescriptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/pending/AnnotationPendingVariantsDescriptor.java index c5ba3d73e83..c606a42043f 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/pending/AnnotationPendingVariantsDescriptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/annotation/pending/AnnotationPendingVariantsDescriptor.java @@ -1,6 +1,7 @@ package org.opencb.opencga.storage.hadoop.variant.annotation.pending; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.util.Bytes; @@ -16,8 +17,7 @@ import java.io.IOException; import java.util.function.Function; -import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema.VariantColumn.SO; -import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema.VariantColumn.TYPE; +import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema.VariantColumn.*; public class AnnotationPendingVariantsDescriptor implements PendingVariantsDescriptor { @@ -50,6 +50,7 @@ public boolean createTableIfNeeded(String tableName, HBaseManager hBaseManager) public Scan configureScan(Scan scan, VariantStorageMetadataManager metadataManager) { scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, ALLELES.bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, SO.bytes()); return scan; } @@ -57,9 +58,22 @@ public Scan configureScan(Scan scan, VariantStorageMetadataManager metadataManag public Function getPendingEvaluatorMapper(VariantStorageMetadataManager metadataManager, boolean overwrite) { return value -> { + byte[] alleles = null; if (overwrite || isPending(value)) { + for (Cell cell : value.rawCells()) { + if (cell.getValueLength() > 0) { + if (Bytes.equals( + cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), + ALLELES.bytes(), 0, ALLELES.bytes().length)) { + alleles = CellUtil.cloneValue(cell); + } + } + } Put put = new Put(value.getRow()); put.addColumn(FAMILY, COLUMN, VALUE); + if (alleles != null) { + put.addColumn(FAMILY, ALLELES.bytes(), alleles); + } return put; } else { return new Delete(value.getRow()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java index 86104fbe0d7..c1de4b9f1b1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveDriver.java @@ -65,7 +65,6 @@ public class ArchiveDriver extends Configured implements Tool { public static final String CONFIG_ARCHIVE_INPUT_FILE_VCF = "opencga.archive.input.file.vcf"; public static final String CONFIG_ARCHIVE_INPUT_FILE_VCF_META = "opencga.archive.input.file.vcf.meta"; - public static final String CONFIG_ARCHIVE_TABLE_NAME = "opencga.archive.table.name"; private static final Logger LOGGER = LoggerFactory.getLogger(ArchiveDriver.class); @@ -82,7 +81,7 @@ public int run(String[] args) throws Exception { URI inputFile = URI.create(conf.get(CONFIG_ARCHIVE_INPUT_FILE_VCF)); URI inputMetaFile = URI.create(conf.get(CONFIG_ARCHIVE_INPUT_FILE_VCF_META)); - String tableName = conf.get(CONFIG_ARCHIVE_TABLE_NAME); + String tableName = conf.get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME); int studyId = conf.getInt(HadoopVariantStorageEngine.STUDY_ID, -1); int fileId = conf.getInt(HadoopVariantStorageEngine.FILE_ID, -1); @@ -223,7 +222,7 @@ public static int privateMain(String[] args, Configuration conf) throws Exceptio conf.set(CONFIG_ARCHIVE_INPUT_FILE_VCF, toolArgs[0]); conf.set(CONFIG_ARCHIVE_INPUT_FILE_VCF_META, toolArgs[1]); conf = HBaseManager.addHBaseSettings(conf, toolArgs[2]); - conf.set(CONFIG_ARCHIVE_TABLE_NAME, toolArgs[3]); + conf.set(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, toolArgs[3]); conf.set(HadoopVariantStorageEngine.STUDY_ID, toolArgs[4]); conf.set(HadoopVariantStorageEngine.FILE_ID, toolArgs[5]); //set the configuration back, so that Tool can configure itself diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java index 7c831e3cc81..3af44e97188 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/archive/ArchiveTableHelper.java @@ -54,6 +54,7 @@ public class ArchiveTableHelper extends GenomeHelper { public static final byte[] NON_REF_COLUMN_SUFIX_BYTES = Bytes.toBytes(NON_REF_COLUMN_SUFIX); public static final String REF_COLUMN_SUFIX = "_R"; public static final byte[] REF_COLUMN_SUFIX_BYTES = Bytes.toBytes(REF_COLUMN_SUFIX); + public static final String CONFIG_ARCHIVE_TABLE_NAME = "opencga.archive.table.name"; private final Logger logger = LoggerFactory.getLogger(ArchiveTableHelper.class); private final AtomicReference meta = new AtomicReference<>(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java index 026a6a5e122..4bfcb0011e1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/HBaseToVariantConverter.java @@ -48,8 +48,7 @@ import java.util.stream.Collectors; import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.SEARCH_INDEX_LAST_TIMESTAMP; -import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory.extractVariantFromResultSet; -import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory.extractVariantFromVariantRowKey; +import static org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory.*; /** * Created on 20/11/15. @@ -210,7 +209,7 @@ private static class ResultToVariantConverter extends HBaseToVariantConverter 0) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/VariantRow.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/VariantRow.java index 4b0b6ef9584..6f4ee0ad344 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/VariantRow.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/VariantRow.java @@ -64,9 +64,7 @@ public static VariantRowWalkerBuilder walker(ResultSet resultSet) { public Variant getVariant() { if (variant == null) { if (result != null) { - byte[] row = result.getRow(); - Objects.requireNonNull(row, "Empty result. Missing variant rowkey."); - variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(row); + variant = VariantPhoenixKeyFactory.extractVariantFromResult(result); } else { variant = VariantPhoenixKeyFactory.extractVariantFromResultSet(resultSet); } @@ -104,11 +102,27 @@ public VariantRowWalkerBuilder walker() { } public void walk(VariantRowWalker walker) { - walk(walker, true, true, true, true, true); + walk(walker, true, true, true, true, true, null); } - protected void walk(VariantRowWalker walker, boolean file, boolean sample, boolean cohort, boolean score, boolean annotation) { - walker.variant(getVariant()); + protected void walk(VariantRowWalker walker, boolean file, boolean sample, boolean cohort, boolean score, boolean annotation, + Variant variant) { + if (variant == null) { + variant = getVariant(); + } else { + byte[] expectedRow = VariantPhoenixKeyFactory.generateVariantRowKey(variant); + byte[] actualRow; + if (result != null) { + actualRow = result.getRow(); + } else { + actualRow = VariantPhoenixKeyFactory.generateVariantRowKey(resultSet); + } + if (!Bytes.equals(expectedRow, actualRow)) { + throw new IllegalStateException("Expected row " + + Bytes.toStringBinary(expectedRow) + " but got " + Bytes.toStringBinary(actualRow)); + } + } + walker.variant(variant); if (resultSet != null) { try { ResultSetMetaData metaData = resultSet.getMetaData(); @@ -136,6 +150,10 @@ protected void walk(VariantRowWalker walker, boolean file, boolean sample, boole walker.fillMissing(studyId, resultSet.getInt(i)); } else if (annotation && columnName.equals(VariantColumn.FULL_ANNOTATION.column())) { walker.variantAnnotation(new BytesVariantAnnotationColumn(bytes)); + } else if (columnName.equals(VariantColumn.TYPE.column())) { + walker.type(VariantType.valueOf(Bytes.toString(bytes))); + } else if (columnName.equals(VariantColumn.ALLELES.column())) { + walker.alleles(Bytes.toString(bytes)); } } } catch (SQLException e) { @@ -164,6 +182,8 @@ protected void walk(VariantRowWalker walker, boolean file, boolean sample, boole walker.variantAnnotation(new BytesVariantAnnotationColumn(cell)); } else if (columnName.equals(VariantColumn.TYPE.column())) { walker.type(VariantType.valueOf(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()))); + } else if (columnName.equals(VariantColumn.ALLELES.column())) { + walker.alleles(Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } } } @@ -177,6 +197,9 @@ protected void variant(Variant variant) { protected void type(VariantType type) { } + protected void alleles(String alleles) { + } + protected void study(int studyId) { } @@ -202,6 +225,7 @@ protected void variantAnnotation(VariantAnnotationColumn column) { public class VariantRowWalkerBuilder extends VariantRowWalker { private IntConsumer studyConsumer = r -> { }; + private Consumer allelesConsumer = r -> { }; private Consumer fileConsumer = r -> { }; private boolean hasFileConsumer = false; private Consumer sampleConsumer = r -> { }; @@ -229,6 +253,11 @@ protected void study(int studyId) { studyConsumer.accept(studyId); } + @Override + protected void alleles(String alleles) { + allelesConsumer.accept(alleles); + } + @Override protected void file(FileColumn fileColumn) { fileConsumer.accept(fileColumn); @@ -265,15 +294,29 @@ public Variant walk() { hasSampleConsumer, hasStatsConsumer, hasVariantScoreConsumer, - hasVariantAnnotationConsummer); + hasVariantAnnotationConsummer, null); return getVariant(); } + public void walk(Variant variant) { + VariantRow.this.walk(this, + hasFileConsumer, + hasSampleConsumer, + hasStatsConsumer, + hasVariantScoreConsumer, + hasVariantAnnotationConsummer, variant); + } + public VariantRowWalkerBuilder onStudy(IntConsumer consumer) { studyConsumer = consumer; return this; } + public VariantRowWalkerBuilder onAlleles(Consumer consumer) { + allelesConsumer = consumer; + return this; + } + public VariantRowWalkerBuilder onFile(Consumer consumer) { fileConsumer = consumer; hasFileConsumer = true; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/annotation/VariantAnnotationToPhoenixConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/annotation/VariantAnnotationToPhoenixConverter.java index c9bb8646c04..d991c8a3c30 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/annotation/VariantAnnotationToPhoenixConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/annotation/VariantAnnotationToPhoenixConverter.java @@ -16,7 +16,6 @@ package org.opencb.opencga.storage.hadoop.variant.converters.annotation; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; @@ -24,6 +23,7 @@ import org.opencb.biodata.models.variant.avro.*; import org.opencb.biodata.tools.commons.Converter; import org.opencb.commons.utils.CompressionUtils; +import org.opencb.opencga.storage.core.variant.annotation.converters.VariantAnnotationModelUtils; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; @@ -62,6 +62,7 @@ public class VariantAnnotationToPhoenixConverter extends AbstractPhoenixConverte } private int annotationId; + private VariantAnnotationModelUtils variantAnnotationModelUtils = new VariantAnnotationModelUtils(); @Deprecated public VariantAnnotationToPhoenixConverter(byte[] columnFamily) { @@ -114,10 +115,6 @@ public VariantAnnotationToPhoenixConverter(byte[] columnFamily, int annotationId // Set hpo = new HashSet<>(); Set drugs = new HashSet<>(); Set proteinKeywords = new HashSet<>(); - // Contains all the xrefs, and the id, the geneNames and transcripts - Set xrefs = new HashSet<>(); - - addNotNull(xrefs, variantAnnotation.getId()); List consequenceTypes = variantAnnotation.getConsequenceTypes() == null ? Collections.emptyList() @@ -125,7 +122,9 @@ public VariantAnnotationToPhoenixConverter(byte[] columnFamily, int annotationId for (ConsequenceType consequenceType : consequenceTypes) { addNotNull(genes, consequenceType.getGeneName()); addNotNull(genes, consequenceType.getGeneId()); + addNotNull(genes, consequenceType.getEnsemblGeneId()); addNotNull(transcripts, consequenceType.getTranscriptId()); + addNotNull(transcripts, consequenceType.getEnsemblTranscriptId()); addNotNull(biotype, consequenceType.getBiotype()); addAllNotNull(flags, consequenceType.getTranscriptFlags()); @@ -192,32 +191,14 @@ public VariantAnnotationToPhoenixConverter(byte[] columnFamily, int annotationId if (proteinVariantAnnotation.getKeywords() != null) { proteinKeywords.addAll(proteinVariantAnnotation.getKeywords()); } - addNotNull(xrefs, proteinVariantAnnotation.getUniprotName()); - addNotNull(xrefs, proteinVariantAnnotation.getUniprotAccession()); - addNotNull(xrefs, proteinVariantAnnotation.getUniprotVariantId()); } } - if (CollectionUtils.isNotEmpty(variantAnnotation.getTraitAssociation())) { - for (EvidenceEntry evidenceEntry : variantAnnotation.getTraitAssociation()) { - addNotNull(xrefs, evidenceEntry.getId()); - } - } - - xrefs.addAll(genes); - xrefs.addAll(transcripts); - if (variantAnnotation.getXrefs() != null) { - for (Xref xref : variantAnnotation.getXrefs()) { - addNotNull(xrefs, xref.getId()); - } - } if (variantAnnotation.getGeneTraitAssociation() != null) { for (GeneTraitAssociation geneTrait : variantAnnotation.getGeneTraitAssociation()) { addNotNull(geneTraitName, geneTrait.getName()); addNotNull(geneTraitId, geneTrait.getId()); - addNotNull(xrefs, geneTrait.getHpo()); - addNotNull(xrefs, geneTrait.getId()); } } @@ -235,6 +216,8 @@ public VariantAnnotationToPhoenixConverter(byte[] columnFamily, int annotationId geneSoFlag.remove(null); soFlag.remove(null); + Set xrefs = variantAnnotationModelUtils.extractXRefs(variantAnnotation); + map.put(CHROMOSOME, variantAnnotation.getChromosome()); map.put(POSITION, variantAnnotation.getStart()); map.put(REFERENCE, variantAnnotation.getReference()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/StudyEntryToHBaseConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/StudyEntryToHBaseConverter.java index e3188e15716..4a4bafa1be8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/StudyEntryToHBaseConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/converters/study/StudyEntryToHBaseConverter.java @@ -119,6 +119,7 @@ public Put convert(Variant variant) { add(put, VariantPhoenixSchema.VariantColumn.CI_END_R, Math.max(0, variant.getSv().getCiEndRight())); } } + add(put, VariantPhoenixSchema.VariantColumn.ALLELES, VariantPhoenixKeyFactory.buildAlleles(variant)); add(put, studyColumn, 0); if (releaseColumn != null) { add(put, releaseColumn, true); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java index da771b0cb47..a8d9b03745c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java @@ -16,20 +16,25 @@ package org.opencb.opencga.storage.hadoop.variant.executors; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.hadoop.util.Tool; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.*; @@ -44,6 +49,30 @@ public abstract class MRExecutor { private ObjectMap options; private List env; private static Logger logger = LoggerFactory.getLogger(MRExecutor.class); + private static final Pattern STDOUT_KEY_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9_]+=[^=]+$"); + private static final Pattern EXCEPTION = Pattern.compile("^Exception in thread \"main\" ([^:]+: .+)$"); + + public static class Result { + private final int exitValue; + private final ObjectMap result; + + public Result(int exitValue, ObjectMap result) { + this.exitValue = exitValue; + this.result = result; + } + + public int getExitValue() { + return exitValue; + } + + public ObjectMap getResult() { + return result; + } + + public String getErrorMessage() { + return result.getString(AbstractHBaseDriver.ERROR_MESSAGE); + } + } public MRExecutor init(ObjectMap options) { this.options = options; @@ -76,24 +105,32 @@ protected static String getOpencgaHome() { return System.getProperty("app.home", ""); } - public void run(Class execClass, String[] args, String taskDescription) + public ObjectMap run(Class execClass, String[] args, String taskDescription) throws StorageEngineException { StopWatch stopWatch = StopWatch.createStarted(); logger.info("------------------------------------------------------"); logger.info(taskDescription); logger.info("------------------------------------------------------"); - int exitValue = run(execClass, args); + Result result = run(execClass, args); + int exitValue = result.getExitValue(); logger.info("------------------------------------------------------"); logger.info("Exit value: {}", exitValue); logger.info("Total time: {}", TimeUtils.durationToString(stopWatch)); if (exitValue != 0) { - throw new StorageEngineException("Error executing MapReduce for : \"" + taskDescription + "\""); + String message = "Error executing MapReduce for : \"" + taskDescription + "\""; + if (StringUtils.isNotEmpty(result.getErrorMessage())) { + message += " : " + result.getErrorMessage(); + } else { + message += " : Unidentified error executing MapReduce job. Check logs for more information."; + } + throw new StorageEngineException(message); } + return result.getResult(); } - protected int run(Class execClass, String[] args) throws StorageEngineException { + protected Result run(Class execClass, String[] args) throws StorageEngineException { String hadoopRoute = options.getString(MR_HADOOP_BIN.key(), MR_HADOOP_BIN.defaultValue()); String jar = getJarWithDependencies(options); String executable = hadoopRoute + " jar " + jar + ' ' + execClass.getName(); @@ -112,7 +149,7 @@ protected int run(Class execClass, String[] args) throws Sto } } - public abstract int run(String executable, String[] args) throws StorageEngineException; + public abstract Result run(String executable, String[] args) throws StorageEngineException; protected ObjectMap getOptions() { return options; @@ -129,4 +166,28 @@ protected static void redactSecureString(String[] args, String key) { } } + protected static ObjectMap readResult(String output) { + ObjectMap result = new ObjectMap(); + List exceptions = new ArrayList<>(); + for (String line : output.split(System.lineSeparator())) { + if (STDOUT_KEY_VALUE_PATTERN.matcher(line).find()) { + String[] split = line.split("="); + if (split.length == 2) { + Object old = result.put(split[0], split[1]); + if (old != null) { + result.put(split[0], old + "," + split[1]); + } + } + } else if (EXCEPTION.matcher(line).find()) { + Matcher matcher = EXCEPTION.matcher(line); + if (matcher.find()) { + exceptions.add(matcher.group(1)); + } + } + } + if (!exceptions.isEmpty() && !result.containsKey(AbstractHBaseDriver.ERROR_MESSAGE)) { + result.put(AbstractHBaseDriver.ERROR_MESSAGE, String.join(", ", exceptions)); + } + return result; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java index 8ef826f153d..b205511f830 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SshMRExecutor.java @@ -7,14 +7,18 @@ import org.opencb.commons.exec.Command; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.*; @@ -30,6 +34,7 @@ public class SshMRExecutor extends MRExecutor { private static final String HADOOP_SSH_KEY_ENV = "HADOOP_SSH_KEY"; // env-var expected by "sshpass -e" private static final String SSHPASS_ENV = "SSHPASS"; + public static final String PID = "PID"; private static Logger logger = LoggerFactory.getLogger(SshMRExecutor.class); @Override @@ -39,20 +44,83 @@ public SshMRExecutor init(ObjectMap options) { } @Override - public int run(String executable, String[] args) throws StorageEngineException { + public Result run(String executable, String[] args) throws StorageEngineException { String commandLine = buildCommand(executable, args); List env = buildEnv(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Thread hook = new Thread(() -> { + logger.info("Shutdown hook. Killing MR jobs"); + logger.info("Read output key-value:"); + ObjectMap result = readResult(new String(outputStream.toByteArray(), Charset.defaultCharset())); + for (Map.Entry entry : result.entrySet()) { + logger.info(" - " + entry.getKey() + ": " + entry.getValue()); + } + String remotePid = result.getString(PID); + logger.info("Remote PID: " + remotePid); + List mrJobs = result.getAsStringList(AbstractHBaseDriver.MR_APPLICATION_ID); + logger.info("MR jobs to kill: " + mrJobs); + for (String mrJob : mrJobs) { + logger.info("Killing MR job " + mrJob); + String commandLineKill = buildCommand("yarn", "application", "-kill", mrJob); + Command command = new Command(commandLineKill, env); + command.run(); + int exitValue = command.getExitValue(); + if (exitValue != 0) { + logger.error("Error killing MR job " + mrJob); + } else { + logger.info("MR job " + mrJob + " killed!"); + } + } + if (remotePid != null) { + int remoteProcessGraceKillPeriod = getOptions() + .getInt(MR_EXECUTOR_SSH_HADOOP_TERMINATION_GRACE_PERIOD_SECONDS.key(), + MR_EXECUTOR_SSH_HADOOP_TERMINATION_GRACE_PERIOD_SECONDS.defaultValue()); + logger.info("Wait up to " + remoteProcessGraceKillPeriod + "s for the remote process to finish"); + String commandLineWaitPid = buildCommand("bash", "-c", "" + + "pid=" + remotePid + "; " + + "i=0; " + + "while [ $(( i++ )) -lt " + remoteProcessGraceKillPeriod + " ] && ps -p $pid > /dev/null; do sleep 1; done; " + + "if ps -p $pid > /dev/null; then " + + " echo \"Kill remote ssh process $pid\"; " + + " ps -p $pid; " + + " kill -15 $pid; " + + "else " + + " echo \"Process $pid finished\"; " + + " fi"); + Command command = new Command(commandLineWaitPid, env); + command.run(); + if (command.getExitValue() != 0) { + logger.error("Error waiting for remote process to finish"); + } else { + logger.info("Remote process finished!"); + } + } + }); + Runtime.getRuntime().addShutdownHook(hook); Command command = new Command(commandLine, env); + command.setErrorOutputStream(outputStream); command.run(); int exitValue = command.getExitValue(); - + Runtime.getRuntime().removeShutdownHook(hook); + ObjectMap result = readResult(new String(outputStream.toByteArray(), Charset.defaultCharset())); if (exitValue == 0) { copyOutputFiles(args, env); } - return exitValue; + return new Result(exitValue, result); } + /** + * Copy output files from remote server to local filesystem. + *

    + * This method will look for the "output" argument in the args array. + * The value of the argument is expected to be a path. + * + * @param args Arguments passed to the executable + * @param env Environment variables + * @return Path to the output file + * @throws StorageEngineException if there is an issue copying the files + */ private Path copyOutputFiles(String[] args, List env) throws StorageEngineException { List argsList = Arrays.asList(args); int outputIdx = argsList.indexOf("output"); @@ -83,6 +151,7 @@ private Path copyOutputFiles(String[] args, List env) throws StorageEngi return Paths.get(targetOutput); } } + // Nothing to copy return null; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java index bb226be8d44..b46be099230 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/SystemMRExecutor.java @@ -17,8 +17,12 @@ package org.opencb.opencga.storage.hadoop.variant.executors; import org.apache.tools.ant.types.Commandline; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.exec.Command; +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; + import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.MR_EXECUTOR_SSH_PASSWORD; /** @@ -29,14 +33,13 @@ public class SystemMRExecutor extends MRExecutor { @Override - public int run(String executable, String[] args) { - return run(buildCommandLine(executable, args)); - } - - public int run(String commandLine) { - Command command = new Command(commandLine, getEnv()); + public Result run(String executable, String[] args) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Command command = new Command(buildCommandLine(executable, args), getEnv()); + command.setErrorOutputStream(outputStream); command.run(); - return command.getExitValue(); + ObjectMap result = readResult(new String(outputStream.toByteArray(), Charset.defaultCharset())); + return new Result(command.getExitValue(), result); } private String buildCommandLine(String executable, String[] args) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/AbstractFillFromArchiveTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/AbstractFillFromArchiveTask.java index d218cf0a470..1e0a01f89b4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/AbstractFillFromArchiveTask.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/AbstractFillFromArchiveTask.java @@ -32,7 +32,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; -import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine.TARGET_VARIANT_TYPE_SET; +import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine.UNSUPPORTED_VARIANT_TYPE_SET; /** * Created on 31/10/17. @@ -348,7 +348,7 @@ protected static Scan buildScan(String regionStr, int fileId, Configuration conf protected static boolean isVariantAlreadyLoaded(VcfSliceProtos.VcfSlice slice, VcfSliceProtos.VcfRecord vcfRecord) { VariantType variantType = VcfRecordProtoToVariantConverter.getVariantType(vcfRecord.getType()); // The variant is not loaded if is a NO_VARIATION (fast check first) - if (!TARGET_VARIANT_TYPE_SET.contains(variantType)) { + if (UNSUPPORTED_VARIANT_TYPE_SET.contains(variantType)) { return false; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java index 36d408f6fef..9b8df0c579d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsFromArchiveMapper.java @@ -6,7 +6,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.mapreduce.Job; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.mr.AbstractArchiveTableMapper; import org.opencb.opencga.storage.hadoop.variant.mr.VariantsTableMapReduceHelper; @@ -63,7 +63,7 @@ protected void setup(Context context) throws IOException, InterruptedException { long timestamp = getMrHelper().getTimestamp(); if (isFillGaps(context.getConfiguration())) { Collection samples = getSamples(context.getConfiguration()); - String archiveTableName = context.getConfiguration().get(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME); + String archiveTableName = context.getConfiguration().get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME); String gapsGenotype = context.getConfiguration().get( FILL_GAPS_GAP_GENOTYPE.key(), FILL_GAPS_GAP_GENOTYPE.defaultValue()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java index 06139af23a5..cacd50da458 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsMapper.java @@ -7,7 +7,7 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.mr.VariantMapper; import java.io.IOException; @@ -28,7 +28,7 @@ protected void setup(Context context) throws IOException, InterruptedException { Configuration configuration = context.getConfiguration(); HBaseManager hBaseManager = new HBaseManager(configuration); - String archiveTableName = context.getConfiguration().get(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME); + String archiveTableName = context.getConfiguration().get(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME); Collection samples = FillGapsFromArchiveMapper.getSamples(configuration); fillGapsTask = new FillGapsFromVariantTask(hBaseManager, archiveTableName, diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingDriver.java index aea25583571..d760ab4543a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingDriver.java @@ -62,6 +62,8 @@ protected Job setupJob(Job job, String archiveTableName, String variantTableName scan.setCacheBlocks(false); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.getStudyColumn(getStudyId()).bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.getFillMissingColumn(getStudyId()).bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); LOG.info(scan.toString()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingMapper.java index 080a1a66a7a..10feadf2cc8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/PrepareFillMissingMapper.java @@ -57,7 +57,7 @@ protected void setup(Context context) throws IOException, InterruptedException { @Override protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException { - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(value.getRow()); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromResult(value); byte[] column = FillMissingFromArchiveTask.getArchiveVariantColumn(variant); long sliceId = rowKeyFactory.getSliceId(variant.getStart()); String chromosome = variant.getChromosome(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexCompoundHeterozygousQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexCompoundHeterozygousQueryExecutor.java index cb2019c2f6d..c7599e7e288 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexCompoundHeterozygousQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexCompoundHeterozygousQueryExecutor.java @@ -5,7 +5,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java index 68eed2db889..4dd50c9eacf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexMendelianErrorQueryExecutor.java @@ -15,6 +15,7 @@ import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.query.SampleIndexQuery; @@ -43,7 +44,7 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) { } @Override - protected Object getOrIterator(Query query, QueryOptions options, boolean iterator, SampleIndexQuery sampleIndexQuery) { + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator, SampleIndexQuery sampleIndexQuery) { List trios = new ArrayList<>(sampleIndexQuery.getMendelianErrorSet().size()); int studyId = metadataManager.getStudyId(sampleIndexQuery.getStudy()); @@ -65,7 +66,7 @@ protected Object getOrIterator(Query query, QueryOptions options, boolean iterat trios.add(new Trio(null, father, mother, sampleMetadata.getName())); } - Object object = super.getOrIterator(query, options, iterator, sampleIndexQuery); + Object object = super.getOrIterator(variantQuery, iterator, sampleIndexQuery); if (iterator) { VariantDBIterator variantIterator = (VariantDBIterator) object; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 510085420dd..371d57ac4da 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -9,7 +9,6 @@ import org.opencb.biodata.models.variant.avro.FileEntry; import org.opencb.biodata.models.variant.avro.SampleEntry; import org.opencb.biodata.tools.commons.Converter; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -17,7 +16,6 @@ import org.opencb.opencga.core.common.BatchUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.IndexFieldConfiguration; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.io.bit.BitBuffer; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; @@ -26,7 +24,7 @@ import org.opencb.opencga.storage.core.variant.adaptors.*; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; -import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutor; import org.opencb.opencga.storage.core.variant.query.projection.VariantQueryProjection; @@ -50,7 +48,6 @@ import java.util.stream.Collectors; import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.NONE; -import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.addSamplesMetadataIfRequested; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.SAMPLE_INDEX_QUERY_SAMPLE_INDEX_ONLY_PD_BATCH; import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions.SAMPLE_INDEX_QUERY_SAMPLE_INDEX_ONLY_PD_BUFFER; import static org.opencb.opencga.storage.hadoop.variant.index.SampleIndexVariantQueryExecutor.SAMPLE_INDEX_TABLE_SOURCE; @@ -64,7 +61,6 @@ public class SampleIndexOnlyVariantQueryExecutor extends VariantQueryExecutor { private final SampleIndexDBAdaptor sampleIndexDBAdaptor; private final VariantHadoopDBAdaptor dbAdaptor; - private final VariantQueryParser variantQueryParser; private final VariantQueryProjectionParser variantQueryProjectionParser; private static Logger logger = LoggerFactory.getLogger(SampleIndexOnlyVariantQueryExecutor.class); @@ -82,7 +78,6 @@ public SampleIndexOnlyVariantQueryExecutor(VariantHadoopDBAdaptor dbAdaptor, Sam super(dbAdaptor.getMetadataManager(), storageEngineId, options); this.sampleIndexDBAdaptor = sampleIndexDBAdaptor; this.dbAdaptor = dbAdaptor; - variantQueryParser = new VariantQueryParser(null, getMetadataManager()); variantQueryProjectionParser = new VariantQueryProjectionParser(getMetadataManager()); partialDataBufferSize = options.getInt(SAMPLE_INDEX_QUERY_SAMPLE_INDEX_ONLY_PD_BUFFER.key(), SAMPLE_INDEX_QUERY_SAMPLE_INDEX_ONLY_PD_BUFFER.defaultValue()); @@ -101,26 +96,16 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) { return false; } - @Override - public DataResult count(Query query) { - StopWatch stopWatch = StopWatch.createStarted(); - SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); - long count = sampleIndexDBAdaptor.count(sampleIndexQuery); - return new DataResult<>(((int) stopWatch.getTime(TimeUnit.MILLISECONDS)), Collections.emptyList(), 1, - Collections.singletonList(count), count); - } - /** * Fetch results exclusively from SampleSecondaryIndex. * - * @param inputQuery Query - * @param options Options + * @param variantQuery Query * @param iterator Shall the resulting object be an iterator instead of a DataResult * @return DataResult or Iterator with the variants that matches the query */ @Override - protected Object getOrIterator(Query inputQuery, QueryOptions options, boolean iterator) { - Query query = new Query(inputQuery); + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { + Query query = new Query(variantQuery.getQuery()); query.put(SampleIndexQueryParser.INCLUDE_PARENTS_COLUMN, true); SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); @@ -128,7 +113,7 @@ protected Object getOrIterator(Query inputQuery, QueryOptions options, boolean i boolean count; Future asyncCountFuture; - if (shouldGetCount(options, iterator)) { + if (shouldGetCount(variantQuery.getInputOptions(), iterator)) { count = true; asyncCountFuture = THREAD_POOL.submit(() -> { StopWatch stopWatch = StopWatch.createStarted(); @@ -141,13 +126,12 @@ protected Object getOrIterator(Query inputQuery, QueryOptions options, boolean i asyncCountFuture = null; } - VariantDBIterator variantIterator = getVariantDBIterator(sampleIndexQuery, inputQuery, options); + VariantDBIterator variantIterator = getVariantDBIterator(sampleIndexQuery, variantQuery); if (iterator) { return variantIterator; } else { - VariantQueryResult result = - addSamplesMetadataIfRequested(variantIterator.toDataResult(), inputQuery, options, getMetadataManager()); + VariantQueryResult result = variantIterator.toDataResult(variantQuery); if (count) { result.setApproximateCount(false); try { @@ -161,8 +145,8 @@ protected Object getOrIterator(Query inputQuery, QueryOptions options, boolean i } } - private VariantDBIterator getVariantDBIterator(SampleIndexQuery sampleIndexQuery, Query inputQuery, QueryOptions options) { - ParsedVariantQuery parsedQuery = variantQueryParser.parseQuery(inputQuery, options, true); + private VariantDBIterator getVariantDBIterator(SampleIndexQuery sampleIndexQuery, ParsedVariantQuery parsedQuery) { + QueryOptions options = parsedQuery.getInputOptions(); VariantDBIterator variantIterator; if (parsedQuery.getProjection().getStudyIds().isEmpty()) { logger.info("Using sample index iterator Iterator"); @@ -174,9 +158,9 @@ private VariantDBIterator getVariantDBIterator(SampleIndexQuery sampleIndexQuery try { rawIterator = sampleIndexDBAdaptor.rawIterator(sampleIndexQuery, options); } catch (IOException e) { - throw VariantQueryException.internalException(e).setQuery(inputQuery); + throw VariantQueryException.internalException(e).setQuery(parsedQuery.getInputQuery()); } - boolean includeAll = inputQuery.getBoolean("includeAllFromSampleIndex", false); + boolean includeAll = parsedQuery.getInputQuery().getBoolean("includeAllFromSampleIndex", false); SampleVariantIndexEntryToVariantConverter converter = new SampleVariantIndexEntryToVariantConverter( parsedQuery, sampleIndexQuery, dbAdaptor.getMetadataManager(), includeAll); variantIterator = VariantDBIterator.wrapper(Iterators.transform(rawIterator, converter::convert)); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java index f193dcad3e2..973a922780b 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantAggregationExecutor.java @@ -9,7 +9,7 @@ import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.io.bit.BitBuffer; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.utils.iterators.CloseableIterator; @@ -142,7 +142,7 @@ protected VariantQueryResult aggregation(Query query, QueryOptions o } return new VariantQueryResult<>((int) stopWatch.getTime(TimeUnit.MILLISECONDS), 1, numMatches, Collections.emptyList(), - fields, null, SampleIndexVariantQueryExecutor.SAMPLE_INDEX_TABLE_SOURCE); + fields, SampleIndexVariantQueryExecutor.SAMPLE_INDEX_TABLE_SOURCE); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java index 64cf8669153..6175128018d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexVariantQueryExecutor.java @@ -3,17 +3,17 @@ import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.commons.lang3.time.StopWatch; import org.opencb.biodata.models.variant.Variant; -import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.MultiVariantDBIterator; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIteratorWithCounts; +import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.core.variant.query.executors.AbstractTwoPhasedVariantQueryExecutor; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; @@ -31,7 +31,6 @@ import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.ID; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.REGION; -import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.addSamplesMetadataIfRequested; /** * Created on 01/04/19. @@ -66,11 +65,6 @@ public boolean canUseThisExecutor(Query query, QueryOptions options) { return false; } - @Override - public DataResult count(Query query) { - throw new UnsupportedOperationException("Count not implemented in " + getClass()); - } - @Override protected long primaryCount(Query query, QueryOptions options) { SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); @@ -81,24 +75,25 @@ protected long primaryCount(Query query, QueryOptions options) { * Intersect result of SampleIndexTable and full phoenix query. * Use {@link org.opencb.opencga.storage.core.variant.adaptors.iterators.MultiVariantDBIterator}. * - * @param inputQuery Query - * @param options Options + * @param variantQuery Parsed query * @param iterator Shall the resulting object be an iterator instead of a DataResult * @return DataResult or Iterator with the variants that matches the query */ @Override - protected Object getOrIterator(Query inputQuery, QueryOptions options, boolean iterator) { - Query query = new Query(inputQuery); + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator) { + Query query = new Query(variantQuery.getQuery()); SampleIndexQuery sampleIndexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(query); - return getOrIterator(query, options, iterator, sampleIndexQuery); + return getOrIterator(variantQuery, iterator, sampleIndexQuery); } - protected Object getOrIterator(Query query, QueryOptions inputOptions, boolean iterator, SampleIndexQuery sampleIndexQuery) { + protected Object getOrIterator(ParsedVariantQuery variantQuery, boolean iterator, SampleIndexQuery sampleIndexQuery) { logger.info("HBase SampleIndex intersect"); + QueryOptions inputOptions = variantQuery.getInputOptions(); + Query uncoveredQuery = new Query(sampleIndexQuery.getUncoveredQuery()); Future asyncCountFuture; boolean asyncCount; - if (shouldGetApproximateCount(inputOptions, iterator) && queryFiltersCovered(query)) { + if (shouldGetApproximateCount(inputOptions, iterator) && queryFiltersCovered(uncoveredQuery)) { asyncCount = true; asyncCountFuture = THREAD_POOL.submit(() -> { StopWatch stopWatch = StopWatch.createStarted(); @@ -120,10 +115,10 @@ protected Object getOrIterator(Query query, QueryOptions inputOptions, boolean i int batchSize = inputOptions.getInt("multiIteratorBatchSize", 200); if (iterator) { // SampleIndex iterator will be closed when closing the variants iterator - return dbAdaptor.iterator(variants, query, inputOptions, batchSize); + return dbAdaptor.iterator(variants, uncoveredQuery, inputOptions, batchSize); } else { - int skip = getSkip(inputOptions); - int limit = getLimit(inputOptions); + int skip = variantQuery.getSkip(); + int limit = variantQuery.getLimitOr(-1); int samplingSize = asyncCount ? 0 : getSamplingSize(inputOptions, DEFAULT_SAMPLING_SIZE, iterator); int tmpLimit = Math.max(limit, samplingSize); @@ -131,7 +126,9 @@ protected Object getOrIterator(Query query, QueryOptions inputOptions, boolean i // Ensure results are sorted and it's not counting from variants dbAdaptor options.put(QueryOptions.SORT, true); options.put(QueryOptions.COUNT, false); - options.put(QueryOptions.LIMIT, tmpLimit); + if (limit > 0) { + options.put(QueryOptions.LIMIT, tmpLimit); + } MultiVariantDBIterator variantDBIterator = dbAdaptor.iterator( new org.opencb.opencga.storage.core.variant.adaptors.iterators.DelegatedVariantDBIterator(variants) { @@ -139,9 +136,8 @@ protected Object getOrIterator(Query query, QueryOptions inputOptions, boolean i public void close() throws Exception { // Do not close this iterator! We'll need to keep iterating to get the approximate count } - }, query, options, batchSize); - VariantQueryResult result = - addSamplesMetadataIfRequested(variantDBIterator.toDataResult(), query, options, getMetadataManager()); + }, uncoveredQuery, options, batchSize); + VariantQueryResult result = variantDBIterator.toDataResult(variantQuery); if (result.getNumResults() < tmpLimit) { // Not an approximate count! @@ -162,11 +158,11 @@ public void close() throws Exception { inputOptions.get(QueryOptions.COUNT)); numTotalResultsOptions.put(VariantStorageOptions.APPROXIMATE_COUNT.key(), inputOptions.get(VariantStorageOptions.APPROXIMATE_COUNT.key())); - setNumTotalResults(variantDBIterator, variants, result, sampleIndexQuery, query, numTotalResultsOptions); + setNumTotalResults(variantDBIterator, variants, result, sampleIndexQuery, uncoveredQuery, numTotalResultsOptions); } // Ensure limit - if (result.getNumResults() > limit) { + if (limit > 0 && result.getNumResults() > limit) { result.setResults(result.getResults().subList(0, limit)); result.setNumResults(limit); } @@ -195,7 +191,7 @@ private boolean queryFiltersCovered(Query query) { // Check if the query is fully covered Set params = VariantQueryUtils.validParams(query, true); params.remove(VariantQueryParam.STUDY); - + logger.info("Uncovered filters : " + params); return params.isEmpty(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/SampleIndexAnnotationLoaderDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/SampleIndexAnnotationLoaderDriver.java index 41826fd4abe..9f593f928c6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/SampleIndexAnnotationLoaderDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/SampleIndexAnnotationLoaderDriver.java @@ -138,6 +138,8 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws } } scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.FULL_ANNOTATION.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); SampleIndexAnnotationLoaderMapper.setHasGenotype(job, hasGenotype); SampleIndexAnnotationLoaderMapper.setMultiFileSamples(job, multiFileSamples); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/VariantTableSampleIndexOrderMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/VariantTableSampleIndexOrderMapper.java index 0d1e45f40be..ab9fc2a1d82 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/VariantTableSampleIndexOrderMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/annotation/mr/VariantTableSampleIndexOrderMapper.java @@ -51,11 +51,7 @@ public void run(Context context) throws IOException, InterruptedException { // Current result does not start in the previous position. // Sort buffer if (buffer.size() > 1) { - buffer.sort((o1, o2) -> { - Variant v1 = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(o1.getSecond().getRow()); - Variant v2 = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(o2.getSecond().getRow()); - return SampleIndexSchema.INTRA_CHROMOSOME_VARIANT_COMPARATOR.compare(v1, v2); - }); + sortBuffer(buffer); } if (!buffer.isEmpty()) { @@ -82,6 +78,9 @@ public void run(Context context) throws IOException, InterruptedException { } } countBufferSize(context, buffer.size(), chromosome, position); + if (buffer.size() > 1) { + sortBuffer(buffer); + } for (Pair pair : buffer) { this.map(pair.getFirst(), pair.getSecond(), context); } @@ -92,6 +91,14 @@ public void run(Context context) throws IOException, InterruptedException { } } + private static void sortBuffer(List> buffer) { + buffer.sort((o1, o2) -> { + Variant v1 = VariantPhoenixKeyFactory.extractVariantFromResult(o1.getSecond()); + Variant v2 = VariantPhoenixKeyFactory.extractVariantFromResult(o2.getSecond()); + return SampleIndexSchema.INTRA_CHROMOSOME_VARIANT_COMPARATOR.compare(v1, v2); + }); + } + private void countBufferSize(Context context, int size, String chromosome, int position) { String counterName; if (size < 5) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java index ada52270419..af632d69436 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java @@ -226,7 +226,8 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws } } } -// scan.addColumn(getHelper().getColumnFamily(), VariantPhoenixHelper.VariantColumn.FULL_ANNOTATION.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); VariantMapReduceUtil.configureMapReduceScan(scan, getConf()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleIndexQuery.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleIndexQuery.java index 664ace07747..4a4803ff6a4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleIndexQuery.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SampleIndexQuery.java @@ -4,6 +4,7 @@ import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.VariantType; +import org.opencb.commons.datastore.core.Query; import org.opencb.opencga.storage.core.variant.query.Values; import org.opencb.opencga.storage.hadoop.variant.index.family.GenotypeCodec; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; @@ -49,6 +50,8 @@ public class SampleIndexQuery { private final Set mendelianErrorSet; private final MendelianErrorType mendelianErrorType; private final boolean includeParentColumns; + + private final Query uncoveredQuery; private final QueryOperation queryOperation; public enum MendelianErrorType { @@ -73,6 +76,7 @@ public SampleIndexQuery(Collection locusQueries, SampleIndexQuery qu this.mendelianErrorSet = query.mendelianErrorSet; this.mendelianErrorType = query.mendelianErrorType; this.includeParentColumns = query.includeParentColumns; + this.uncoveredQuery = query.uncoveredQuery; this.queryOperation = query.queryOperation; } @@ -83,7 +87,7 @@ public SampleIndexQuery(SampleIndexSchema schema, Collection locusQu Map> fileFilterMap, SampleAnnotationIndexQuery annotationIndexQuery, Set mendelianErrorSet, MendelianErrorType mendelianErrorType, boolean includeParentColumns, - QueryOperation queryOperation) { + QueryOperation queryOperation, Query uncoveredQuery) { this.schema = schema; this.locusQueries = locusQueries; this.extendedFilteringRegion = extendedFilteringRegion; @@ -100,6 +104,7 @@ public SampleIndexQuery(SampleIndexSchema schema, Collection locusQu this.mendelianErrorType = mendelianErrorType; this.includeParentColumns = includeParentColumns; this.queryOperation = queryOperation; + this.uncoveredQuery = uncoveredQuery; } public SampleIndexSchema getSchema() { @@ -227,6 +232,10 @@ public boolean isIncludeParentColumns() { return includeParentColumns; } + public Query getUncoveredQuery() { + return uncoveredQuery; + } + public QueryOperation getQueryOperation() { return queryOperation; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SingleSampleIndexQuery.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SingleSampleIndexQuery.java index 35a9e67e718..51f44824872 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SingleSampleIndexQuery.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/query/SingleSampleIndexQuery.java @@ -36,7 +36,8 @@ protected SingleSampleIndexQuery(SampleIndexQuery query, String sample, List scans; if (multiScan) { @@ -220,6 +221,8 @@ protected Job setupJob(Job job, String archiveTable, String table) throws IOExce if (StringUtils.isNotEmpty(region)) { VariantHBaseQueryParser.addRegionFilter(scan, Region.parseRegion(region)); } + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); scan.setFilter(filter); scans.add(scan); for (int sample : samplesSubSet) { @@ -247,6 +250,8 @@ protected Job setupJob(Job job, String archiveTable, String table) throws IOExce VariantHBaseQueryParser.addRegionFilter(scan, Region.parseRegion(region)); } scan.setFilter(filter); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); int approxExpectedNumColumns = sampleIds.size() diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 673061dec30..027d241282c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -274,7 +274,7 @@ public SampleIndexQuery parse(Query query) { //} else if (isValidParam(query, FILE)) { // Add FILEs filter ? } else { - throw new IllegalStateException("Unable to query SamplesIndex"); + throw new IllegalStateException("Unable to query SamplesIndex. Missing sample filter! Query: " + query.toJson()); } boolean requireFamilyIndex = !mendelianErrorSet.isEmpty(); SampleIndexSchema schema = schemaFactory.getSchema(studyId, sampleGenotypeQuery.keySet(), false, requireFamilyIndex); @@ -551,7 +551,7 @@ public SampleIndexQuery parse(Query query) { return new SampleIndexQuery(schema, regionGroups, extendedFilteringRegion, variantTypes, study, sampleGenotypeQuery, multiFileSamples, negatedSamples, fatherFilterMap, motherFilterMap, - fileIndexMap, annotationIndexQuery, mendelianErrorSet, mendelianErrorType, includeParentsField, queryOperation); + fileIndexMap, annotationIndexQuery, mendelianErrorSet, mendelianErrorType, includeParentsField, queryOperation, query); } private Set findParents(Set childrenSet, Map> parentsMap) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java index 1004dfa36c2..f1b167ae9c1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexVariantBiConverter.java @@ -661,7 +661,7 @@ private Variant toVariantEncodedAlleles(String chromosome, int batchStart, byte[ String[] refAlt = AlleleCodec.decode(bytes[offset]); int start = batchStart + (read24bitInteger(bytes, offset) & 0x0F_FF_FF); - return VariantPhoenixKeyFactory.buildVariant(chromosome, start, refAlt[0], refAlt[1], null); + return VariantPhoenixKeyFactory.buildVariant(chromosome, start, refAlt[0], refAlt[1], null, null); } private Variant toVariant(String chromosome, int batchStart, byte[] bytes, int offset, int referenceLength, int alternateLength) { @@ -671,7 +671,7 @@ private Variant toVariant(String chromosome, int batchStart, byte[] bytes, int o offset += referenceLength + SEPARATOR_LENGTH; // add reference, and separator String alternate = readString(bytes, offset, alternateLength); - return VariantPhoenixKeyFactory.buildVariant(chromosome, start, reference, alternate, null); + return VariantPhoenixKeyFactory.buildVariant(chromosome, start, reference, alternate, null, null); } private int readNextSeparator(byte[] bytes, int offset) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/io/VariantExporterDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/io/VariantExporterDriver.java index 196ab0e1689..aa342c10a92 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/io/VariantExporterDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/io/VariantExporterDriver.java @@ -33,6 +33,7 @@ import org.opencb.opencga.storage.core.variant.query.ParsedVariantQuery; import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.hadoop.variant.AbstractVariantsTableDriver; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantQueryParser; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHBaseQueryParser; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantSqlQueryParser; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; @@ -188,7 +189,7 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws VariantMapReduceUtil.setNoneReduce(job); } - VariantQueryParser variantQueryParser = new VariantQueryParser(null, getMetadataManager()); + VariantQueryParser variantQueryParser = new HadoopVariantQueryParser(null, getMetadataManager()); ParsedVariantQuery variantQuery = variantQueryParser.parseQuery(query, options); Query query = variantQuery.getQuery(); if (VariantHBaseQueryParser.isSupportedQuery(query)) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseTaskMetadataDBAdaptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseTaskMetadataDBAdaptor.java index 290f937631b..4fe8d236cc1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseTaskMetadataDBAdaptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/metadata/HBaseTaskMetadataDBAdaptor.java @@ -28,7 +28,7 @@ public TaskMetadata getTask(int studyId, int taskId, Long timeStamp) { @Override public Iterator taskIterator(int studyId, List statusFilter, boolean reversed) { - if (statusFilter == null) { + if (statusFilter == null || statusFilter.isEmpty()) { return iterator(getTaskRowKeyPrefix(studyId), TaskMetadata.class, reversed); } else if (statusFilter.contains(TaskMetadata.Status.READY)) { EnumSet set = EnumSet.copyOf(statusFilter); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/SampleIndexTableRecordReader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/SampleIndexTableRecordReader.java index 2efd1115846..7e747813b0e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/SampleIndexTableRecordReader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/SampleIndexTableRecordReader.java @@ -7,6 +7,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableRecordReader; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.opencb.biodata.models.core.Region; @@ -186,9 +187,9 @@ public void initialize(InputSplit inputsplit, TaskAttemptContext context) throws startChr = null; start = 0; } else { - Variant startVariant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(firstRow); - startChr = startVariant.getChromosome(); - start = startVariant.getStart(); + Pair startLocus = VariantPhoenixKeyFactory.extractChrPosFromVariantRowKey(firstRow); + startChr = startLocus.getFirst(); + start = startLocus.getSecond(); } String stopChr; Integer end; @@ -197,9 +198,9 @@ public void initialize(InputSplit inputsplit, TaskAttemptContext context) throws stopChr = null; end = Integer.MAX_VALUE; } else { - Variant stopVariant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(lastRow); - stopChr = stopVariant.getChromosome(); - end = stopVariant.getStart(); + Pair stopLocus = VariantPhoenixKeyFactory.extractChrPosFromVariantRowKey(lastRow); + stopChr = stopLocus.getFirst(); + end = stopLocus.getSecond(); } List regions = new ArrayList<>(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantMapReduceUtil.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantMapReduceUtil.java index 623deec75da..367196742db 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantMapReduceUtil.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantMapReduceUtil.java @@ -29,11 +29,11 @@ import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; -import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; import org.opencb.opencga.storage.hadoop.utils.AbstractHBaseDriver; import org.opencb.opencga.storage.hadoop.variant.AbstractVariantsTableDriver; import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantQueryParser; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHBaseQueryParser; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; @@ -153,7 +153,7 @@ public static void initTableMapperJobFromPhoenix(Job job, String variantTable, S public static void initVariantMapperJob(Job job, Class mapperClass, String variantTable, VariantStorageMetadataManager metadataManager, Query query, QueryOptions queryOptions, boolean skipSampleIndex) throws IOException { - query = new VariantQueryParser(null, metadataManager).preProcessQuery(query, queryOptions); + query = new HadoopVariantQueryParser(null, metadataManager).preProcessQuery(query, queryOptions); setQuery(job, query); setQueryOptions(job, queryOptions); @@ -266,7 +266,7 @@ public static void initVariantRowMapperJob(Job job, Class mapperClass, String variantTable, VariantStorageMetadataManager metadataManager, Query query, QueryOptions queryOptions, boolean skipSampleIndex) throws IOException { - query = new VariantQueryParser(null, metadataManager).preProcessQuery(query, queryOptions); + query = new HadoopVariantQueryParser(null, metadataManager).preProcessQuery(query, queryOptions); setQuery(job, query); setQueryOptions(job, queryOptions); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java index a1b12db30fa..3daa349abb1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/mr/VariantTableHelper.java @@ -30,7 +30,7 @@ import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; -import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveDriver; +import org.opencb.opencga.storage.hadoop.variant.archive.ArchiveTableHelper; import org.opencb.opencga.storage.hadoop.variant.utils.HBaseVariantTableNameGenerator; import java.io.IOException; @@ -122,7 +122,7 @@ public static String getVariantsTable(Configuration conf) { } public static void setArchiveTable(Configuration conf, String archiveTable) { - conf.set(ArchiveDriver.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); + conf.set(ArchiveTableHelper.CONFIG_ARCHIVE_TABLE_NAME, archiveTable); } public HBaseVariantTableNameGenerator getHBaseVariantTableNameGenerator() { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsDescriptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsDescriptor.java index 152c0a86c3c..972f1aeecfe 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsDescriptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsDescriptor.java @@ -35,6 +35,13 @@ default boolean createTableIfNeeded(HBaseVariantTableNameGenerator generator, HB boolean createTableIfNeeded(String tableName, HBaseManager hBaseManager) throws IOException; + /** + * Configure the scan to read from the variants table. + * + * @param scan Scan to configure + * @param metadataManager Metadata manager + * @return The same scan object + */ Scan configureScan(Scan scan, VariantStorageMetadataManager metadataManager); Function getPendingEvaluatorMapper(VariantStorageMetadataManager metadataManager, boolean overwrite); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsReader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsReader.java index 3a58ebdb1bd..54d93c5c6ba 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsReader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/pending/PendingVariantsReader.java @@ -52,7 +52,7 @@ public boolean pre() { protected List convert(List results) { List variants = new ArrayList<>(results.size()); for (Result result : results) { - variants.add(VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(result.getRow())); + variants.add(VariantPhoenixKeyFactory.extractVariantFromResult(result)); } return variants; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneDriver.java index a2ff5f08b4a..fddaa7c189e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneDriver.java @@ -23,7 +23,6 @@ import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; import org.opencb.opencga.storage.hadoop.variant.AbstractVariantsTableDriver; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper; -import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.opencb.opencga.storage.hadoop.variant.converters.VariantRow; import org.opencb.opencga.storage.hadoop.variant.mr.VariantMapReduceUtil; @@ -37,6 +36,7 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static org.opencb.biodata.models.variant.StudyEntry.DEFAULT_COHORT; @@ -45,6 +45,7 @@ public class VariantPruneDriver extends AbstractVariantsTableDriver { private Logger logger = LoggerFactory.getLogger(VariantPruneManager.class); + public static final String ATTRIBUTE_DELETION_VARIANT = "d_variant"; public static final String ATTRIBUTE_DELETION_STUDIES = "d_studies"; public static final String ATTRIBUTE_DELETION_TYPE = "d_type"; public static final byte[] ATTRIBUTE_DELETION_TYPE_FULL = Bytes.toBytes(VariantPruneReportRecord.Type.FULL.toString()); @@ -95,6 +96,8 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); scan.setFilter(filterList); + scan.addColumn(COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + scan.addColumn(COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); scan.addColumn(COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.INDEX_NOT_SYNC.bytes()); scan.addColumn(COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.INDEX_UNKNOWN.bytes()); scan.addColumn(COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.INDEX_STUDIES.bytes()); @@ -210,7 +213,7 @@ protected void map(ImmutableBytesWritable key, Result value, Context context) List emptyStudies = new ArrayList<>(); List studies = new ArrayList<>(); List studiesWithStats = new ArrayList<>(); - + AtomicReference alleles = new AtomicReference<>(); Variant variant = variantRow.walker() .onStudy(studies::add) .onCohortStats(c -> { @@ -220,6 +223,7 @@ protected void map(ImmutableBytesWritable key, Result value, Context context) emptyStudies.add(c.getStudyId()); } }) + .onAlleles(alleles::set) .walk(); for (Integer studyWithStats : studiesWithStats) { @@ -253,13 +257,18 @@ protected void map(ImmutableBytesWritable key, Result value, Context context) // Drop variant && add to deleted variants list context.getCounter(COUNTER_GROUP_NAME, "variants_deleted").increment(1); - context.write(pendingDeletionVariantsTable, - new Put(value.getRow()).addColumn(COLUMN_FAMILY_BYTES, COLUMN, VALUE)); + Put put = new Put(value.getRow()) + .addColumn(COLUMN_FAMILY_BYTES, COLUMN, VALUE); + if (alleles.get() != null) { + put.addColumn(COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes(), Bytes.toBytes(alleles.get())); + } + context.write(pendingDeletionVariantsTable, put); context.write(pendingAnnotationVariantsTable, new Delete(value.getRow())); Delete delete = new Delete(value.getRow()); delete.addFamily(COLUMN_FAMILY_BYTES); + delete.setAttribute(ATTRIBUTE_DELETION_VARIANT, Bytes.toBytes(variant.toString())); delete.setAttribute(ATTRIBUTE_DELETION_TYPE, ATTRIBUTE_DELETION_TYPE_FULL); delete.setAttribute(ATTRIBUTE_DELETION_STUDIES, Bytes.toBytes(emptyStudies.stream() @@ -285,12 +294,16 @@ protected void map(ImmutableBytesWritable key, Result value, Context context) // This block is here to prevent accidental "full row" deletes. throw new IllegalStateException("Unexpected empty delete at partial variant prune in variant " + variant); } + delete.setAttribute(ATTRIBUTE_DELETION_VARIANT, Bytes.toBytes(variant.toString())); delete.setAttribute(ATTRIBUTE_DELETION_TYPE, ATTRIBUTE_DELETION_TYPE_PARTIAL); delete.setAttribute(ATTRIBUTE_DELETION_STUDIES, Bytes.toBytes(emptyStudies.stream().map(Object::toString).collect(Collectors.joining(",")))); Put updateSecondaryIndexColumns = new Put(value.getRow()); - + if (alleles.get() != null) { + updateSecondaryIndexColumns.addColumn(COLUMN_FAMILY_BYTES, + VariantPhoenixSchema.VariantColumn.ALLELES.bytes(), Bytes.toBytes(alleles.get())); + } HadoopVariantSearchIndexUtils.addNotSyncStatus(updateSecondaryIndexColumns); context.write(variantsTable, delete); @@ -360,8 +373,7 @@ public ReportRecordWriter(DataOutputStream out, ImmutableBytesWritable variantsT public synchronized void write(ImmutableBytesWritable key, Mutation mutation) throws IOException { if (mutation instanceof Delete && key.equals(variantsTable)) { - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(mutation.getRow()); - out.write(variant.toString().getBytes(StandardCharsets.UTF_8)); + out.write(mutation.getAttribute(ATTRIBUTE_DELETION_VARIANT)); out.write(SEPARATOR); out.write(mutation.getAttribute(ATTRIBUTE_DELETION_TYPE)); out.write(SEPARATOR); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneManager.java index 93d6801d448..43299e6e503 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/prune/VariantPruneManager.java @@ -205,7 +205,7 @@ private void checkReportedVariants(Path report, long count) throws IOException, Result[] get = table.get(gets); for (int i = 0; i < get.length; i++) { Result result = get[i]; - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(result.getRow()); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromResult(result); VariantPruneReportRecord record = batch.get(i); if (!variant.sameGenomicVariant(record.getVariant())) { throw new IllegalStateException("Error checking report! Expected " diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java index 59c55cf8dda..cc6323b2c0a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/search/SecondaryIndexPendingVariantsDescriptor.java @@ -54,6 +54,7 @@ public boolean createTableIfNeeded(String tableName, HBaseManager hBaseManager) @Override public Scan configureScan(Scan scan, VariantStorageMetadataManager metadataManager) { scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, ALLELES.bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, FULL_ANNOTATION.bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, ANNOTATION_ID.bytes()); scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, INDEX_NOT_SYNC.bytes()); @@ -61,7 +62,7 @@ public Scan configureScan(Scan scan, VariantStorageMetadataManager metadataManag scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, INDEX_STUDIES.bytes()); for (Integer studyId : metadataManager.getStudyIds()) { scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.getStudyColumn(studyId).bytes()); - for (CohortMetadata cohort : metadataManager.getCalculatedCohorts(studyId)) { + for (CohortMetadata cohort : metadataManager.getCalculatedOrPartialCohorts(studyId)) { scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.getStatsColumn(studyId, cohort.getId()).bytes()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/CheckVariantStatsDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/CheckVariantStatsDriver.java index 091a93eef54..c39ade7a754 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/CheckVariantStatsDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/CheckVariantStatsDriver.java @@ -369,7 +369,7 @@ protected void setup(Context context) throws IOException, InterruptedException { @Override protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException { - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(value.getRow()); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromResult(value); VariantType type = variant.getType(); int chromosomeIdx = FileStatsWritable.getChromosomeIdx(variant.getChromosome()); for (Cell cell : value.rawCells()) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/SaturationStatsDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/SaturationStatsDriver.java index 100f1f7f73d..0c509a0d7e7 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/SaturationStatsDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/SaturationStatsDriver.java @@ -106,6 +106,7 @@ protected Job setupJob(Job job, String archiveTable, String variantTable) throws maxSampleId += maxSampleIdInStudy; } scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); // scan.setFilter(new KeyOnlyFilter()); scan.setFilter( diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantHistogramDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantHistogramDriver.java index 306a59faf7b..7c4db8f70d3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantHistogramDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantHistogramDriver.java @@ -76,7 +76,8 @@ protected void parseAndValidateParameters() throws IOException { protected Job setupJob(Job job, String archiveTable, String variantTable) throws IOException { Scan scan = new Scan(); - + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); for (Integer studyId : getMetadataManager().getStudyIds()) { scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.getStudyColumn(studyId).bytes()); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsDriver.java index eb76771b159..c1dc34e2e21 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsDriver.java @@ -178,6 +178,7 @@ protected Job setupJob(Job job, String archiveTableName, String variantTableName // See #1600 // Add TYPE column to force scan ALL rows to avoid unlikely but possible timeouts fetching new variants scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.TYPE.bytes()); + scan.addColumn(GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes()); // Remove STUDY filter scan.setFilter(null); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsFromResultMapper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsFromResultMapper.java index 28b0115a326..7bc90517688 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsFromResultMapper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/stats/VariantStatsFromResultMapper.java @@ -165,7 +165,7 @@ public void run(Context context) throws IOException, InterruptedException { // } protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException { - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(value.getRow()); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromResult(value); VariantStatsWrapper wrapper = new VariantStatsWrapper(variant, new ArrayList<>(calculators.size())); calculators.forEach((cohort, calculator) -> { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java index b1bd9d84879..fbcbd773477 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSVTest.java @@ -4,23 +4,22 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; -import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.variant.VariantStorageEngineSVTest; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; -import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import static org.junit.Assert.assertEquals; +import static org.opencb.opencga.core.api.ParamConstants.OVERWRITE; /** * Created on 26/06/18. @@ -32,6 +31,7 @@ public class HadoopVariantStorageEngineSVTest extends VariantStorageEngineSVTest @ClassRule public static HadoopExternalResource externalResource = new HadoopExternalResource(); + private Logger logger = LoggerFactory.getLogger(getClass()); @Override protected void loadFiles() throws Exception { @@ -39,6 +39,12 @@ protected void loadFiles() throws Exception { VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); } + @Test + public void testRecreateSampleIndex() throws Exception { + variantStorageEngine.sampleIndex(studyMetadata.getName(), Collections.singletonList("all"), new QueryOptions(OVERWRITE, true)); + variantStorageEngine.sampleIndex(studyMetadata2.getName(), Collections.singletonList("all"), new QueryOptions(OVERWRITE, true)); + } + @Test public void checkSampleIndex() throws Exception { for (Variant variant : variantStorageEngine.iterable(new VariantQuery() @@ -46,21 +52,34 @@ public void checkSampleIndex() throws Exception { .includeSampleId(true) , new QueryOptions())) { Set samplesInVariant = new HashSet<>(); - for (String sample : metadataManager.getIndexedSamplesMap(studyMetadata.getId()).keySet()) { - QueryOptions options = new QueryOptions(VariantHadoopDBAdaptor.NATIVE, false); - VariantQueryResult result = variantStorageEngine.get(new Query() - .append(VariantQueryParam.SAMPLE.key(), sample) - .append(VariantQueryParam.ID.key(), variant.toString()), options); - String genotype = variant.getStudies().get(0).getSample(sample).getData().get(0); - if (GenotypeClass.MAIN_ALT.test(genotype)) { - Assert.assertNotNull(result.first()); - samplesInVariant.add(sample); - } else { - Assert.assertNull("Sample=" + sample + " with GT=" + genotype + " in variant=" + variant, result.first()); + for (Map.Entry entry : metadataManager.getStudies().entrySet()) { + String studyName = entry.getKey(); + Integer studyId = entry.getValue(); + StudyEntry studyEntry = variant.getStudy(studyName); + if (studyEntry != null) { + for (String sample : metadataManager.getIndexedSamplesMap(studyId).keySet()) { + QueryOptions options = new QueryOptions(VariantHadoopDBAdaptor.NATIVE, false); + VariantQueryResult result = variantStorageEngine.get(new VariantQuery() + .study(studyName) + .id(variant.toString()) + .sample(sample), options); + String genotype = studyEntry.getSample(sample).getData().get(0); + String message = "Study=" + studyName + " Sample=" + sample + " with GT=" + genotype + " in variant=" + variant; + if (GenotypeClass.MAIN_ALT.test(genotype)) { + Assert.assertNotNull(message, result.first()); + samplesInVariant.add(sample); + } else { + Assert.assertNull(message, result.first()); + } + } + logger.info("Variant " + variant + " with samples " + samplesInVariant); + logger.info("Query variant " + variant + " in study " + studyName + " from sampleData"); + Variant sampleDataVariant = variantStorageEngine.getSampleData(variant.toString(), studyName, new QueryOptions()).first(); + List actualSampleNames = sampleDataVariant.getSampleNames(studyName); + logger.info("Variant " + variant + " with actual samples " + actualSampleNames); + Assert.assertEquals(samplesInVariant, new HashSet<>(actualSampleNames)); } } - List actualSampleNames = variantStorageEngine.getSampleData(variant.toString(), studyMetadata.getName(), new QueryOptions()).first().getSampleNames(studyMetadata.getName()); - Assert.assertEquals(samplesInVariant, new HashSet<>(actualSampleNames)); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java index 7ef850e2063..6d111878d57 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java @@ -2,8 +2,11 @@ import htsjdk.variant.vcf.VCFConstants; import org.apache.commons.lang3.StringUtils; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.*; import org.junit.experimental.categories.Category; +import org.mockito.Mockito; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.FileEntry; @@ -13,6 +16,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -24,8 +28,10 @@ import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.core.variant.VariantStoragePipeline; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.annotation.annotators.CellBaseRestVariantAnnotator; @@ -50,6 +56,7 @@ public class HadoopVariantStorageEngineSplitDataTest extends VariantStorageBaseTest implements HadoopVariantStorageTest { public static final List SAMPLES = Arrays.asList("NA19600", "NA19660", "NA19661", "NA19685"); + public static final String MOCKED_EXCEPTION = "Mocked exception"; @ClassRule public static HadoopExternalResource externalResource = new HadoopExternalResource(); @@ -63,6 +70,91 @@ public void tearDown() throws Exception { VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); } + @Test + public void testMultiChromosomeSplitDataConcurrentFail() throws Exception { + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + failAtLoadingFile("by_chr/", "chr20.variant-test-file.vcf.gz", outputUri); + // Will fail if LOAD_SPLIT_DATA is not set + thrown.expect(StoragePipelineException.class); + variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), outputUri); + } + + @Test + public void testMultiChromosomeSplitDataConcurrentFailOneIndexOther() throws Exception { + // Test goal: + // - Index chr20 and chr21 concurrently with shared samples and LOAD_SPLIT_DATA=CHROMOSOME + // Test steps: + // - Fail at loading chr20 (left the file in status "RUNNING") + // - Index chr21 correctly with LOAD_SPLIT_DATA=CHROMOSOME + + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + failAtLoadingFile("by_chr/", "chr20.variant-test-file.vcf.gz", outputUri); + + // Won't fail if LOAD_SPLIT_DATA is set correctly + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); + variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), outputUri); + } + + @Test + public void testMultiChromosomeSplitDataConcurrentFail3() throws Exception { + // Test goal: + // - Ensure file can be loaded after deleting a file with shared samples without any LOAD_SPLIT_DATA. + // Test steps: + // - Fail at loading chr20 (left the file in status "RUNNING") + // - Force remove chr20 + // - Index chr21 correctly + + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + failAtLoadingFile("by_chr/", "chr20.variant-test-file.vcf.gz", outputUri); + + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), true); + variantStorageEngine.removeFile(STUDY_NAME, "chr20.variant-test-file.vcf.gz", outputUri); + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), false); + + variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), outputUri); + } + + @Test + public void testMultiChromosomeSplitDataConcurrentFailDelete() throws Exception { + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + failAtLoadingFile("by_chr/", "chr20.variant-test-file.vcf.gz", outputUri); + + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); + variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), outputUri); + + try { + // FORCE=true is needed, as the file is not correctly indexed + // Set FORCE=false to assert exception + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), false); + variantStorageEngine.removeFile(STUDY_NAME, "chr20.variant-test-file.vcf.gz", outputUri); + fail(); + } catch (StorageEngineException e) { + assertEquals("Unable to remove non indexed file: chr20.variant-test-file.vcf.gz", e.getMessage()); + } + + // FORCE=true is needed, as the file is not correctly indexed + failAtDeletingFile("chr20.variant-test-file.vcf.gz", outputUri, 1, new ObjectMap(VariantStorageOptions.FORCE.key(), true)); + + try { + // FORCE=true is needed, as the file is not correctly indexed + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), true); + // RESUME=true is needed, as the delete task is in status "RUNNING" from the previous failAtDeletingFile + // Set RESUME=false to assert exception + variantStorageEngine.getOptions().put(VariantStorageOptions.RESUME.key(), false); + variantStorageEngine.removeFile(STUDY_NAME, "chr20.variant-test-file.vcf.gz", outputUri); + fail(); + } catch (StorageEngineException e) { + assertEquals("Operation \"remove\" for files [\"chr20.variant-test-file.vcf.gz\" (id=1)] in status \"RUNNING\". Relaunch with resume=true to finish the operation.", e.getMessage()); + } + + + // FORCE=true is needed, as the file is not correctly indexed + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), true); + // RESUME=true is needed, as the delete task is in status "RUNNING" + variantStorageEngine.getOptions().put(VariantStorageOptions.RESUME.key(), true); + variantStorageEngine.removeFile(STUDY_NAME, "chr20.variant-test-file.vcf.gz", outputUri); + } + @Test public void testMultiChromosomeSplitData() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); @@ -131,6 +223,63 @@ public void testMultiChromosomeSplitData() throws Exception { }, QueryOptions.empty()); } + private void failAtLoadingFile(String x, String file1, URI outputUri) throws Exception { + failAtLoadingFile(x, file1, outputUri, 1); + } + + private void failAtLoadingFile(String x, String file1, URI outputUri, int expectedRunningTasks) throws Exception { + try { + VariantStorageEngine engine = getMockedStorageEngine(new ObjectMap(VariantStorageOptions.STUDY.key(), STUDY_NAME)); + engine.index(Collections.singletonList(getResourceUri(x + file1)), outputUri); + fail("Should have thrown an exception"); + } catch (StoragePipelineException e) { + try { + assertEquals(MOCKED_EXCEPTION, e.getCause().getMessage()); + int studyId = metadataManager.getStudyId(STUDY_NAME); + FileMetadata fileMetadata = metadataManager.getFileMetadata(studyId, file1); + assertEquals(TaskMetadata.Status.NONE, fileMetadata.getIndexStatus()); + List runningTasks = new ArrayList<>(); + metadataManager.getRunningTasks(studyId).forEach(runningTasks::add); + assertEquals(expectedRunningTasks, runningTasks.size()); + TaskMetadata taskMetadata = runningTasks.get(runningTasks.size() - 1); + assertEquals(TaskMetadata.Type.LOAD, taskMetadata.getType()); + assertEquals(TaskMetadata.Status.RUNNING, taskMetadata.currentStatus()); + assertEquals(Arrays.asList(fileMetadata.getId()), taskMetadata.getFileIds()); + } catch (AssertionError error) { + error.addSuppressed(e); + e.printStackTrace(); + throw error; + } + } + } + + private void failAtDeletingFile(String file, URI outputUri, int expectedRunningTasks, ObjectMap options) throws Exception { + int studyId = metadataManager.getStudyId(STUDY_NAME); + FileMetadata fileMetadata = metadataManager.getFileMetadata(studyId, file); + try { + getMockedStorageEngine(options).removeFile(STUDY_NAME, file, outputUri); + fail("Should have thrown an exception"); + } catch (StorageEngineException e) { + try { + assertEquals(MOCKED_EXCEPTION, e.getMessage()); + assertEquals(TaskMetadata.Status.NONE, fileMetadata.getIndexStatus()); + List runningTasks = new ArrayList<>(); + metadataManager.getRunningTasks(studyId).forEach(runningTasks::add); + assertEquals(expectedRunningTasks, runningTasks.size()); + Optional optional = runningTasks.stream() + .filter(t -> t.getType() == TaskMetadata.Type.REMOVE && Arrays.asList(fileMetadata.getId()).equals(t.getFileIds())) + .findFirst(); + assertTrue(optional.isPresent()); + assertEquals(TaskMetadata.Type.REMOVE, optional.get().getType()); + assertEquals(TaskMetadata.Status.RUNNING, optional.get().currentStatus()); + assertEquals(Arrays.asList(fileMetadata.getId()), optional.get().getFileIds()); + } catch (AssertionError error) { + e.printStackTrace(); + throw error; + } + } + } + @Test public void testMultiChromosomeSplitDataVirtualFile() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); @@ -407,6 +556,226 @@ private void checkIssueEntries_22_44681612_A_G(Variant v) { } + @Test + public void testLoadMultiFileDataConcurrency() throws Exception { + + URI outDir = newOutputUri(); + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), false); + variantStorageEngine.getOptions().put(VariantStorageOptions.FAMILY.key(), true); + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + + String resourceDir = "by_chr/"; + String file1 = "chr22.variant-test-file.vcf.gz"; + String file2 = "chr22_1-2-DUP.variant-test-file.vcf.gz"; + + failAtLoadingFile(resourceDir, file1, outDir); + + try { + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file2)), outDir); + } catch (StoragePipelineException e) { + MatcherAssert.assertThat(e.getCause().getMessage(), startsWith("Can not \"Load\" files")); + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString(file2)); + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString(file1)); + } + + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), true); + variantStorageEngine.removeFile(STUDY_NAME, file1, outDir); + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file2)), outDir); + + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), true); +// variantStorageEngine.getOptions().put(VariantStorageOptions.RESUME.key(), true); + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir); + } + + + @Test + public void testLoadMultiFileDataConcurrencyDeleteMany() throws Exception { + + URI outDir = newOutputUri(); + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), false); + variantStorageEngine.getOptions().put(VariantStorageOptions.FAMILY.key(), true); + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + + String resourceDir = "platinum/"; + String file1 = "1K.end.platinum-genomes-vcf-NA12877_S1.vcf.gz"; + String file2 = "1K.end.platinum-genomes-vcf-NA12878_S1.vcf.gz"; + + failAtLoadingFile(resourceDir, file1, outDir); + failAtLoadingFile(resourceDir, file2, outDir, 2); +// try { +// getMockedStorageEngine().index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir); +// fail("Should have thrown an exception"); +// } catch (StoragePipelineException e) { +// assertEquals(MOCKED_EXCEPTION, e.getCause().getMessage()); +// } +// try { +// getMockedStorageEngine().index(Collections.singletonList(getResourceUri(resourceDir + file2)), outDir); +// fail("Should have thrown an exception"); +// } catch (StoragePipelineException e) { +// assertEquals(MOCKED_EXCEPTION, e.getCause().getMessage()); +// } + + + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), true); + try { + variantStorageEngine.removeFile(STUDY_NAME, file1, outDir); + fail(); + } catch (StorageEngineException e) { + MatcherAssert.assertThat(e.getMessage(), startsWith("Can not \"remove\" files")); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString(file1)); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString(file2)); + } + try { + variantStorageEngine.removeFile(STUDY_NAME, file2, outDir); + fail(); + } catch (StorageEngineException e) { + MatcherAssert.assertThat(e.getMessage(), startsWith("Can not \"remove\" files")); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString(file1)); + MatcherAssert.assertThat(e.getMessage(), CoreMatchers.containsString(file2)); + } + variantStorageEngine.removeFiles(STUDY_NAME, Arrays.asList(file1, file2), outDir); + + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file2)), outDir); + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir); + } + + @Test + public void testLoadMultiFileDataConcurrencyFail() throws Exception { + + URI outDir = newOutputUri(); + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), false); + variantStorageEngine.getOptions().put(VariantStorageOptions.FAMILY.key(), true); + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + + String resourceDir = "by_chr/"; + String file1 = "chr22.variant-test-file.vcf.gz"; + String file2 = "chr22_1-2-DUP.variant-test-file.vcf.gz"; + + failAtLoadingFile(resourceDir, file1, outDir); + + try { + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file2)), outDir); + } catch (StoragePipelineException e) { + MatcherAssert.assertThat(e.getCause().getMessage(), startsWith("Can not \"Load\" files")); + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString(file2)); + MatcherAssert.assertThat(e.getCause().getMessage(), CoreMatchers.containsString(file1)); + } + + variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), true); + variantStorageEngine.getOptions().put(VariantStorageOptions.RESUME.key(), true); + variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir); + + } + + private VariantStorageEngine getMockedStorageEngine() throws Exception { + return getMockedStorageEngine(new ObjectMap()); + } + + private VariantStorageEngine getMockedStorageEngine(ObjectMap options) throws Exception { + HadoopVariantStorageEngine mockedStorageEngine = Mockito.spy(getVariantStorageEngine()); + mockedStorageEngine.getOptions().putAll(options); + mockedStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + VariantStoragePipeline mockedPipeline = Mockito.spy(variantStorageEngine.newStoragePipeline(true)); + + Mockito.doReturn(mockedPipeline).when(mockedStorageEngine).newStoragePipeline(Mockito.anyBoolean()); +// Mockito.doThrow(new StoragePipelineException(MOCKED_EXCEPTION, Collections.emptyList())).when(mockedPipeline).load(Mockito.any(), Mockito.any()); + Mockito.doAnswer(invocation -> { + // Throw StorageEngineException when calling load + System.out.printf("MOCKED load(%s, %s)%n", invocation.getArgument(0), invocation.getArgument(1)); + System.out.println("MOCKED load throw StorageEngineException"); + throw new StoragePipelineException(MOCKED_EXCEPTION, Collections.emptyList()); + }).when(mockedPipeline).load(Mockito.any(), Mockito.any()); + + Mockito.doAnswer(invocation -> { + // Call real method when calling preRemove, then throw StorageEngineException + System.out.printf("MOCKED preRemove(%s, %s, %s)%n", invocation.getArgument(0), invocation.getArgument(1), invocation.getArgument(2)); + System.out.println("MOCKED preRemove callRealMethod"); + invocation.callRealMethod(); + System.out.println("MOCKED preRemove callRealMethod DONE"); + System.out.println("MOCKED preRemove throw StorageEngineException"); + throw new StorageEngineException(MOCKED_EXCEPTION); + }).when(mockedStorageEngine).preRemove(Mockito.any(), Mockito.any(), Mockito.any()); + return mockedStorageEngine; + } + + @Test + public void testDeleteErrorFiles() throws Exception { + URI outDir = newOutputUri(); + + VariantStorageMetadataManager mm = variantStorageEngine.getMetadataManager(); + + variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); + URI file = variantStorageEngine.index(Collections.singletonList(getPlatinumFile(1)), outDir).get(0).getInput(); + String fileName = UriUtils.fileName(file); + + int studyId = mm.getStudyId(STUDY_NAME); + int fileId = mm.getFileId(studyId, fileName); + FileMetadata fileMetadata = mm.updateFileMetadata(studyId, fileId, fm -> { + fm.setIndexStatus(TaskMetadata.Status.INVALID); + }); + assertFalse(mm.isFileIndexed(studyId, fileId)); + assertFalse(fileMetadata.isIndexed()); + LinkedHashSet samples = fileMetadata.getSamples(); + + for (Integer sample : samples) { + mm.updateSampleMetadata(studyId, sample, sampleMetadata -> { + sampleMetadata.setIndexStatus(TaskMetadata.Status.INVALID); + }); + } + + try { + variantStorageEngine.get(new VariantQuery().file(fileName), new QueryOptions()); + fail(); + } catch (VariantQueryException e) { + String expected = VariantQueryException.fileNotIndexed(fileName, STUDY_NAME).getMessage(); + assertEquals(expected, e.getMessage()); + } + + try { + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), true); + variantStorageEngine.index(Collections.singletonList(getPlatinumFile(1)), outDir); + fail(); + } catch (StorageEngineException e) { + try { + String expected = StorageEngineException.invalidFileStatus(fileId, fileName).getMessage(); + assertEquals(expected, e.getCause().getMessage()); + } catch (AssertionError error) { + e.printStackTrace(); + throw error; + } + } + + try { + variantStorageEngine.getOptions().put(VariantStorageOptions.FORCE.key(), false); + variantStorageEngine.index(Collections.singletonList(getPlatinumFile(1)), outDir); + fail(); + } catch (StorageEngineException e) { + try { + String expected = StorageEngineException.invalidFileStatus(fileId, fileName).getMessage(); + assertEquals(expected, e.getCause().getMessage()); + } catch (AssertionError error) { + e.printStackTrace(); + throw error; + } + } + + variantStorageEngine.removeFile(STUDY_NAME, fileName, outDir); + + fileMetadata = mm.getFileMetadata(studyId, fileId); + assertEquals(TaskMetadata.Status.NONE, fileMetadata.getIndexStatus()); + for (Integer sample : samples) { + assertEquals(TaskMetadata.Status.NONE, mm.getSampleMetadata(studyId, sample).getIndexStatus()); + } + + variantStorageEngine.index(Collections.singletonList(getPlatinumFile(1)), outDir); + + fileMetadata = mm.getFileMetadata(studyId, fileId); + assertEquals(TaskMetadata.Status.READY, fileMetadata.getIndexStatus()); + for (Integer sample : samples) { + assertEquals(TaskMetadata.Status.READY, mm.getSampleMetadata(studyId, sample).getIndexStatus()); + } + } + @Test public void testLoadByRegion() throws Exception { URI outDir = newOutputUri(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java index 787ea7c4412..d11635abe6d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java @@ -534,7 +534,7 @@ public static void setStaticConfiguration(Configuration staticConfiguration) { @Override - public int run(Class clazz, String[] args) throws StorageEngineException { + public Result run(Class clazz, String[] args) throws StorageEngineException { try { // Copy configuration Configuration conf = new Configuration(false); @@ -547,14 +547,14 @@ public int run(Class clazz, String[] args) throws StorageEng if (((Number) o).intValue() != 0) { throw new StorageEngineException("Error executing MapReduce. Exit code: " + o); } - return ((Number) o).intValue(); + return new Result(((Number) o).intValue(), new ObjectMap()); } catch (Exception e) { throw new StorageEngineException("Error executing MapReduce.", e); } } @Override - public int run(String executable, String[] args) { + public Result run(String executable, String[] args) { try { // Copy configuration Configuration conf = new Configuration(false); @@ -569,7 +569,7 @@ public int run(String executable, String[] args) { if (((Number) o).intValue() != 0) { throw new RuntimeException("Exit code = " + o); } - return ((Number) o).intValue(); + return new Result(((Number) o).intValue(), new ObjectMap()); } catch (Exception e) { // e.printStackTrace(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopStoragePipelineTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopStoragePipelineTest.java index 7abd5c71449..773f017a67c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopStoragePipelineTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopStoragePipelineTest.java @@ -207,7 +207,7 @@ public void checkVariantTable() throws IOException { int num = 0; ResultScanner resultScanner = table.getScanner(GenomeHelper.COLUMN_FAMILY_BYTES); for (Result result : resultScanner) { - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(result.getRow()); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromResult(result); System.out.println("Variant = " + variant); num++; } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java index 3557c5e55ff..8da9d0911f0 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHbaseTestUtils.java @@ -46,6 +46,7 @@ import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; @@ -64,6 +65,7 @@ import java.io.*; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -72,7 +74,6 @@ import java.util.zip.DataFormatException; import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getTmpRootDir; -import static org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest.configuration; /** * Utility class for VariantStorage hadoop tests @@ -159,7 +160,7 @@ public static void printVariantsFromVariantsTable(VariantHadoopDBAdaptor dbAdapt for (Result result : resultScanner) { Variant variant; try { - variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(result.getRow()); + variant = VariantPhoenixKeyFactory.extractVariantFromResult(result); } catch (RuntimeException e) { os.println(Arrays.toString(result.getRow())); os.println("--------------------"); @@ -236,8 +237,11 @@ private static void printVariantsFromDBAdaptor(VariantHadoopDBAdaptor dbAdaptor, } private static void printVariantsFromDBAdaptor(VariantHadoopDBAdaptor dbAdaptor, PrintStream out) { - VariantDBIterator iterator = dbAdaptor.iterator(new Query(VariantQueryParam.INCLUDE_SAMPLE_DATA.key(), "all,SAMPLE_ID") - .append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL), + if (dbAdaptor.getMetadataManager().getStudyIds().isEmpty()) { + out.println("No studies found!"); + return; + } + VariantDBIterator iterator = dbAdaptor.iterator(new VariantQuery().includeSampleId(true).includeSampleAll(), new QueryOptions("simpleGenotypes", true)); ObjectMapper mapper = new ObjectMapper().configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true); while (iterator.hasNext()) { @@ -406,7 +410,7 @@ public static void printVariants(Collection studies, VariantHadoo } private static void printVcf(StudyMetadata studyMetadata, VariantHadoopDBAdaptor dbAdaptor, Path outDir) throws IOException { - try (OutputStream os = new FileOutputStream(outDir.resolve("variant." + studyMetadata.getName() + ".vcf").toFile())) { + try (OutputStream os = Files.newOutputStream(outDir.resolve("variant." + studyMetadata.getName() + ".vcf"))) { Query query = new Query(VariantQueryParam.STUDY.key(), studyMetadata.getName()) .append(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL) .append(VariantQueryParam.UNKNOWN_GENOTYPE.key(), "."); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java index be47dd08890..bebcb5ce4da 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java @@ -10,8 +10,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorNativeTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorNativeTest.java index 7d6b6f19ba5..e03253739d3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorNativeTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorNativeTest.java @@ -9,7 +9,7 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java index 18798990bf1..05b6d81a5b3 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixKeyFactoryTest.java @@ -1,5 +1,9 @@ package org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; @@ -11,7 +15,9 @@ import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.StructuralVariation; import org.opencb.opencga.core.testclassification.duration.ShortTests; +import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.opencb.opencga.storage.hadoop.HBaseCompat; import java.sql.SQLException; @@ -36,6 +42,7 @@ public void testVariantRowKey() throws Exception { checkVariantRowKeyGeneration(new Variant("5", 21648, "A", "T")); checkVariantRowKeyGeneration(new Variant("5", 21648, "AAAAAA", "T")); checkVariantRowKeyGeneration(new Variant("5", 21648, "A", "")); + checkVariantRowKeyGeneration(new Variant("5", 21648, "", "T")); checkVariantRowKeyGeneration(new Variant("5", 21648, "AAT", "TTT")); checkVariantRowKeyGeneration(new Variant("X", 21648, "", "TTT")); checkVariantRowKeyGeneration(new Variant("MT", 21648, "", "")); @@ -54,6 +61,16 @@ public void testStructuralVariantRowKey() throws Exception { checkVariantRowKeyGeneration(new Variant("5:100:A:A]:chr5:234]")); } + @Test + public void testExtraLargeVariantRowKey() throws Exception { + String allele1 = RandomStringUtils.random(50000, "ACGT"); + checkVariantRowKeyGeneration(new Variant("5:1000:-:" + allele1)); + StructuralVariation sv = new StructuralVariation(); + sv.setLeftSvInsSeq(allele1); + sv.setRightSvInsSeq(allele1); + checkVariantRowKeyGeneration(new Variant("5:1000:A:").setSv(sv)); + } + @Test public void testExtractChrPosFromVariantRowKeyPartial() { byte[] phoenixRowKey = VariantPhoenixKeyFactory.generateVariantRowKey("1", 20 << 16); @@ -73,11 +90,18 @@ public void checkVariantRowKeyGeneration(Variant variant) { // System.out.println("expected = " + Bytes.toStringBinary(phoenixRowKey)); byte[] variantRowkey = VariantPhoenixKeyFactory.generateVariantRowKey(variant); + byte[] alleles = Bytes.toBytes(VariantPhoenixKeyFactory.buildAlleles(variant)); // System.out.println("actual = " + Bytes.toStringBinary(variantRowkey)); - Variant generatedVariant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(variantRowkey); + Variant generatedVariant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(variantRowkey, null, alleles); + + Result result = Result.create(Collections.singletonList(new KeyValue(phoenixRowKey, GenomeHelper.COLUMN_FAMILY_BYTES, VariantPhoenixSchema.VariantColumn.ALLELES.bytes(), + alleles))); + + Variant generatedVariant2 = VariantPhoenixKeyFactory.extractVariantFromResult(result); assertArrayEquals(variant.toString(), phoenixRowKey, variantRowkey); assertEquals(variant, generatedVariant); + assertEquals(variant, generatedVariant2); } public byte[] generateVariantRowKeyPhoenix(Variant variant) { @@ -104,12 +128,23 @@ public byte[] generateVariantRowKeyPhoenix(Variant variant) { } ImmutableBytesWritable key = new ImmutableBytesWritable(); + String reference = variant.getReference(); + String alternate = VariantPhoenixKeyFactory.buildSymbolicAlternate(reference, variant.getAlternate(), variant.getEnd(), variant.getSv()); table.newKey(key, new byte[][]{ Bytes.toBytes(variant.getChromosome()), Bytes.toBytes(variant.getStart()), - Bytes.toBytes(variant.getReference()), - Bytes.toBytes(VariantPhoenixKeyFactory.buildSymbolicAlternate(variant.getReference(), variant.getAlternate(), variant.getEnd(), variant.getSv())), + Bytes.toBytes(reference), + Bytes.toBytes(alternate), }); + if (key.getLength() > HConstants.MAX_ROW_LENGTH) { + key = new ImmutableBytesWritable(); + table.newKey(key, new byte[][]{ + Bytes.toBytes(variant.getChromosome()), + Bytes.toBytes(variant.getStart()), + Bytes.toBytes(VariantPhoenixKeyFactory.hashAllele(reference)), + Bytes.toBytes(VariantPhoenixKeyFactory.hashAllele(alternate)), + }); + } if (key.getLength() == key.get().length) { return key.get(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java index 3ae2a36bfbc..fdb5d42f1b9 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java @@ -18,7 +18,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.models.operations.variant.VariantAggregateParams; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; @@ -356,8 +356,7 @@ private void checkInputValuesAreUnmodified(String aggregatedStudy, String refere dbAdaptor.getHBaseManager().act(dbAdaptor.getVariantTable(), table -> { table.getScanner(new Scan()).iterator().forEachRemaining(r -> { - byte[] row = r.getRow(); - Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(row); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromResult(r); NavigableMap cells = r.getFamilyMap(GenomeHelper.COLUMN_FAMILY_BYTES); for (Map.Entry entry : cells.entrySet()) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index ccafd14be63..f67d5734d7c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -16,7 +16,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java index 27f8a56f317..dc87e7d6d11 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDBAdaptorTest.java @@ -46,7 +46,7 @@ public void testSampleIdFF() throws Exception { SampleIndexQuery query = new SampleIndexQuery(SampleIndexSchema.defaultSampleIndexSchema(), Collections.emptyList(), 0, null, "ST", Collections.singletonMap(sampleName, Collections.singletonList("0/1")), Collections.emptySet(), null, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), new SampleAnnotationIndexQuery(SampleIndexSchema.defaultSampleIndexSchema()), - Collections.emptySet(), null, false, VariantQueryUtils.QueryOperation.AND); + Collections.emptySet(), null, false, VariantQueryUtils.QueryOperation.AND, null); new SampleIndexDBAdaptor(new HBaseManager(new Configuration()), new HBaseVariantTableNameGenerator("default", "my_dbname"), metadataManager) .parse(query.forSample(sampleName), null); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDuplicatedVariantsTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDuplicatedVariantsTest.java index 6a9c35eb0da..be83e365bed 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDuplicatedVariantsTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexDuplicatedVariantsTest.java @@ -63,7 +63,7 @@ public void test2FilesSampleIndex() throws Exception { SampleIndexOnlyVariantQueryExecutor queryExecutor = new SampleIndexOnlyVariantQueryExecutor(dbAdaptor, sampleIndexDBAdaptor, "", new ObjectMap()); List expectedVariants = new ArrayList<>(); - queryExecutor.iterator(new VariantQuery().sample("s1"), new QueryOptions()).forEachRemaining(expectedVariants::add); + queryExecutor.iterator(variantStorageEngine.parseQuery(new VariantQuery().sample("s1"), new QueryOptions())).forEachRemaining(expectedVariants::add); int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); String actualSampleIndexTableName = sampleIndexDBAdaptor.getSampleIndexTableNameLatest(studyId); @@ -81,7 +81,7 @@ public void test2FilesSampleIndex() throws Exception { VariantHbaseTestUtils.printVariants(dbAdaptor, newOutputUri()); List actualVariants = new ArrayList<>(); - queryExecutor.iterator(new VariantQuery().sample("s1"), new QueryOptions()).forEachRemaining(actualVariants::add); + queryExecutor.iterator(variantStorageEngine.parseQuery(new VariantQuery().sample("s1"), new QueryOptions())).forEachRemaining(actualVariants::add); Assert.assertEquals(expectedVariants, actualVariants); @@ -105,7 +105,7 @@ public void test3FilesSampleIndex() throws Exception { SampleIndexOnlyVariantQueryExecutor queryExecutor = new SampleIndexOnlyVariantQueryExecutor(dbAdaptor, sampleIndexDBAdaptor, "", new ObjectMap()); List expectedVariants = new ArrayList<>(); - queryExecutor.iterator(new VariantQuery().sample("s1"), new QueryOptions()).forEachRemaining(expectedVariants::add); + queryExecutor.iterator(variantStorageEngine.parseQuery(new VariantQuery().sample("s1"), new QueryOptions())).forEachRemaining(expectedVariants::add); int studyId = engine.getMetadataManager().getStudyId(STUDY_NAME); String actualSampleIndexTableName = sampleIndexDBAdaptor.getSampleIndexTableNameLatest(studyId); @@ -123,7 +123,7 @@ public void test3FilesSampleIndex() throws Exception { VariantHbaseTestUtils.printVariants(dbAdaptor, newOutputUri()); List actualVariants = new ArrayList<>(); - queryExecutor.iterator(new VariantQuery().sample("s1"), new QueryOptions()).forEachRemaining(actualVariants::add); + queryExecutor.iterator(variantStorageEngine.parseQuery(new VariantQuery().sample("s1"), new QueryOptions())).forEachRemaining(actualVariants::add); Assert.assertEquals(expectedVariants, actualVariants); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java index 7933889d93f..a631c07e099 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexEntryFilterTest.java @@ -280,7 +280,7 @@ private SingleSampleIndexQuery getSingleSampleIndexQuery(SampleAnnotationIndexQu private SingleSampleIndexQuery getSingleSampleIndexQuery(SampleAnnotationIndexQuery annotationIndexQuery, Map> fileFilterMap) { return new SampleIndexQuery( - schema, Collections.emptyList(), 0, null, "study", Collections.singletonMap("S1", Arrays.asList("0/1", "1/1")), Collections.emptySet(), null, Collections.emptyMap(), Collections.emptyMap(), fileFilterMap, annotationIndexQuery, Collections.emptySet(), null, false, VariantQueryUtils.QueryOperation.AND) + schema, Collections.emptyList(), 0, null, "study", Collections.singletonMap("S1", Arrays.asList("0/1", "1/1")), Collections.emptySet(), null, Collections.emptyMap(), Collections.emptyMap(), fileFilterMap, annotationIndexQuery, Collections.emptySet(), null, false, VariantQueryUtils.QueryOperation.AND, null) .forSample("S1"); } } \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java index fa1cc7ffe8e..99cf76e71cc 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParserTest.java @@ -26,8 +26,8 @@ import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageMetadataDBAdaptorFactory; import org.opencb.opencga.storage.core.variant.query.Values; -import org.opencb.opencga.storage.core.variant.query.VariantQueryParser; import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantQueryParser; import org.opencb.opencga.storage.hadoop.variant.index.annotation.AnnotationIndexConverter; import org.opencb.opencga.storage.hadoop.variant.index.core.IndexField; import org.opencb.opencga.storage.hadoop.variant.index.core.RangeIndexField; @@ -127,7 +127,7 @@ public void setUp() throws Exception { } private SampleIndexQuery parse(final Query query) { - Query newQuery = new VariantQueryParser(null, mm).preProcessQuery(query, new QueryOptions()); + Query newQuery = new HadoopVariantQueryParser(null, mm).preProcessQuery(query, new QueryOptions()); query.clear(); query.putAll(newQuery); return sampleIndexQueryParser.parse(query); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 6267f8d8e23..55d9d98a0f8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -26,7 +26,7 @@ import org.opencb.opencga.core.config.storage.IndexFieldConfiguration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.models.variant.VariantAnnotationConstants; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; @@ -369,8 +369,8 @@ public void regenerateSampleIndex() throws Exception { } public VariantQueryResult dbAdaptorQuery(Query query, QueryOptions options) { - query = variantStorageEngine.preProcessQuery(query, options); - return dbAdaptor.get(query, options); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); + return dbAdaptor.get(variantQuery); } @Test @@ -463,26 +463,26 @@ public void testLocusQueryOverlap() throws Exception { VariantQuery query = new VariantQuery().study(STUDY_NAME_5).sample("NA19600"); // System.out.println("query = " + query.toJson()); List variants = sampleIndexDBAdaptor.iterator(new Query(query), new QueryOptions()) - .toDataResult().getResults(); + .toList(); assertEquals(2, variants.size()); query.region("1:2000200-5500000"); // System.out.println("query = " + query.toJson()); variants = sampleIndexDBAdaptor.iterator(new Query(query), new QueryOptions()) - .toDataResult().getResults(); + .toList(); assertEquals(2, variants.size()); query.region("1:200-2500000"); // System.out.println("query = " + query.toJson()); variants = sampleIndexDBAdaptor.iterator(new Query(query), new QueryOptions()) - .toDataResult().getResults(); + .toList(); assertEquals(1, variants.size()); assertEquals("1:1000001-4000000:-:", variants.get(0).toString()); query.region("1:2000200-2500000"); // System.out.println("query = " + query.toJson()); variants = sampleIndexDBAdaptor.iterator(new Query(query), new QueryOptions()) - .toDataResult().getResults(); + .toList(); assertEquals(1, variants.size()); assertEquals("1:1000001-4000000:-:", variants.get(0).toString()); } @@ -668,10 +668,10 @@ public SampleIndexQuery testQueryIndex(Query testQuery, Query query, boolean sam SampleIndexQuery indexQuery = sampleIndexDBAdaptor.parseSampleIndexQuery(sampleIndexVariantQuery); // int onlyIndex = (int) ((HadoopVariantStorageEngine) variantStorageEngine).getSampleIndexDBAdaptor() // .count(indexQuery, "NA19600"); - DataResult result = ((HadoopVariantStorageEngine) variantStorageEngine).getSampleIndexDBAdaptor() - .iterator(indexQuery).toDataResult(); + List result = ((HadoopVariantStorageEngine) variantStorageEngine).getSampleIndexDBAdaptor() + .iterator(indexQuery).toList(); // System.out.println("result.getResults() = " + result.getResults()); - List onlyIndex = result.getResults().stream().map(Variant::toString).sorted().collect(toList()); + List onlyIndex = result.stream().map(Variant::toString).sorted().collect(toList()); // Query SampleIndex+DBAdaptor System.out.println("#Query SampleIndex+DBAdaptor"); @@ -815,7 +815,7 @@ public void testCount() throws StorageEngineException { System.out.println("Count = " + actualCount); stopWatch = StopWatch.createStarted(); - long actualCountIterator = sampleIndexDBAdaptor.iterator(sampleIndexDBAdaptor.parseSampleIndexQuery(new Query(query))).toDataResult().getNumResults(); + long actualCountIterator = sampleIndexDBAdaptor.iterator(sampleIndexDBAdaptor.parseSampleIndexQuery(new Query(query))).toList().size(); System.out.println("---"); System.out.println("Count indexTable iterator " + stopWatch.getTime(TimeUnit.MILLISECONDS) / 1000.0); System.out.println("Count = " + actualCountIterator); @@ -1200,24 +1200,24 @@ private void testSampleIndexOnlyVariantQueryExecutor(VariantQuery query, QueryOp private void testSampleIndexOnlyVariantQueryExecutor(VariantQuery query, QueryOptions options, Class expected, Function mapper) { - VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor( - query, - options); + ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); + + VariantQueryExecutor variantQueryExecutor = variantStorageEngine.getVariantQueryExecutor(variantQuery); assertEquals(expected, variantQueryExecutor.getClass()); - ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options); List expectedVariants = new ArrayList<>(1000); - dbAdaptor.iterator(variantQuery, new QueryOptions(options)) + dbAdaptor.iterator(variantQuery) .forEachRemaining(expectedVariants::add); List actualVariants = new ArrayList<>(1000); - variantQueryExecutor.iterator(variantQuery.getQuery(), options) + variantQueryExecutor.iterator(variantStorageEngine.parseQuery(variantQuery.getQuery(), options)) .forEachRemaining(actualVariants::add); - VariantQueryResult result = variantQueryExecutor.get(variantQuery.getQuery(), new QueryOptions(options) + ParsedVariantQuery limitedQuery = variantStorageEngine.parseQuery(variantQuery.getQuery(), new QueryOptions(options) .append(QueryOptions.LIMIT, 10) .append(QueryOptions.COUNT, true)); + VariantQueryResult result = variantQueryExecutor.get(limitedQuery); assertEquals(10, result.getNumResults()); assertEquals(10, result.getResults().size()); long count = result.getNumMatches(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java new file mode 100644 index 00000000000..aa62af0944f --- /dev/null +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java @@ -0,0 +1,37 @@ +package org.opencb.opencga.storage.hadoop.variant.query.executors; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.experimental.categories.Category; +import org.junit.rules.ExternalResource; +import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.variant.query.executors.VariantQueryExecutorTest; +import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; +import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils; + + +@Category(LongTests.class) +public class HadoopVariantQueryExecutorTest extends VariantQueryExecutorTest implements HadoopVariantStorageTest { + + @ClassRule + public static ExternalResource externalResource = new HadoopExternalResource(); + + @Override + public void initSolr() throws Exception { + if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) { + super.initSolr(); + } else { + System.out.println("Solr testing not available"); + } + } + + @Override + @Before + public void setUp() throws Exception { + boolean firstRun = !super.fileIndexed; + super.setUp(); + if (firstRun) { + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); + } + } +} diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml index 3675d760a35..44bbbe4acbf 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml index ee08e659ae7..674db7f319c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-emr6.13/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml index ffd41bdf7ca..312d6ce845d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdi5.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml index 435818c24cf..3529773711c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/opencga-storage-hadoop-lib-hdp3.1/pom.xml @@ -7,7 +7,7 @@ org.opencb.opencga opencga-storage-hadoop-lib - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml index 495de2ebf89..9f791862b43 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-lib/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 7de2add12c5..e72f24d30d9 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index d6d92f43dda..2a91d2753e4 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/RestStorageServer.java b/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/RestStorageServer.java index 226c8ab46ce..5343a17eb1d 100644 --- a/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/RestStorageServer.java +++ b/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/RestStorageServer.java @@ -16,11 +16,15 @@ package org.opencb.opencga.storage.server.rest; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; +import org.opencb.opencga.core.config.RestServerConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.storage.server.common.AbstractStorageServer; import org.slf4j.LoggerFactory; @@ -60,7 +64,14 @@ public void start() throws Exception { ServletHolder sh = new ServletHolder("opencga", sc); logger.info("Server in port : {}", port); - server = new Server(port); + server = new Server(); + + HttpConfiguration httpConfig = getHttpConfiguration(); + + ServerConnector httpConnector = new ServerConnector(server, new HttpConnectionFactory(httpConfig)); + httpConnector.setPort(port); + + server.addConnector(httpConnector); ServletContextHandler context = new ServletContextHandler(server, null, ServletContextHandler.SESSIONS); context.addServlet(sh, "/opencga/webservices/rest/*"); @@ -103,6 +114,27 @@ public void run() { AdminRestWebService.setServer(this); } + private HttpConfiguration getHttpConfiguration() { + HttpConfiguration httpConfig = new HttpConfiguration(); + RestServerConfiguration.HttpConfiguration restHttpConf = storageConfiguration.getServer().getRest().getHttpConfiguration(); + if (restHttpConf.getOutputBufferSize() > 0) { + httpConfig.setOutputBufferSize(restHttpConf.getOutputBufferSize()); + } + if (restHttpConf.getOutputAggregationSize() > 0) { + httpConfig.setOutputAggregationSize(restHttpConf.getOutputAggregationSize()); + } + if (restHttpConf.getRequestHeaderSize() > 0) { + httpConfig.setRequestHeaderSize(restHttpConf.getRequestHeaderSize()); + } + if (restHttpConf.getResponseHeaderSize() > 0) { + httpConfig.setResponseHeaderSize(restHttpConf.getResponseHeaderSize()); + } + if (restHttpConf.getHeaderCacheSize() > 0) { + httpConfig.setHeaderCacheSize(restHttpConf.getHeaderCacheSize()); + } + return httpConfig; + } + @Override public void stop() throws Exception { // By setting exit to true the monitor thread will close the Jetty server diff --git a/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/VariantRestWebService.java b/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/VariantRestWebService.java index 00832a76dd9..c6292d4f2f1 100644 --- a/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/VariantRestWebService.java +++ b/opencga-storage/opencga-storage-server/src/main/java/org/opencb/opencga/storage/server/rest/VariantRestWebService.java @@ -20,7 +20,7 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.storage.core.variant.query.VariantQueryResult; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index 316589d12ff..83f9de6ce82 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index 76a6360d5e2..88dbdeb7783 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index b187199de8b..84e170213b8 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 3.2.0-SNAPSHOT + 3.2.1-SNAPSHOT pom OpenCGA @@ -43,12 +43,13 @@ - 3.2.0_dev - 3.2.0_dev - 6.2.0-SNAPSHOT - 3.2.0-SNAPSHOT - 5.2.0-SNAPSHOT - 3.2.0-SNAPSHOT + + 3.2.1_dev + 3.2.1_dev + 6.1.1-SNAPSHOT + 3.2.1-SNAPSHOT + 5.2.1-SNAPSHOT + 3.2.1-SNAPSHOT 0.2.0 2.14.3 @@ -88,7 +89,7 @@ 2.3 2.8.0 9.31 - 1.5.4 + 1.7.14 4.0.1 1.13.0 @@ -97,7 +98,7 @@ 4.5.6 5.64.4 1.6.1 - 4.3.0 + 6.13.1 3.1.0 1.10.12 4.4.13 @@ -845,6 +846,12 @@ com.databricks SnpEff ${snpeff.version} + + + distlib + distlib + + org.apache.parquet @@ -970,6 +977,21 @@ kubernetes-model ${kubernetes.version} + + io.fabric8 + kubernetes-model-core + ${kubernetes.version} + + + io.fabric8 + kubernetes-client-api + ${kubernetes.version} + + + io.fabric8 + kubernetes-model-batch + ${kubernetes.version} + javax.servlet javax.servlet-api @@ -1347,6 +1369,7 @@ opencga LOCAL + 5 https://ws.opencb.org/opencga-prod @@ -1362,12 +1385,6 @@ 20 - http://localhost:8983/solr/ - 30000 - 2000 - - - false hadoop @@ -1412,7 +1429,7 @@ https://uk.ws.zettagenomics.com/cellbase/ - v5.2 + v5.8