Merge branch 'develop' into TASK-5516

.github/workflows/develop.yml

@@ -13,23 +13,24 @@ jobs:
     with:
       maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
 
-  test:
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
-    with:
-      test_profile: runShortTests,runMediumTests
-    secrets: inherit
+## TASK-4970: remove the test job from the develop.yml workflow because it is not needed anymore.
+#  test:
+#    uses: ./.github/workflows/test-analysis.yml
+#    needs: build
+#    with:
+#      test_profile: runShortTests,runMediumTests
+#    secrets: inherit
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
     secrets: inherit

.github/workflows/long-test-analysis.yml

@@ -14,8 +14,8 @@ jobs:
       matrix:
         hadoop: [ "hdp3.1", "hdi5.1", "emr6.1", "emr6.13" ]
     uses: ./.github/workflows/test-analysis.yml
-    secrets: inherit
     with:
       test_profile: runShortTests,runMediumTests,runLongTests
       hadoop: ${{ matrix.hadoop }}
+    secrets: inherit
 

.github/workflows/manual-deploy-docker.yml

@@ -51,10 +51,9 @@ jobs:
           name: build-folder
           path: build
 
-
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ inputs.tag }}
-    secrets: inherit
+    secrets: inherit

.github/workflows/manual-deploy-ext-tools.yml

@@ -46,11 +46,9 @@ jobs:
           name: build-folder
           path: build
 
-
   deploy-docker-ext-tools:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }}
     secrets: inherit
-

.github/workflows/pull-request-approved.yml

@@ -0,0 +1,19 @@
+name: Pull request approve workflow
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  build:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+
+  test:
+    name: "Run all tests before merging, ie. short, medium and long tests."
+    uses: ./.github/workflows/test-analysis.yml
+    needs: build
+    with:
+      test_profile: runShortTests,runMediumTests,runLongTests
+    secrets: inherit

.github/workflows/task.yml

@@ -17,9 +17,9 @@ jobs:
   test:
     uses: ./.github/workflows/test-analysis.yml
     needs: build
-    secrets: inherit
     with:
       test_profile: runShortTests
+    secrets: inherit
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop

.github/workflows/test-analysis.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: '0'
+          fetch-depth: '10'
       - name: Set up JDK 11
         uses: actions/setup-java@v4
         with:
@@ -65,7 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: '0'
+          fetch-depth: '10'
       - name: Set up JDK 8
         uses: actions/setup-java@v4
         with:
@@ -91,7 +91,7 @@ jobs:
         run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Run Junit tests
         run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
-      - name: Publish Test Report
+      - name: Publish Test Report on GitHub
         uses: scacap/action-surefire-report@v1
         env:
           NODE_OPTIONS: '--max_old_space_size=4096'

opencga-analysis/pom.xml

@@ -357,6 +357,17 @@
     </dependencyManagement>
 
     <build>
+        <testResources>
+            <testResource>
+                <directory>../opencga-core/src/test/resources</directory>
+                <includes>
+                    <include>log4j2-test.xml</include>
+                </includes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
+        </testResources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java

@@ -54,6 +54,10 @@ protected void check() throws Exception {
         super.check();
 
         // Sanity check
+        if (StringUtils.isEmpty(getJobId())) {
+            throw new ToolException("Missing job ID");
+        }
+
         if (StringUtils.isEmpty(getStudy())) {
             throw new ToolException("Missing study when computing alignment coverage");
         }
@@ -168,20 +172,22 @@ protected void run() throws Exception {
                         + ") was not create, please, check log files.");
             }
 
-            // Try to copy the BW file into the BAM file directory
+            // Try to move the BW file into the BAM file directory
+            boolean moveSuccessful = false;
             Path targetPath = Paths.get(bamCatalogFile.getUri()).getParent().resolve(bwPath.getFileName());
             try {
-                Files.move(bwPath, targetPath);
+                Path movedPath = Files.move(bwPath, targetPath);
+                moveSuccessful = targetPath.equals(movedPath);
             } catch (Exception e) {
-                // Do nothing
-                logger.info("Moving from {} to {}: {}", bwPath, targetPath, e.getMessage());
+                // Log message
+                logger.info("Error moving the coverage file into the BAM folder {} to {}", bwPath, targetPath, e);
             }
 
-            if (targetPath.toFile().exists()) {
+            if (moveSuccessful) {
                 bwPath = targetPath;
-                logger.info("Coverage file was copied into the BAM folder: {}", bwPath);
+                logger.info("Coverage file was moved into the BAM folder: {}", bwPath);
             } else {
-                logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
+                logger.info("Couldn't move the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
                         bwPath);
             }
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java

@@ -16,29 +16,34 @@
 
 package org.opencb.opencga.analysis.alignment;
 
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.tools.alignment.BamManager;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.tools.OpenCgaTool;
+import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
+import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.InternalStatus;
 import org.opencb.opencga.core.models.file.*;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
+import org.opencb.opencga.core.tools.annotations.ToolParams;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 @Tool(id = AlignmentIndexOperation.ID, resource = Enums.Resource.ALIGNMENT, description = "Index alignment.")
-public class AlignmentIndexOperation extends OpenCgaTool {
+public class AlignmentIndexOperation extends OpenCgaToolScopeStudy {
 
     public static final String ID = "alignment-index-run";
     public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";
 
-    private String study;
-    private String inputFile;
+    @ToolParams
+    protected final AlignmentIndexParams indexParams = new AlignmentIndexParams();
 
     private File inputCatalogFile;
     private Path inputPath;
@@ -48,14 +53,23 @@ public class AlignmentIndexOperation extends OpenCgaTool {
     protected void check() throws Exception {
         super.check();
 
+        // Sanity check
+        if (StringUtils.isEmpty(getJobId())) {
+            throw new ToolException("Missing job ID");
+        }
+
+        if (StringUtils.isEmpty(getStudy())) {
+            throw new ToolException("Missing study when computing alignment index");
+        }
+
         OpenCGAResult<File> fileResult;
         try {
-            fileResult = catalogManager.getFileManager().get(getStudy(), inputFile, QueryOptions.empty(), token);
+            fileResult = catalogManager.getFileManager().get(getStudy(), indexParams.getFileId(), QueryOptions.empty(), token);
         } catch (CatalogException e) {
-            throw new ToolException("Error accessing file '" + inputFile + "' of the study " + study + "'", e);
+            throw new ToolException("Error accessing file '" + indexParams.getFileId() + "' of the study " + study + "'", e);
         }
         if (fileResult.getNumResults() <= 0) {
-            throw new ToolException("File '" + inputFile + "' not found in study '" + study + "'");
+            throw new ToolException("File '" + indexParams.getFileId() + "' not found in study '" + study + "'");
         }
 
         inputCatalogFile = fileResult.getResults().get(0);
@@ -64,7 +78,7 @@ protected void check() throws Exception {
 
         // Check if the input file is .bam or .cram
         if (!filename.endsWith(AlignmentConstants.BAM_EXTENSION) && !filename.endsWith(AlignmentConstants.CRAM_EXTENSION)) {
-            throw new ToolException("Invalid input alignment file '" + inputFile + "': it must be in BAM or CRAM format");
+            throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format");
         }
 
         outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
@@ -73,6 +87,9 @@ protected void check() throws Exception {
 
     @Override
     protected void run() throws Exception {
+        setUpStorageEngineExecutor(study);
+
+        logger.info("Running with parameters {}", indexParams);
 
         step(ID, () -> {
             // Compute index if necessary
@@ -82,51 +99,36 @@ protected void run() throws Exception {
             bamManager.close();
 
             if (!outputPath.toFile().exists()) {
-                throw new ToolException("Something wrong happened when computing index file for '" + inputFile + "'");
+                throw new ToolException("Something wrong happened when computing index file for '" + indexParams.getFileId() + "'");
             }
 
-            // Try to copy the BAI file into the BAM file directory
+            // Try to move the BAI file into the BAM file directory
+            boolean moveSuccessful = false;
             Path targetPath = inputPath.getParent().resolve(outputPath.getFileName());
             try {
-                Files.move(outputPath, targetPath);
+                Path movedPath = Files.move(outputPath, targetPath);
+                moveSuccessful = targetPath.equals(movedPath);
             } catch (Exception e) {
-                // Do nothing
-                logger.info("Moving from {} to {}: {}", outputPath, targetPath, e.getMessage());
+                // Log message
+                logger.info("Error moving from {} to {}", outputPath, targetPath, e);
             }
 
-            if (targetPath.toFile().exists()) {
+            if (moveSuccessful) {
                 outputPath = targetPath;
-                logger.info("Alignment index file was copied into the BAM folder: {}", outputPath);
+                logger.info("Alignment index file was moved into the BAM folder: {}", outputPath);
             } else {
-                logger.info("Couldn't copy the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
+                logger.info("Couldn't move the alignment index file into the BAM folder. The index file is in the job folder instead: {}",
                         outputPath);
             }
 
             // Link generated BAI file and update samples info, related file
-            File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager, token);
+            File baiCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(inputCatalogFile, outputPath, getJobId(), study, catalogManager,
+                    token);
 
             // Update BAM file internal in order to set the alignment index (BAI)
             FileInternalAlignmentIndex fileAlignmentIndex = new FileInternalAlignmentIndex(new InternalStatus(InternalStatus.READY),
                     baiCatalogFile.getId(), "HTSJDK library");
             catalogManager.getFileManager().updateFileInternalAlignmentIndex(study, inputCatalogFile, fileAlignmentIndex, token);
         });
     }
-
-    public String getStudy() {
-        return study;
-    }
-
-    public AlignmentIndexOperation setStudy(String study) {
-        this.study = study;
-        return this;
-    }
-
-    public String getInputFile() {
-        return inputFile;
-    }
-
-    public AlignmentIndexOperation setInputFile(String inputFile) {
-        this.inputFile = inputFile;
-        return this;
-    }
 }