Merge branch 'develop' into TASK-7102

.github/workflows/manual-delete-docker.yml

@@ -0,0 +1,17 @@
+name: Manual Delete Docker Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      task:
+        description: "Tag of the Docker image to delete (must start with 'TASK')"
+        required: true
+        type: string
+
+jobs:
+  call-delete-docker:
+    name: Call Reusable Delete Docker Workflow
+    uses: ./.github/workflows/delete-docker.yml
+    with:
+      task: ${{ inputs.task }}
+    secrets: inherit

.github/workflows/pull-request-approved.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   calculate-xetabase-branch:
+    if: github.event.review.state == 'approved'
     name: Calculate Xetabase branch
     runs-on: ubuntu-22.04
     outputs:
@@ -24,14 +25,15 @@ jobs:
           chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
           echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}"
           echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
-          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }})
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.ref }})
           echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
           echo "xetabase_branch: ${xetabase_branch}"
           echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
         env:
           ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
 
   test:
+    if: github.event.review.state == 'approved'
     name: "Run all tests before merging"
     needs: calculate-xetabase-branch
     uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop

.github/workflows/pull-request-merged.yml

@@ -16,8 +16,12 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
-      - name: delete docker images
+      - uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: Delete in Docker Hub
         run: |
-          echo "Deleting docker images"
+          echo "Deleting docker image ${{ github.head_ref }}"
           python3 ./opencga-app/app/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
 

.github/workflows/reusable-delete-docker.yml

@@ -0,0 +1,32 @@
+name: Reusable delete docker workflow from DockerHub
+
+on:
+  workflow_call:
+    inputs:
+      task:
+        type: string
+        required: true
+    secrets:
+      DOCKER_HUB_USER:
+        required: true
+      DOCKER_HUB_PASSWORD:
+        required: true
+
+jobs:
+  delete-docker:
+    name: Execute delete docker image
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+      - name: Validate task name
+        if: ${{ !startsWith(inputs.task, 'TASK') }}
+        run: |
+          echo "Error: Can't delete ${{ inputs.task }}. Only Docker images related to TASK branches can be deleted"
+          exit 1
+      - name: Delete in Docker Hub
+        if: ${{ startsWith(inputs.task, 'TASK') }}
+        run: |
+          echo "Deleting docker image ${{ inputs.task }}"
+          python3 ./opencga-app/app/cloud/docker/docker-build.py delete --images base --tag ${{ inputs.task }} --username ${{ secrets.DOCKER_HUB_USER }} --password ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/scripts/get-xetabase-branch.sh

@@ -3,24 +3,25 @@
 # Function to calculate the corresponding branch of Xetabase project
 get_xetabase_branch() {
   # Input parameter (branch name)
-  input_branch="$1"
+  target_branch="$1"
+  current_branch="$1"
 
   # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
-  if [[ $input_branch == TASK* ]]; then
-    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
-      echo "$input_branch";
+  if [[ $current_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$target_branch" )" ] ; then
+      echo "$target_branch";
       return 0;
     fi
   fi
 
   # Check if the branch name is "develop" in that case return the same branch name
-  if [[ "$input_branch" == "develop" ]]; then
+  if [[ "$target_branch" == "develop" ]]; then
     echo "develop"
     return 0
   fi
 
   # Check if the branch name starts with "release-" and follows the patterns "release-a.x.x" or "release-a.b.x"
-  if [[ "$input_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
+  if [[ "$target_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$target_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
     # Extract the MAJOR part of the branch name
     MAJOR=${BASH_REMATCH[1]}
     # Calculate the XETABASE_MAJOR by subtracting 1 from MAJOR of opencga
@@ -31,7 +32,7 @@ get_xetabase_branch() {
       return 1
     fi
     # Construct and echo the new branch name
-    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    echo "release-$XETABASE_MAJOR.${target_branch#release-$MAJOR.}"
     return 0
   fi
 
@@ -41,10 +42,11 @@ get_xetabase_branch() {
 }
 
 # Check if the script receives exactly one argument
-if [ "$#" -ne 1 ]; then
-  echo "Usage: $0 <branch-name>"
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <target-branch> <current-branch>"
   exit 1
 fi
 
+
 # Call the function with the input branch name
-get_xetabase_branch "$1"
+get_xetabase_branch "$1" "$2"

.github/workflows/test-analysis.yml

@@ -87,14 +87,28 @@ jobs:
         with:
           mongodb-version: 6.0
           mongodb-replica-set: rs-test
-      - name: Maven build
+      - name: Maven build (skip tests)
         run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+      - name: Build Junit log file name
+        id: BuildJunitLogFileName
+        run: |
+          MODULE=$(basename ${{ (inputs.module == '' || inputs.module == 'all') && 'opencga' || inputs.module }} )
+          if [[ -z "$MODULE" ]]; then
+            MODULE="opencga"
+          fi
+          TAGS=$(echo ${{ inputs.test_profile }} | sed -e 's/run\([^,]*\)Tests/\1/g'  | tr ',' '_' | tr '[:upper:]' '[:lower:]' )
+          echo "TESTS_LOG_FILE_NAME=junit_${{ inputs.hadoop }}_${TAGS}_${MODULE}.log" >> $GITHUB_OUTPUT
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}  |& tee ${{ steps.BuildJunitLogFileName.outputs.TESTS_LOG_FILE_NAME }} |& grep -P '^\[[^\]]*(INFO|WARNING|ERROR)' --colour=never --line-buffered
+      - name: Upload Junit test logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.BuildJunitLogFileName.outputs.TESTS_LOG_FILE_NAME }}
+          path: ${{ steps.BuildJunitLogFileName.outputs.TESTS_LOG_FILE_NAME }}
       - name: Publish Test Report on GitHub
         uses: scacap/action-surefire-report@v1
         env:
-          NODE_OPTIONS: '--max_old_space_size=4096'
+          NODE_OPTIONS: '--max_old_space_size=6144'
         ## Skip cancelled()
         ## https://docs.github.com/en/actions/learn-github-actions/expressions#cancelled
         if: success() || failure()

.gitignore

@@ -86,7 +86,7 @@ git.properties
 *.Rcheck*
 opencga-client/src/main/R_packages
 opencga-client/src/main/opencgaR_*.tar.gz
-opencga-client/src/main/R/vignettes/opencgaR.html
+#opencga-client/src/main/R/vignettes/opencgaR.html
 opencga-client/src/main/R/vignettes/opencgaR.md
 
 /opencga-catalog/src/test/java/org/opencb/opencga/catalog/config/CatalogConfigurationTest.java

opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java

@@ -32,6 +32,7 @@
 import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis;
 import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
+import org.opencb.opencga.core.config.Analysis;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.family.Family;
 import org.opencb.opencga.core.models.individual.Individual;
@@ -117,7 +118,7 @@ public static RelatednessReport compute(String study, Family family, List<String
         } catch (IOException e) {
             throw new ToolException("Something wrong happened when copying files during the relatedness analysis execution");
         }
-        File outFile = runIBD(FILTERED_BASENAME, freqPath, outDir);
+        File outFile = runIBD(FILTERED_BASENAME, freqPath, outDir, storageManager.getCatalogManager().getConfiguration().getAnalysis());
 
         if (!outFile.exists()) {
             throw new ToolException("Something wrong happened executing relatedness analysis");
@@ -298,7 +299,7 @@ public static void filterFamilyTpedFile(Path tPedPath, Path tPedFilteredPath, Pa
         bw.close();
     }
 
-    private static File runIBD(String basename, Path freqPath, Path outDir) throws ToolException {
+    private static File runIBD(String basename, Path freqPath, Path outDir, Analysis analysisConf) throws ToolException {
         // Input bindings
         List<AbstractMap.SimpleEntry<String, String>> inputBindings = new ArrayList<>();
         inputBindings.add(new AbstractMap.SimpleEntry<>(freqPath.getParent().toString(), "/input"));
@@ -311,8 +312,8 @@ private static File runIBD(String basename, Path freqPath, Path outDir) throws T
         String plinkParams = "plink1.9 --tfile /output/" + basename + " --genome rel-check --read-freq /input/" + FREQ_FILENAME
                 + " --out /output/" + basename;
         try {
-            PlinkWrapperAnalysisExecutor plinkExecutor = new PlinkWrapperAnalysisExecutor();
-            DockerUtils.run(plinkExecutor.getDockerImageName(), inputBindings, outputBinding, plinkParams, null);
+            String dockerImageName = PlinkWrapperAnalysisExecutor.getDockerImageName(analysisConf);
+            DockerUtils.run(dockerImageName, inputBindings, outputBinding, plinkParams, null);
         } catch (IOException e) {
             throw new ToolException(e);
         }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaTool.java

@@ -192,17 +192,11 @@ public final ExecutionResult start() throws ToolException {
             if (!erm.isClosed()) {
                 String message = "Unexpected system shutdown. Job killed by the system.";
                 privateLogger.error(message);
+                if (exception == null) {
+                    exception = new RuntimeException(message);
+                }
                 try {
-                    if (scratchDir != null) {
-                        deleteScratchDirectory();
-                    }
-                    if (exception == null) {
-                        exception = new RuntimeException(message);
-                    }
-                    logException(exception);
-                    ExecutionResult result = erm.close(exception);
-                    privateLogger.info("------- Tool '" + getId() + "' executed in "
-                            + TimeUtils.durationToString(result.getEnd().getTime() - result.getStart().getTime()) + " -------");
+                    close(exception);
                 } catch (ToolException e) {
                     privateLogger.error("Error closing ExecutionResult", e);
                 }
@@ -271,13 +265,25 @@ public final ExecutionResult start() throws ToolException {
             }
             throw e;
         } finally {
+            // If the shutdown hook has been executed, the ExecutionResultManager is already closed
+            if (!erm.isClosed()) {
+                result = close(exception);
+            } else {
+                result = erm.read();
+            }
+        }
+        return result;
+    }
+
+    private ExecutionResult close(Throwable exception) throws ToolException {
+        if (scratchDir != null) {
             deleteScratchDirectory();
-            stopMemoryMonitor();
-            result = erm.close(exception);
-            logException(exception);
-            privateLogger.info("------- Tool '" + getId() + "' executed in "
-                    + TimeUtils.durationToString(result.getEnd().getTime() - result.getStart().getTime()) + " -------");
         }
+        logException(exception);
+        stopMemoryMonitor();
+        ExecutionResult result = erm.close(exception);
+        privateLogger.info("------- Tool '" + getId() + "' executed in "
+                + TimeUtils.durationToString(result.getEnd().getTime() - result.getStart().getTime()) + " -------");
         return result;
     }
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/VariantExportTool.java

@@ -20,18 +20,14 @@
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
 import org.opencb.opencga.analysis.tools.OpenCgaTool;
-import org.opencb.opencga.catalog.io.IOManager;
-import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.variant.VariantExportParams;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory;
 
-import java.net.URI;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -70,9 +66,8 @@ protected List<String> getSteps() {
 
     @Override
     protected void run() throws Exception {
-        List<URI> uris = new ArrayList<>(2);
         step(ID, () -> {
-            Path outDir = getScratchDir();
+            Path outDir = getOutDir();
             String outputFile = StringUtils.isEmpty(toolParams.getOutputFileName())
                     ? outDir.toString()
                     : outDir.resolve(toolParams.getOutputFileName()).toString();
@@ -81,17 +76,9 @@ protected void run() throws Exception {
             for (VariantQueryParam param : VariantQueryParam.values()) {
                 queryOptions.remove(param.key());
             }
-            uris.addAll(variantStorageManager.exportData(outputFile,
+            variantStorageManager.exportData(outputFile,
                     outputFormat,
-                    toolParams.getVariantsFile(), query, queryOptions, token));
-        });
-        step("move-files", () -> {
-            IOManager ioManager = catalogManager.getIoManagerFactory().get(uris.get(0));
-            for (URI uri : uris) {
-                String fileName = UriUtils.fileName(uri);
-                logger.info("Moving file -- " + fileName);
-                ioManager.move(uri, getOutDir().resolve(fileName).toUri());
-            }
+                    toolParams.getVariantsFile(), query, queryOptions, token);
         });
     }
 }