Merge branch 'TASK-6766' of github.com:opencb/opencga into TASK-6766

.github/workflows/pull-request-approved.yml

@@ -1,19 +1,41 @@
 name: Pull request approve workflow
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
 
 on:
   pull_request_review:
-    types: [submitted]
+    types: [ submitted ]
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-    with:
-      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone project
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}"
+          echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }})
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
 
   test:
-    name: "Run all tests before merging, ie. short, medium and long tests."
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
+    name: "Run all tests before merging"
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
     with:
-      test_profile: runShortTests,runMediumTests
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: ${{ github.event.pull_request.head.ref }}
     secrets: inherit
+

.github/workflows/scripts/get-xetabase-branch.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Function to calculate the corresponding branch of Xetabase project
+get_xetabase_branch() {
+  # Input parameter (branch name)
+  input_branch="$1"
+
+  # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
+  if [[ $input_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
+      echo $input_branch;
+      return 0;
+    fi
+  fi
+
+  # Check if the branch name is "develop" in that case return the same branch name
+  if [[ "$input_branch" == "develop" ]]; then
+    echo "develop"
+    return 0
+  fi
+
+  # Check if the branch name starts with "release-" and follows the patterns "release-a.x.x" or "release-a.b.x"
+  if [[ "$input_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
+    # Extract the MAJOR part of the branch name
+    MAJOR=${BASH_REMATCH[1]}
+    # Calculate the XETABASE_MAJOR by subtracting 1 from MAJOR of opencga
+    XETABASE_MAJOR=$((MAJOR - 1))
+    # Check if the XETABASE_MAJOR is negative
+    if (( XETABASE_MAJOR < 0 )); then
+      echo "Error: 'MAJOR' digit after subtraction results in a negative number."
+      return 1
+    fi
+    # Construct and echo the new branch name
+    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    return 0
+  fi
+
+  # If the branch name does not match any of the expected patterns
+  echo "Error: The branch name is not correct."
+  return 1
+}
+
+# Check if the script receives exactly one argument
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <branch-name>"
+  exit 1
+fi
+
+# Call the function with the input branch name
+get_xetabase_branch "$1"

opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java

@@ -16,9 +16,13 @@
 
 package org.opencb.opencga.analysis;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.commons.utils.FileUtils;
+import org.opencb.opencga.core.config.AnalysisTool;
 import org.opencb.opencga.core.config.Configuration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,10 +32,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 public class ConfigurationUtils {
     private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class);
 
+    private ConfigurationUtils() {
+        throw new IllegalStateException("Utility class");
+    }
     /**
      * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml.
      *
@@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome)
                     .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml"));
         }
     }
+
+    public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException {
+        List<AnalysisTool> tools = new ArrayList<>();
+        for (AnalysisTool tool : configuration.getAnalysis().getTools()) {
+            if (tool.getId().equals(toolId)) {
+                tools.add(tool);
+            }
+        }
+        if (CollectionUtils.isEmpty(tools)) {
+            throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file");
+        }
+        if (tools.size() == 1) {
+            return tools.get(0).getVersion();
+        }
+        String defaultVersion = null;
+        for (AnalysisTool tool : tools) {
+            if (tool.isDefaultVersion()) {
+                if (!StringUtils.isEmpty(defaultVersion)) {
+                    throw new ToolException("More than one default version found for tool ID '" + toolId + "'");
+                } else {
+                    defaultVersion = tool.getVersion();
+                }
+            }
+        }
+        if (StringUtils.isEmpty(defaultVersion)) {
+            throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true");
+        }
+        return defaultVersion;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/ResourceUtils.java

@@ -28,42 +28,167 @@
 import org.opencb.opencga.core.models.JwtPayload;
 import org.opencb.opencga.core.models.project.Project;
 import org.opencb.opencga.core.response.OpenCGAResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 
 public class ResourceUtils {
 
     public static final String URL = "http://resources.opencb.org/opencb/opencga/";
 
+    public static final String RESOURCES_TXT_FILENAME = "resources.txt";
+    public static final String ANALYSIS_FOLDER_NAME = "analysis";
+    private static final String RESOURCES_FOLDER_NAME = "resources";
+
+    // Locking mechanism to prevent concurrent downloads for the same analysis
+    private static final Map<String, Lock> analysisLocks = new ConcurrentHashMap<>();
+    private static final long LOCK_TIMEOUT = 2; // Timeout of 2 hours
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ResourceUtils.class);
+
     public static File downloadThirdParty(URL url, Path outDir) throws IOException {
         return URLUtils.download(url, outDir);
     }
 
     public static File downloadAnalysis(String analysisId, String resouceName, Path outDir, Path openCgaHome) throws IOException {
         Path path = null;
-        String filename = "analysis/" + analysisId + "/" + resouceName;
+        String filename = ANALYSIS_FOLDER_NAME + "/" + analysisId + "/" + resouceName;
         if (openCgaHome != null) {
             path = openCgaHome.resolve(filename);
         }
         if (path != null && path.toFile().exists()) {
             File outFile = outDir.resolve(path.toFile().getName()).toFile();
-            System.out.println("downloadAnalysis from path: " + path + " to " + outFile.getAbsolutePath());
+            LOGGER.info("downloadAnalysis from path: " + path + " to " + outFile.getAbsolutePath());
             FileUtils.copyFile(path.toFile(), outFile);
 
             return outFile;
         } else {
-            System.out.println("downloadAnalysis from URL: " + (URL + filename) + ", (path does not exist: " + path + ")");
+            LOGGER.info("downloadAnalysis from URL: " + (URL + filename) + ", (path does not exist: " + path + ")");
             return URLUtils.download(new URL(URL + filename), outDir);
         }
     }
 
-    public static DownloadedRefGenome downloadRefGenome(String assembly, Path outDir, Path openCgaHome) throws IOException {
+    public static List<File> getResourceFiles(String analysisId, Path openCgaHome) throws IOException {
+        return getResourceFiles(URL, analysisId, openCgaHome);
+    }
+
+    public static List<File> getResourceFiles(String baseUrl, String analysisId, Path openCgaHome) throws IOException {
+        List<File> downloadedFiles = new ArrayList<>();
+
+        // Get resource filenames for the input analysis
+        Path resourcesTxt = openCgaHome.resolve(ANALYSIS_FOLDER_NAME).resolve(analysisId).resolve(RESOURCES_TXT_FILENAME);
+        List<String> filenames = readAllLines(resourcesTxt);
+
+        // Create a lock for the analysis if not present, and get it for the input analysis
+        analysisLocks.putIfAbsent(analysisId, new ReentrantLock());
+        Lock lock = analysisLocks.get(analysisId);
+
+        boolean lockAcquired = false;
+        try {
+            // Try to acquire the lock within the specified timeout
+            lockAcquired = lock.tryLock(LOCK_TIMEOUT, TimeUnit.HOURS);
+
+            if (lockAcquired) {
+                // Create the analysis resources directory if it doesn't exist
+                Path analysisResourcesPath = openCgaHome.resolve(ANALYSIS_FOLDER_NAME).resolve(RESOURCES_FOLDER_NAME).resolve(analysisId);
+                if (!Files.exists(analysisResourcesPath)) {
+                    Files.createDirectories(analysisResourcesPath);
+                }
+
+                // Download each file
+                for (String fileName : filenames) {
+                    downloadedFiles.add(downloadFile(baseUrl, analysisId, fileName, analysisResourcesPath).toFile());
+                }
+                LOGGER.info("Download complete for analysis '" + analysisId + "'");
+            } else {
+                String msg = "Could not acquire lock for analysis '" + analysisId + "' within " + LOCK_TIMEOUT + " hours. Skipping...";
+                LOGGER.error(msg);
+                throw new RuntimeException(msg);
+            }
+        } catch (InterruptedException e) {
+            // Restore interrupt status
+            Thread.currentThread().interrupt();
+            String msg = "Interrupted while trying to acquire lock for analysis '" + analysisId + "'";
+            LOGGER.error(msg);
+            throw new RuntimeException(msg, e);
+        } finally {
+            if (lockAcquired) {
+                // Release the lock
+                lock.unlock();
+            }
+        }
+
+        return downloadedFiles;
+    }
+
+    public static List<String> readAllLines(Path path) throws IOException {
+        List<String> lines;
+        if (!Files.exists(path)) {
+            String msg = "Filename '" + path + "' does not exist";
+            LOGGER.error(msg);
+            throw new IOException(msg);
+        }
+        try {
+            lines = Files.readAllLines(path);
+            if (CollectionUtils.isEmpty(lines)) {
+                String msg = "Filename '" + path + "' is empty";
+                LOGGER.error(msg);
+                throw new IOException(msg);
+            }
+        } catch (IOException e) {
+            LOGGER.error(e.getMessage(), e);
+            throw e;
+        }
+        return lines;
+    }
+
+    private static Path downloadFile(String baseUrl, String analysisId, String filename, Path localPath) throws IOException {
+        String fileUrl = baseUrl + analysisId + "/" + filename;
+        Path localFile = localPath.resolve(filename);
+
+        // Check if the file already exists
+        if (Files.exists(localFile)) {
+            LOGGER.info("File " + filename + " already exists, skipping download");
+            return localFile;
+        }
 
+        try (BufferedInputStream in = new BufferedInputStream(new URL(fileUrl).openStream());
+             FileOutputStream fileOutputStream = new FileOutputStream(localFile.toFile())) {
+            byte[] dataBuffer = new byte[1024];
+            int bytesRead;
+            while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
+                fileOutputStream.write(dataBuffer, 0, bytesRead);
+            }
+        } catch (IOException e) {
+            LOGGER.error("Failed to download file: " + fileUrl, e);
+            throw e;
+        }
+
+        if (!Files.exists(localFile)) {
+            String msg = "Something wrong happened, file '" + filename + "' + does not exist after downloading at '" + fileUrl + "'";
+            LOGGER.error(msg);
+            throw new IOException(msg);
+        }
+
+        return localFile;
+    }
+
+    public static DownloadedRefGenome downloadRefGenome(String assembly, Path outDir, Path openCgaHome) throws IOException {
         // Download files
         File gzFile = null;
         File faiFile = null;
@@ -84,12 +209,12 @@ public static DownloadedRefGenome downloadRefGenome(String assembly, Path outDir
             }
             if (path != null && path.toFile().exists()) {
                 File outFile = outDir.resolve(path.toFile().getName()).toFile();
-                System.out.println("downloadRefGenome from path: " + path + " to " + outFile.getAbsolutePath());
+                LOGGER.info("downloadRefGenome from path: " + path + " to " + outFile.getAbsolutePath());
                 FileUtils.copyFile(path.toFile(), outFile);
                 file = outFile;
             } else {
                 URL url = new URL(URL + "analysis/commons/reference-genomes/" + filename);
-                System.out.println("downloadAnalysis from URL: " + URL + ", (path does not exist: " + path + ")");
+                LOGGER.info("downloadAnalysis from URL: " + URL + ", (path does not exist: " + path + ")");
                 file = URLUtils.download(url, outDir);
                 if (file == null) {
                     // Something wrong happened, remove downloaded files
@@ -122,7 +247,7 @@ public static String getAssembly(CatalogManager catalogManager, String studyId,
         JwtPayload jwtPayload = catalogManager.getUserManager().validateToken(sessionId);
         CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyId, jwtPayload);
         String organizationId = studyFqn.getOrganizationId();
-        
+
         projectQueryResult = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), studyId),
                 new QueryOptions(QueryOptions.INCLUDE, ProjectDBAdaptor.QueryParams.ORGANISM.key()), sessionId);
         if (CollectionUtils.isNotEmpty(projectQueryResult.getResults())