Merge branch 'release-2.8.2.x' into TASK-4144

opencb · Jul 3, 2023 · ba3144e · ba3144e
2 parents 9f3ca93 + ca109ef
commit ba3144e
Show file tree

Hide file tree

Showing 258 changed files with 6,178 additions and 3,731 deletions.
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -9,29 +9,35 @@ on:
 
 jobs:
   build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x
     with:
       maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
 
-  test:
-    uses: opencb/opencga/.github/workflows/test-analysis.yml@develop
+  deploy-docker-ext-tools:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }}
+    secrets: inherit
+
+  test:
+    uses: ./.github/workflows/test-analysis.yml
+    needs: deploy-docker-ext-tools
     secrets: inherit
     with:
       test_profile: runShortTests,runMediumTests
       report_context: development
 
   deploy-maven:
-    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.8.x
     needs: test
     with:
       maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: test
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
     secrets: inherit
-
diff --git a/.github/workflows/docker-aws-emr.yml b/.github/workflows/docker-aws-emr.yml
@@ -8,12 +8,12 @@ on:
 
 jobs:
   build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x
     with:
       maven_opts: -P storage-hadoop,emr6.1 -Dopencga.war.name=opencga
 
   deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init

diff --git a/.github/workflows/docker-tools.yml b/.github/workflows/docker-tools.yml
@@ -12,15 +12,13 @@ on:
 
 jobs:
   build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x
     with:
       maven_opts:
 
   deploy-docker-ext-tools:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }}
     secrets: inherit
-#      DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
-#      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
diff --git a/.github/workflows/long-test-analysis.yml b/.github/workflows/long-test-analysis.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   test:
-    uses: opencb/opencga/.github/workflows/test-analysis.yml@develop
+    uses: ./.github/workflows/test-analysis.yml
     secrets: inherit
     with:
       test_profile: runLongTests,runMediumTests,runShortTests

diff --git a/.github/workflows/merge.yml → .github/workflows/pull-request-merge.yml b/.github/workflows/merge.yml → .github/workflows/pull-request-merge.yml
@@ -1,4 +1,4 @@
-name: "Merge Workflow"
+name: "Pull Request Merge Workflow"
 
 on:
   pull_request:
@@ -10,7 +10,7 @@ on:
 
 jobs:
   delete-docker:
-    uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/[email protected]
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
     secrets: inherit
-
-
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -8,50 +8,52 @@ on:
 
 jobs:
   build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x
     with:
       maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
 
-  test:
-    uses: opencb/opencga/.github/workflows/test-analysis.yml@develop
+  deploy-docker-ext-tools:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }}
+    secrets: inherit
+
+  test:
+    uses: ./.github/workflows/test-analysis.yml
+    needs: deploy-docker-ext-tools
     secrets: inherit
     with:
-      report_context: xetabase
       test_profile: runShortTests,runMediumTests
+      report_context: xetabase
 
   deploy-maven:
-    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.8.x
     needs: test
     with:
       maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: test
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
     secrets: inherit
 
-  deploy-docker-ext-tools:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
-    needs: [ build, test ]
-    with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }}
-    secrets: inherit
+
 
   deploy-python:
-    uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@release-4.8.x
     needs: test
     with:
       cli: ./clients/python/python-build.sh push
       artifact: build-folder
     secrets: inherit
 
   release:
-    uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
-    needs: test
+    uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@release-4.8.x
+    needs: [ build,test ]
     with:
       artifact: build-folder
       file: |

diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml
@@ -8,12 +8,12 @@ on:
 
 jobs:
   build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x
     with:
       maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
 
   test:
-    uses: opencb/opencga/.github/workflows/test-analysis.yml@develop
+    uses: ./.github/workflows/test-analysis.yml
     needs: build
     secrets: inherit
     with:
@@ -22,13 +22,13 @@ jobs:
 
   deploy-maven:
     name: Deploy in maven only for renamed versions
-    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.8.x
     needs: [ build, test ]
     if: contains( needs.build.outputs.version ,'TASK')
     secrets: inherit
 
   deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x
     needs: test
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ github.ref_name }}
@@ -52,4 +52,3 @@ jobs:
     steps:
       - name: test-version-from-check
         run: echo "Project version is " ${{ needs.build.outputs.version }}
-
diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -1,5 +1,4 @@
 name: Build and test the project
-
 on:
   workflow_call:
     inputs:
@@ -20,10 +19,8 @@ on:
         required: true
       SSH_TESTING_SERVER_PASSWORD:
         required: true
-
 env:
-  xb_version: "1.6.0-SNAPSHOT"
-
+  xb_version: "1.6.2"
 jobs:
   analysis:
     name: Execute Sonar Analysis
@@ -43,7 +40,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: mvn -T 2 clean install -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
-
   test:
     name: Execute JUnit and Jacoco tests
     runs-on: ubuntu-22.04
@@ -71,14 +67,13 @@ jobs:
         with:
           name: workdir
           path: "**/target/site"
-
   publish-test:
     name: Publish test results
     runs-on: ubuntu-22.04
     needs: test
     strategy:
       matrix:
-        module: [ "opencga-app","opencga-catalog","opencga-client","opencga-clinical","opencga-core","opencga-master","opencga-server","opencga-storage","opencga-storage/opencga-storage-app","opencga-storage/opencga-storage-benchmark","opencga-storage/opencga-storage-core","opencga-storage/opencga-storage-hadoop","opencga-storage/opencga-storage-server","opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core","opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps" ]
+        module: ["opencga-app", "opencga-catalog", "opencga-client", "opencga-clinical", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-storage/opencga-storage-app", "opencga-storage/opencga-storage-benchmark", "opencga-storage/opencga-storage-core", "opencga-storage/opencga-storage-hadoop", "opencga-storage/opencga-storage-server", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps"]
     steps:
       - name: Download result dir
         uses: actions/download-artifact@v3

diff --git a/...workflows/test-long-test-analysis.yml_bkp → ...hub/workflows/test-long-test-analysis.yml b/...workflows/test-long-test-analysis.yml_bkp → ...hub/workflows/test-long-test-analysis.yml
@@ -3,13 +3,12 @@ name: LONG TEST ANALYSIS
 on:
   push:
     branches:
-      - TASK-*
+      - TASK-4149
 
 jobs:
   test:
-    uses: opencb/opencga/.github/workflows/test-analysis.yml@TASK-3980
+    uses: ./.github/workflows/test-analysis.yml
     secrets: inherit
     with:
       test_profile: runLongTests,runMediumTests,runShortTests
       report_context: development
-
diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>2.8.0-SNAPSHOT</version>
+        <version>2.8.2.1-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java b/...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java
@@ -230,6 +230,7 @@ public OpenCGAResult<GeneCoverageStats> coverageStats(String studyIdStr, String
         String species = projectQueryResult.first().getOrganism().getScientificName();
         String assembly = projectQueryResult.first().getOrganism().getAssembly();
         String dataRelease = projectQueryResult.first().getCellbase().getDataRelease();
+        String cellbaseToken = projectQueryResult.first().getCellbase().getToken();
 
         for (String geneName : geneNames) {
 
@@ -250,8 +251,8 @@ public OpenCGAResult<GeneCoverageStats> coverageStats(String studyIdStr, String
 
 
             // Query CellBase to get gene coordinates and then apply the offset (up and downstream) to create a gene region
-            CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, projectQueryResult.first().getCellbase()
-                    .toClientConfiguration());
+            CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, cellbaseToken,
+                    projectQueryResult.first().getCellbase().toClientConfiguration());
             GeneClient geneClient = cellBaseClient.getGeneClient();
             Gene gene = geneClient.get(Collections.singletonList(geneName), QueryOptions.empty()).firstResult();
             if (gene != null) {
@@ -450,8 +451,9 @@ public List<Region> mergeRegions(List<Region> regions, List<String> genes, boole
         String species = projectQueryResult.first().getOrganism().getScientificName();
         String assembly = projectQueryResult.first().getOrganism().getAssembly();
         String dataRelease = projectQueryResult.first().getCellbase().getDataRelease();
-        CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, projectQueryResult.first().getCellbase()
-                .toClientConfiguration());
+        String cellbaseToken = projectQueryResult.first().getCellbase().getToken();
+        CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, cellbaseToken,
+                projectQueryResult.first().getCellbase().toClientConfiguration());
         GeneClient geneClient = cellBaseClient.getGeneClient();
         List<Gene> response = geneClient.get(genes, QueryOptions.empty()).allResults();
         if (CollectionUtils.isNotEmpty(response)) {

diff --git a/...sis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java b/...sis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java
@@ -311,7 +311,7 @@ public OpenCGAResult<ClinicalVariant> get(Query query, QueryOptions queryOptions
                             query.getString(ParamConstants.INCLUDE_INTERPRETATION));
                     for (ClinicalVariant primaryFinding : interpretation.getPrimaryFindings()) {
                         for (ClinicalVariant clinicalVariant : clinicalVariants) {
-                            if (clinicalVariant.toStringSimple().equals(primaryFinding.toStringSimple())) {
+                            if (clinicalVariant.sameGenomicVariant(primaryFinding)) {
                                 // Only it's updated the following fields
                                 // Important to note that the results include the "new" clinical evidences
                                 clinicalVariant.setComments(primaryFinding.getComments())
@@ -321,13 +321,31 @@ public OpenCGAResult<ClinicalVariant> get(Query query, QueryOptions queryOptions
                                         .setAttributes(primaryFinding.getAttributes());
 
                                 // Update clinical evidence review if it is necessary
-                                if (CollectionUtils.isNotEmpty(primaryFinding.getEvidences())
-                                        && CollectionUtils.isNotEmpty(clinicalVariant.getEvidences())) {
-                                    for (ClinicalVariantEvidence primaryFindingEvidence : primaryFinding.getEvidences()) {
-                                        for (ClinicalVariantEvidence clinicalVariantEvidence : clinicalVariant.getEvidences()) {
-                                            if (ClinicalUtils.matchEvidence(primaryFindingEvidence, clinicalVariantEvidence)) {
-                                                clinicalVariantEvidence.setReview(primaryFindingEvidence.getReview());
+                                if (CollectionUtils.isNotEmpty(primaryFinding.getEvidences())) {
+                                    if (CollectionUtils.isEmpty(clinicalVariant.getEvidences())) {
+                                        clinicalVariant.setEvidences(primaryFinding.getEvidences());
+                                    } else {
+                                        List<ClinicalVariantEvidence> evidencesToAdd = new ArrayList<>();
+                                        for (ClinicalVariantEvidence primaryFindingEvidence : primaryFinding.getEvidences()) {
+                                            boolean found = false;
+                                            for (ClinicalVariantEvidence clinicalVariantEvidence : clinicalVariant.getEvidences()) {
+                                                if (ClinicalUtils.matchEvidences(primaryFindingEvidence, clinicalVariantEvidence)) {
+                                                    clinicalVariantEvidence.setInterpretationMethodName(primaryFindingEvidence
+                                                            .getInterpretationMethodName());
+                                                    clinicalVariantEvidence.setModeOfInheritances(primaryFindingEvidence
+                                                            .getModeOfInheritances());
+                                                    clinicalVariantEvidence.setReview(primaryFindingEvidence.getReview());
+                                                    clinicalVariantEvidence.setAttributes(primaryFindingEvidence.getAttributes());
+                                                    found = true;
+                                                    break;
+                                                }
                                             }
+                                            if (!found) {
+                                                evidencesToAdd.add(primaryFindingEvidence);
+                                            }
+                                        }
+                                        if (CollectionUtils.isNotEmpty(evidencesToAdd)) {
+                                            clinicalVariant.getEvidences().addAll(evidencesToAdd);
                                         }
                                     }
                                 }