Merge branch 'develop' into TASK-5610

opencb · Apr 11, 2024 · 38d57aa · 38d57aa
2 parents c61a302 + fcd5084
commit 38d57aa
Show file tree

Hide file tree

Showing 338 changed files with 7,654 additions and 4,007 deletions.
diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml
@@ -1,8 +1,36 @@
 name: Check junits
+run-name: "Check Junit. Hadoop ${{ inputs.hadoop }} ${{ inputs.module == 'all' && ' ' || format(':{0}. ', inputs.module ) }}${{ inputs.short_tests && 'Short ' || '' }}${{ inputs.medium_tests && 'Medium ' || '' }}${{ inputs.long_tests && 'Long ' || '' }}${{ ! ( inputs.short_tests || inputs.medium_tests || inputs.long_tests ) && 'Short Medium Long ' || '' }}tests."
 
 on:
   workflow_dispatch:
     inputs:
+      hadoop:
+        type: choice
+        description: 'Hadoop flavour.'
+        required: false
+        default: "hdp3.1"
+        options:
+          - "all"
+          - "hdp3.1"
+          - "hdi5.1"
+          - "emr6.1"
+          - "emr6.13"
+      module:
+        type: choice
+        description: 'OpenCGA module to test.'
+        required: false
+        default: "all"
+        options:
+          - "all"
+          - "opencga-analysis"
+          - "opencga-app"
+          - "opencga-catalog"
+          - "opencga-client"
+          - "opencga-core"
+          - "opencga-master"
+          - "opencga-server"
+          - "opencga-storage"
+          - "opencga-test"
       short_tests:
         type: boolean
         required: false
@@ -25,31 +53,49 @@ jobs:
     name: Test JUnit
     runs-on: ubuntu-22.04
     outputs:
-      profiles: ${{ steps.getter.outputs.profiles }}
+      profiles: ${{ steps.validate.outputs.profiles }}
+      modules: ${{ steps.validate.outputs.modules }}
+      hadoop: ${{ steps.validate.outputs.hadoop }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
       - name: Building string profiles to run
-        id: getter
+        id: validate
         run: |
           if [ -f "./.github/workflows/scripts/get_profiles.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_profiles.sh
-            echo "profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})" >> $GITHUB_OUTPUT
+            profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})
+            modules='["${{ inputs.module }}"]'
+            hadoop='["${{ inputs.hadoop }}"]'
+          
+            if [ "${{ inputs.hadoop }}" == "all" ]; then
+              hadoop='["hdp3.1", "hdi5.1", "emr6.1", "emr6.13"]'
+            elif [ "${{ inputs.module }}" == "all" ]; then
+              # Only execute modules with matrix strategy if we are testing one single hadoop profile
+              modules='["opencga-analysis", "opencga-app", "opencga-catalog", "opencga-client", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-test"]'
+            fi
+            echo "profiles=[\"$profiles\"]" >> $GITHUB_OUTPUT
+            echo "modules=$modules" >> $GITHUB_OUTPUT
+            echo "hadoop=$hadoop" >> $GITHUB_OUTPUT
+            echo "Executing testing profiles -> $profiles" >> $GITHUB_STEP_SUMMARY
+            echo "Modules -> $modules" >> $GITHUB_STEP_SUMMARY
+            echo "Hadoop -> $hadoop" >> $GITHUB_STEP_SUMMARY
+            #echo "matrix={\"hadoop\": $hadoop, \"modules\": $modules}" >> $GITHUB_OUTPUT
           fi
-
-  echo_log:
-    needs: get_profiles
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Echo profiles
-        id: log
-        run: echo "Executing testing profiles -> ${{ needs.get_profiles.outputs.profiles }}"
-
   test:
-    needs: [ get_profiles, echo_log ]
+    strategy:
+      fail-fast: false
+      matrix:
+        hadoop: ${{ fromJson(needs.get_profiles.outputs.hadoop) }}
+        module: ${{ fromJson(needs.get_profiles.outputs.modules) }}
+        profile: ${{ fromJson(needs.get_profiles.outputs.profiles) }}
+    needs: [ get_profiles ]
     uses: ./.github/workflows/test-analysis.yml
     with:
-      test_profile: ${{ needs.get_profiles.outputs.profiles }}
+      test_profile: ${{ matrix.profile }}
+      hadoop: ${{ matrix.hadoop }}
+      module: ${{ matrix.module }}
       mvn_opts: ${{ inputs.mvn_opts }}
+      sonar: false
     secrets: inherit
diff --git a/.github/workflows/long-test-analysis.yml b/.github/workflows/long-test-analysis.yml
@@ -9,9 +9,13 @@ on:
 
 jobs:
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        hadoop: [ "hdp3.1", "hdi5.1", "emr6.1", "emr6.13" ]
     uses: ./.github/workflows/test-analysis.yml
     secrets: inherit
     with:
-      test_profile: runShortTests,runMediumTests,runLongTests,
-
+      test_profile: runShortTests,runMediumTests,runLongTests
+      hadoop: ${{ matrix.hadoop }}
 
diff --git a/.github/workflows/manual-deploy-docker.yml b/.github/workflows/manual-deploy-docker.yml
@@ -10,6 +10,11 @@ on:
         description: "The tag for the new docker."
         type: string
         required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
 
 jobs:
   build:
@@ -18,12 +23,12 @@ jobs:
     outputs:
       version: ${{ steps.get_project_version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
           ref: "${{ inputs.branch }}"
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -32,11 +37,11 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
-      - uses: actions/upload-artifact@v3
+        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }},RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      - uses: actions/upload-artifact@v4
         with:
           name: build-folder
           path: build

diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml
@@ -10,6 +10,11 @@ on:
         description: "The tag for the new docker."
         type: string
         required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
 
 jobs:
   build:
@@ -18,12 +23,12 @@ jobs:
     outputs:
       version: ${{ steps.get_project_version.outputs.version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
           ref: "${{ inputs.branch }}"
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -32,11 +37,11 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
         run: mvn -T 2 clean install -DskipTests
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: build-folder
           path: build

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -23,7 +23,7 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ needs.build.outputs.version }}
     secrets: inherit
 
   deploy-python:

diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh
@@ -1,21 +1,25 @@
 #!/bin/bash
 
 BRANCH_NAME=$1
+HADOOP=${2:-hdp3.1}
 
-if [[ -z $BRANCH_NAME  ]]; then
+if [[ -z "$BRANCH_NAME"  ]]; then
   echo "The first parameter is mandatory and must be a valid branch name."
   exit 1
 fi
 
-
 function install(){
   local REPO=$1
   cd /home/runner/work/ || exit 2
   git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME"
   if [ -d "./$REPO" ]; then
     cd "$REPO" || exit 2
     echo "Branch name $BRANCH_NAME already exists."
-    mvn clean install -DskipTests
+    if [[ "$REPO" == "opencga-hadoop-thirdparty" ]]; then
+      ./dev/build.sh "$HADOOP"
+    else
+      mvn clean install -DskipTests
+    fi
   else
     echo "$CURRENT Branch is NOT EQUALS $BRANCH_NAME "
   fi
@@ -24,3 +28,4 @@ function install(){
 install "java-common-libs"
 install "biodata"
 install "cellbase"
+install "opencga-hadoop-thirdparty"
diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -1,14 +1,29 @@
 name: Build and test the project
+run-name: "Build and test the project. Hadoop flavour: ${{ inputs.hadoop }}. Test profile: ${{ inputs.test_profile }}. Test module: ${{ inputs.module }}"
 
 on:
   workflow_call:
     inputs:
       test_profile:
+        description: 'Maven test profile. Any combination of : [runShortTests, runMediumTests, runLongTests]'
         type: string
         required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
       mvn_opts:
         type: string
-
+        required: false
+        default: ""
+      sonar:
+        type: boolean
+        required: false
+        default: true
+      module:
+        type: string
+        description: "Maven modules to test. Empty means all. Only top-level modules. Example: 'opencga-storage'"
         required: false
         default: ""
     secrets:
@@ -19,12 +34,13 @@ jobs:
   analysis:
     name: Execute Sonar Analysis
     runs-on: ubuntu-22.04
+    if: ${{ inputs.sonar }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '17'
@@ -33,25 +49,25 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           else
              echo "./.github/workflows/scripts/get_same_branch.sh does not exist."
           fi
       - name: Test and Analyze
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: mvn -T 2 clean install -P hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
+        run: mvn -T 2 clean install -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
 
   test:
     name: Execute JUnit and Jacoco tests
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: '0'
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -60,7 +76,7 @@ jobs:
         run: |
           if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
             chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           else
              echo "./.github/workflows/scripts/get_same_branch.sh does not exist."
           fi
@@ -71,8 +87,10 @@ jobs:
         with:
           mongodb-version: 6.0
           mongodb-replica-set: rs-test
+      - name: Maven build
+        run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -P hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+        run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Publish Test Report
         uses: scacap/action-surefire-report@v1
         env:
@@ -81,9 +99,10 @@ jobs:
         ## https://docs.github.com/en/actions/learn-github-actions/expressions#cancelled
         if: success() || failure()
         with:
-          check_name: "Surefire tests report"
+          check_name: "Surefire tests report ${{ inputs.hadoop }} ${{ inputs.module }} ${{ inputs.test_profile }}"
           report_paths: './**/surefire-reports/TEST-*.xml'
           commit: '${{ github.sha }}'
           fail_on_test_failures: true
+          fail_if_no_tests: false
 
 
diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.0.0-SNAPSHOT</version>
+        <version>3.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -287,6 +287,12 @@
                 <version>${jetty-for-hadoop-test.version}</version>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty.http2</groupId>
+                <artifactId>http2-hpack</artifactId>
+                <version>${jetty-for-hadoop-test.version}</version>
+                <scope>test</scope>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.jetty</groupId>
                 <artifactId>jetty-io</artifactId>