Merge branch 'develop' into TASK-7972'

Author: dapregi

.github/workflows/build-clients.yml

@@ -2,6 +2,11 @@ name: Build clients for opencga
 
 on:
   workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
 
 jobs:
   build-clients:
@@ -10,10 +15,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: '10'
+          ref: ${{ github.event.inputs.branch }}
       - name: Build clients
         run: |
-          chmod +x build_clients.sh 
-          ./build_clients.sh
+          chmod +x client-builder.sh 
+          ./client-builder.sh
       - name: Upload clients folder as artifact
         uses: actions/upload-artifact@v4
         with:

.github/workflows/check-junit-test.yml

@@ -32,18 +32,22 @@ on:
           - "opencga-storage"
           - "opencga-test"
       short_tests:
+        description: 'Run short tests.'
         type: boolean
         required: false
         default: true
       medium_tests:
+        description: 'Run medium tests.'
         type: boolean
         required: false
         default: false
       long_tests:
+        description: 'Run long tests.'
         type: boolean
         required: false
         default: false
       mvn_opts:
+        description: 'Maven options.'
         type: string
         required: false
         default: ""

.github/workflows/clients-bulder.yml

@@ -13,22 +13,31 @@ on:
           - r
           - python
           - javascript
-
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
 jobs:
   call-build-r-docker:
     if: ${{ github.event.inputs.language == 'r' || github.event.inputs.language == 'all' }}
     name: Call Reusable Build R Client Workflow
     uses: opencb/opencga/.github/workflows/reusable-build-r-client.yml@develop
+    with:
+      branch: ${{ github.event.inputs.branch }}
     secrets: inherit
 
   call-build-python-docker:
     if: ${{ github.event.inputs.language == 'python' || github.event.inputs.language == 'all' }}
     name: Call Reusable Build Python Client Workflow
     uses: opencb/opencga/.github/workflows/reusable-build-python-client.yml@develop
+    with:
+      branch: ${{ github.event.inputs.branch }}
     secrets: inherit
 
   call-build-javascript-docker:
     if: ${{ github.event.inputs.language == 'javascript' || github.event.inputs.language == 'all' }}
     name: Call Reusable Build JavaScript Client Workflow
     uses: opencb/opencga/.github/workflows/reusable-build-javascript-client.yml@develop
+    with:
+      branch: ${{ github.event.inputs.branch }}
     secrets: inherit

.github/workflows/deploy-ext-tools.yml

@@ -0,0 +1,49 @@
+name: Push ext-tools for develop branches
+
+on:
+  push:
+    branches:
+      - develop
+      - release-*
+      - TASK-*
+    tags:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  check-push-ext-tools-docker-tag:
+    runs-on: ubuntu-latest
+    outputs:
+      exists: ${{ steps.check.outputs.exists }}
+      version: ${{ steps.check.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Make script executable
+        run: chmod +x .github/workflows/scripts/check_docker_tag.sh
+      - name: Check Docker tag existence
+        id: check
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_HUB_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+        run: |
+          .github/workflows/scripts/check_docker_tag.sh
+      - uses: docker/login-action@v3
+        if: steps.check.outputs.exists == 'false'
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          logout: true
+      - name: Deploy in Docker Hub
+        if: steps.check.outputs.exists == 'false'
+        run: python3 ./opencga-app/app/cloud/docker/docker-build.py push --images ext-tools --tag ${{ steps.check.outputs.version }} --org opencb
+
+
+
+
+

.github/workflows/develop.yml

@@ -11,18 +11,26 @@ jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga
+
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
     needs: build
     with:
-      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
+      maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga -U -DskipTests
     secrets: inherit
 
-  deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+  deploy-zetta-docker:
+    uses: opencb/java-common-libs/.github/workflows/deploy-zetta-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow,init
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }}
     secrets: inherit
+
+  deploy-docker-workflow:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images workflow --tag ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/docker-ext-tools.yml renamed to .github/workflows/dockerfile-ext-tools.yml

@@ -2,10 +2,6 @@ name: Deploy Docker Ext-Tools
 
 on:
   push:
-    branches:
-      - develop
-      - release-*
-      - TASK-*
     paths:
       - opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile
 

.github/workflows/manual-deploy-docker.yml

@@ -45,15 +45,22 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip  --no-transfer-progress
+        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip  --no-transfer-progress -pl ':opencga-app' --also-make
       - uses: actions/upload-artifact@v4
         with:
           name: build-folder
           path: build
 
-  deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+  deploy-zetta-docker:
+    uses: opencb/java-common-libs/.github/workflows/deploy-zetta-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow,init --tag ${{ inputs.tag }}
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }}
     secrets: inherit
+
+  deploy-docker-workflow:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images workflow --tag ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/manual-deploy-r-builder.yml

@@ -7,7 +7,7 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
       maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
-
+      build_folder: build-
 
   build-and-push-docker:
     runs-on: ubuntu-latest
@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: ${{ inputs.build_folder }}
+          name: build-folder
           path: build
 
       - name: Set up Docker Buildx
@@ -36,11 +36,10 @@ jobs:
         run: |
           TAG=$(grep 'Version:' build/clients/R/DESCRIPTION | cut -d' ' -f2)
           echo "version=$TAG" >> $GITHUB_OUTPUT
-
       - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
-          context: .
+          context: "{{defaultContext}}:opencga-app/app/cloud/docker/opencga-r-builder/"
           push: true
           tags: |
             opencb/opencga-r-builder:latest

.github/workflows/pull-request-approved.yml

@@ -41,4 +41,3 @@ jobs:
       branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
       task: ${{ github.event.pull_request.head.ref }}
     secrets: inherit
-

.github/workflows/release.yml

@@ -11,15 +11,15 @@ jobs:
   build-hdp:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl ':opencga-app' --also-make
       build_folder: build-folder
 
-  # Add the deploy-docker-hdp job that depends on the build-hdp job
-  deploy-docker-hdp:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+  # Add the deploy-zetta-docker job that depends on the build-hdp job
+  deploy-zetta-docker:
+    uses: opencb/java-common-libs/.github/workflows/deploy-zetta-docker-hub-workflow.yml@develop
     needs: build-hdp
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow,init --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1"
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1"
       build_folder: build-folder
     secrets: inherit
 
@@ -62,62 +62,63 @@ jobs:
   # Add the release job that depends on all the previous jobs
   release:
     uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
-    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python, deploy-docker-python-notebook]
+    needs: [ build-hdp, deploy-zetta-docker, deploy-maven, deploy-python, deploy-docker-python-notebook]
     with:
       artifact: build-folder
       file: |
         opencga-client-${{ needs.build-hdp.outputs.version }}.tar.gz
-        clients/R/opencgaR_${{ needs.build-hdp.outputs.version }}.tar.gz
+        dist/*.*
 
   deploy-docker-python-notebook:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: [ build-hdp, deploy-python ]
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ needs.build-hdp.outputs.version }} --docker-build-args "--build-arg VERSION=${{ needs.build-hdp.outputs.version }}"
+      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ needs.build-hdp.outputs.version }} --docker-build-args "--build-arg VERSION=${{ needs.build-hdp.outputs.version }}" --org opencb
     secrets: inherit
 
   # Compile and deploy other hadoop flavours
   # Add the build-hdi and deploy-docker-hdi jobs
   build-hdi:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P hdi5.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      maven_opts: -P hdi5.1-Dopencga.war.name=opencga -Dcheckstyle.skip
       build_folder: build-folder-hdi
 
   deploy-docker-hdi:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-zetta-docker-hub-workflow.yml@develop
     needs: build-hdi
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdi.outputs.version }}-hdi5.1"
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdi.outputs.version }}-hdi5.1" --org zettagenomics
       build_folder: build-folder-hdi
     secrets: inherit
 
   # Add the build-emr and deploy-docker-emr jobs
   build-emr:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P emr6.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      maven_opts: -P emr6.1 -Dopencga.war.name=opencga -Dcheckstyle.skip
       build_folder: build-folder-emr
 
   deploy-docker-emr:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-zetta-docker-hub-workflow.yml@develop
     needs: build-emr
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr.outputs.version }}-emr6.1"
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr.outputs.version }}-emr6.1" --org zettagenomics
       build_folder: build-folder-emr
     secrets: inherit
 
   # Add the build-emr613 and deploy-docker-emr613 jobs
   build-emr613:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P emr6.13,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      maven_opts: -P emr6.13 -Dopencga.war.name=opencga -Dcheckstyle.skip
       build_folder: build-folder-emr613
 
   deploy-docker-emr613:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    uses: opencb/java-common-libs/.github/workflows/deploy-zetta-docker-hub-workflow.yml@develop
     needs: build-emr613
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr613.outputs.version }}-emr6.13"
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr613.outputs.version }}-emr6.13" --org zettagenomics
       build_folder: build-folder-emr613
     secrets: inherit
+