Use internal snapshot repository for OSS snapshot images [5.4.z] (#781)

Backport of #774 - Use internal snapshot maven repository for getting OSS snapshot distribution zip - Use internal snapshot docker registry for OSS snapshot images Fixes https://hazelcast.atlassian.net/browse/DI-169
hazelcast · Jul 9, 2024 · 46c7619 · 46c7619
1 parent 204ff8c
commit 46c7619
Show file tree

Hide file tree

Showing 10 changed files with 106 additions and 63 deletions.
diff --git a/.github/scripts/assert.sh/assert.sh b/.github/scripts/assert.sh/assert.sh
@@ -178,7 +178,7 @@ assert_contain() {
   fi
 
   if [ -z "${haystack##*$needle*}" ]; then
-    log_success "Array [$(join_by "," $haystack)] contains $needle :: $msg" || true
+    log_success "[$(join_by "," $haystack)] contains $needle :: $msg" || true
     return 0
   else
     [ "${#msg}" -gt 0 ] && log_failure "Array [$(join_by "," $haystack)] doesn't contain $needle :: $msg" || true

diff --git a/.github/scripts/ee-build.functions_tests.sh b/.github/scripts/ee-build.functions_tests.sh
@@ -36,9 +36,18 @@ function assert_get_hz_dist_zip {
   assert_eq "$expected_url" "$actual_url" "Expected URL for variant \"$hz_variant\", version \"$hz_version\"" || TESTS_RESULT=$?
 }
 
+function assert_get_hz_dist_zip_for_snapshot {
+  local hz_variant=$1
+  local hz_version=$2
+  local expected_url=$3
+  local actual_url=$(get_hz_dist_zip "$hz_variant" "$hz_version")
+  assert_contain "$actual_url" "$expected_url" "Expected URL for variant \"$hz_variant\", version \"$hz_version\" should contain $expected_url" || TESTS_RESULT=$?
+}
+
 log_header "Tests for get_hz_dist_zip"
 assert_get_hz_dist_zip slim 5.4.0 https://repository.hazelcast.com/release/com/hazelcast/hazelcast-enterprise-distribution/5.4.0/hazelcast-enterprise-distribution-5.4.0-slim.zip
 assert_get_hz_dist_zip "" 5.4.0 https://repository.hazelcast.com/release/com/hazelcast/hazelcast-enterprise-distribution/5.4.0/hazelcast-enterprise-distribution-5.4.0.zip
-assert_get_hz_dist_zip "" 5.4.0-SNAPSHOT https://repository.hazelcast.com/snapshot/com/hazelcast/hazelcast-enterprise-distribution/5.4.0-SNAPSHOT/hazelcast-enterprise-distribution-5.4.0-20240301.103418-1664.zip
+CURRENT_SNAPSHOT_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' "$SCRIPT_DIR/../../hazelcast-enterprise/Dockerfile")
+assert_get_hz_dist_zip_for_snapshot "" "$CURRENT_SNAPSHOT_VERSION" "https://repository.hazelcast.com/snapshot/com/hazelcast/hazelcast-enterprise-distribution/$CURRENT_SNAPSHOT_VERSION/hazelcast-enterprise-distribution-${CURRENT_SNAPSHOT_VERSION%-SNAPSHOT}"
 
 assert_eq 0 "$TESTS_RESULT" "All tests should pass"
diff --git a/.github/scripts/oss-build.functions.sh b/.github/scripts/oss-build.functions.sh
@@ -11,9 +11,7 @@ function get_hz_dist_zip() {
 
   if [[ "${hz_version}" == *"SNAPSHOT"* ]]
   then
-      # DI-95 - Do not rely on the OSS distribution zip in the hazelcast-docker PR builder
-      # https://hazelcast.atlassian.net/browse/DI-95
-      url="$(aws s3 presign "s3://hazelcast/distribution-snapshot/hazelcast-${hz_version}${suffix}.zip" --expires-in 600)"
+      url="https://${HZ_SNAPSHOT_INTERNAL_USERNAME}:${HZ_SNAPSHOT_INTERNAL_PASSWORD}@repository.hazelcast.com/snapshot-internal/com/hazelcast/hazelcast-distribution/${hz_version}/hazelcast-distribution-${hz_version}${suffix}.zip"
   else
       url="https://repo1.maven.org/maven2/com/hazelcast/hazelcast-distribution/${hz_version}/hazelcast-distribution-${hz_version}${suffix}.zip"
   fi

diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
@@ -11,9 +11,13 @@ env:
   docker_log_file_ee: docker-hazelcast-ee-test.log
 
 jobs:
-  build-pr:
+  prepare:
     runs-on: ubuntu-latest
-    name: Build with default JDK
+    name: Prepare environment
+    outputs:
+      HZ_VERSION_OSS: ${{ steps.get_oss_vars.outputs.HZ_VERSION_OSS }}
+      HZ_VERSION_EE: ${{ steps.get_ee_vars.outputs.HZ_VERSION_EE }}
+      HAZELCAST_EE_ZIP_URL: ${{ steps.get_ee_vars.outputs.HAZELCAST_EE_ZIP_URL }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -30,35 +34,52 @@ jobs:
       - name: Install xmllint
         uses: ./.github/actions/install-xmllint
 
-      - name: Set up Docker
-        uses: docker/setup-buildx-action@v3
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v3
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: 'us-east-1'
-
       - name: Test scripts
         run: |
           .github/scripts/test_scripts.sh
 
-      - name: Get OSS dist ZIP URL
+      - name: Setup OSS variables
+        id: get_oss_vars
         run: |
-          . .github/scripts/oss-build.functions.sh
-          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-oss/Dockerfile)
-          echo "HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" ${HZ_VERSION})" >> $GITHUB_ENV
+          HZ_VERSION_OSS=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-oss/Dockerfile)
+          echo "HZ_VERSION_OSS=$HZ_VERSION_OSS" >> $GITHUB_OUTPUT
+
+      - name: Setup EE variables
+        id: get_ee_vars
+        run: |
+          HZ_VERSION_EE=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-enterprise/Dockerfile)
+          . .github/scripts/ee-build.functions.sh
+          echo "HZ_VERSION_EE=$HZ_VERSION_EE" >> $GITHUB_OUTPUT
+          echo "HAZELCAST_EE_ZIP_URL=$(get_hz_dist_zip "" ${HZ_VERSION_EE})" >> $GITHUB_OUTPUT
+
+  build-pr:
+    runs-on: ubuntu-latest
+    name: Build with default JDK
+    needs: [ prepare ]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
 
       - name: Build OSS image
         run: |
           DOCKER_PATH=hazelcast-oss
-          # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version
-          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile)
-
+          HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_OSS }}"
+          
+          # duplicated block as GH doesn't support passing sensitive data between jobs
+          . .github/scripts/oss-build.functions.sh
+          export HZ_SNAPSHOT_INTERNAL_PASSWORD=${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }}
+          export HZ_SNAPSHOT_INTERNAL_USERNAME=${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }}
+          HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}")
+          
+          curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output $DOCKER_PATH/hazelcast-distribution.zip; 
+          
           docker buildx build --load \
           --build-arg HZ_VERSION=${HZ_VERSION} \
-          --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \
           --tag hazelcast-oss:test \
           ${DOCKER_PATH}
 
@@ -67,21 +88,14 @@ jobs:
         run: |
           .github/scripts/simple-smoke-test.sh hazelcast-oss:test ${{ env.test_container_name_oss }} oss
 
-      - name: Get EE dist ZIP URL
-        run: |
-          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-enterprise/Dockerfile)
-          . .github/scripts/ee-build.functions.sh
-          echo "HAZELCAST_EE_ZIP_URL=$(get_hz_dist_zip "" ${HZ_VERSION})" >> $GITHUB_ENV
-
       - name: Build Test EE image
         run: |
           DOCKER_PATH=hazelcast-enterprise
-          # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version
-          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile)
-
+          HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_EE }}"
+          curl --fail --silent --show-error --location "${{ needs.prepare.outputs.HAZELCAST_EE_ZIP_URL }}" --output $DOCKER_PATH/hazelcast-enterprise-distribution.zip;
+          
           docker buildx build --load \
           --build-arg HZ_VERSION=${HZ_VERSION} \
-          --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_EE_ZIP_URL \
           --tag hazelcast-ee:test \
           ${DOCKER_PATH}
 
@@ -111,7 +125,7 @@ jobs:
 
   build-pr-custom-jdk:
     runs-on: ubuntu-latest
-    needs: jdks
+    needs: [ jdks, prepare ]
     name: Build with jdk-${{ matrix.jdk }}
     strategy:
       fail-fast: false
@@ -129,23 +143,22 @@ jobs:
       - name: Set up Docker
         uses: docker/setup-buildx-action@v3
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v3
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: 'us-east-1'
-
       - name: Build OSS image
         run: |
           DOCKER_PATH=hazelcast-oss
-          # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version
-          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile)
+          HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_OSS }}"
+          
+          # duplicated block as GH doesn't support passing sensitive data between jobs
+          . .github/scripts/oss-build.functions.sh
+          export HZ_SNAPSHOT_INTERNAL_PASSWORD=${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }}
+          export HZ_SNAPSHOT_INTERNAL_USERNAME=${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }}
+          HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}")
+          
+          curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output $DOCKER_PATH/hazelcast-distribution.zip;
 
           docker buildx build --load \
           --build-arg JDK_VERSION=${{ matrix.jdk }} \
           --build-arg HZ_VERSION=${HZ_VERSION} \
-          --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \
           --tag hazelcast-oss:test \
           ${DOCKER_PATH}
 
@@ -157,13 +170,12 @@ jobs:
       - name: Build Test EE image
         run: |
           DOCKER_PATH=hazelcast-enterprise
-          # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version
-          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile)
+          HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_EE }}"
+          curl --fail --silent --show-error --location "${{ needs.prepare.outputs.HAZELCAST_EE_ZIP_URL }}" --output $DOCKER_PATH/hazelcast-enterprise-distribution.zip;
 
           docker buildx build --load \
           --build-arg JDK_VERSION=${{ matrix.jdk }} \
           --build-arg HZ_VERSION=${HZ_VERSION} \
-          --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_EE_ZIP_URL \
           --tag hazelcast-ee:test \
           ${DOCKER_PATH}
 

diff --git a/.github/workflows/oss_latest_snapshot_push.yml b/.github/workflows/oss_latest_snapshot_push.yml
@@ -28,7 +28,14 @@ jobs:
           - variant: slim
           - variant: ''
     env:
-      DOCKER_ORG: hazelcast
+      DOCKER_REGISTRY: ${{ secrets.HZ_SNAPSHOT_INTERNAL_DOCKER_REGISTRY }}
+      DOCKER_USERNAME: ${{ secrets.JFROG_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.JFROG_PASSWORD }}
+
+      # required by OSS get_hz_dist_zip function
+      HZ_SNAPSHOT_INTERNAL_USERNAME: ${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }}
+      HZ_SNAPSHOT_INTERNAL_PASSWORD: ${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }}
+
       HZ_VERSION: ${{ github.event.inputs.HZ_VERSION }}
     steps:
       - name: Compute Suffix
@@ -62,16 +69,17 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: 'us-east-1'
 
-      - name: Get OSS dist ZIP URL
+      - name: Get OSS dist ZIP
         run: |
           . .github/scripts/oss-build.functions.sh
-          echo "HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "${{ matrix.variant }}" "${HZ_VERSION}")" >> $GITHUB_ENV
+          HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "${{ matrix.variant }}" "${HZ_VERSION}")
+        
+          curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output hazelcast-oss/hazelcast-distribution.zip;
 
       - name: Build Test OSS image
         run: |
           docker buildx build --load \
             --build-arg JDK_VERSION=${{ matrix.jdk }} \
-            --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \
             --tag hazelcast-oss:test \
             hazelcast-oss
 
@@ -95,8 +103,8 @@ jobs:
           path: |
             ${{ env.DOCKER_LOG_FILE_OSS }}
 
-      - name: Login to Docker Hub
-        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+      - name: Login to Docker Registry
+        run: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin $DOCKER_REGISTRY
 
       - name: Build/Push OSS image
         run: |
@@ -110,7 +118,7 @@ jobs:
           fi
           
           DOCKER_DIR=hazelcast-oss
-          IMAGE_NAME=${{ env.DOCKER_ORG }}/hazelcast
+          IMAGE_NAME=$DOCKER_REGISTRY/docker/hazelcast/hazelcast
           DEFAULT_JDK="$(get_default_jdk $DOCKER_DIR)"
 
           TAGS_TO_PUSH=$(augment_with_suffixed_tags "${VERSIONS[*]}" "${{ env.SUFFIX }}" "${{ matrix.jdk }}" "$DEFAULT_JDK")
@@ -125,7 +133,6 @@ jobs:
           docker buildx build --push \
             --build-arg JDK_VERSION=${{ matrix.jdk }} \
             --label hazelcast.revision=${{ github.event.inputs.HZ_REVISION }} \
-            --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \
             $TAGS_ARG \
             --platform=${PLATFORMS} $DOCKER_DIR
       - name: Slack notification

diff --git a/.github/workflows/scheduled_vulnerability_scan.yml b/.github/workflows/scheduled_vulnerability_scan.yml
@@ -15,5 +15,4 @@ jobs:
         uses: ./.github/workflows/vulnerability_scan_subworkflow.yml
         with:
             ref: ${{ matrix.ref }}
-        secrets:
-            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        secrets: inherit
diff --git a/.github/workflows/vulnerability_scan.yml b/.github/workflows/vulnerability_scan.yml
@@ -12,6 +12,5 @@ jobs:
         uses: ./.github/workflows/vulnerability_scan_subworkflow.yml
         with:
             ref: ${{ github.ref }}
-        secrets:
-            SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        secrets: inherit
 
diff --git a/.github/workflows/vulnerability_scan_subworkflow.yml b/.github/workflows/vulnerability_scan_subworkflow.yml
@@ -29,6 +29,15 @@ jobs:
       - name: Install xmllint
         uses: ./.github/actions/install-xmllint
 
+      - name: Get OSS dist ZIP
+        run: |
+          . .github/scripts/oss-build.functions.sh
+          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-oss/Dockerfile)
+          export HZ_SNAPSHOT_INTERNAL_PASSWORD=${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }}
+          export HZ_SNAPSHOT_INTERNAL_USERNAME=${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }}
+          HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}")        
+          curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output hazelcast-oss/hazelcast-distribution.zip;
+
       - name: Build OSS image
         run: |
             docker build -t hazelcast/oss:${{ github.sha }} hazelcast-oss
@@ -70,6 +79,16 @@ jobs:
         with:
           ref: ${{ inputs.ref }}
 
+      - name: Install xmllint
+        uses: ./.github/actions/install-xmllint
+
+      - name: Get EE dist ZIP
+        run: |
+          . .github/scripts/ee-build.functions.sh
+          HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-enterprise/Dockerfile)          
+          HAZELCAST_EE_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}")        
+          curl --fail --silent --show-error --location "$HAZELCAST_EE_ZIP_URL" --output hazelcast-enterprise/hazelcast-enterprise-distribution.zip;
+
       - name: Build EE image
         run: |
           docker build -t hazelcast/ee:${{ github.sha }} hazelcast-enterprise

diff --git a/hazelcast-enterprise/Dockerfile b/hazelcast-enterprise/Dockerfile
@@ -50,7 +50,7 @@ RUN echo "Installing new packages" \
         fi; \
         echo "Downloading Hazelcast distribution zip from ${HAZELCAST_ZIP_URL}..."; \
         mkdir --parents ${HZ_HOME}; \
-        curl -sfL ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-enterprise-distribution.zip; \
+        curl --fail --silent --show-error --location ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-enterprise-distribution.zip; \
     else \
            echo "Using local hazelcast-enterprise-distribution.zip"; \
     fi \

diff --git a/hazelcast-oss/Dockerfile b/hazelcast-oss/Dockerfile
@@ -39,7 +39,7 @@ RUN echo "Upgrading APK packages" \
        fi; \
        echo "Downloading Hazelcast distribution zip from ${HAZELCAST_ZIP_URL}..."; \
        mkdir --parents ${HZ_HOME}; \
-       curl -sfL ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-distribution.zip; \
+       curl --fail --silent --show-error --location ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-distribution.zip; \
     else \
            echo "Using local hazelcast-distribution.zip"; \
     fi \