From 46c7619aff50e86d7370dd7343df5dd56c02779f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Dziedziul?= Date: Tue, 9 Jul 2024 22:52:23 +0200 Subject: [PATCH] Use internal snapshot repository for OSS snapshot images [5.4.z] (#781) Backport of https://github.com/hazelcast/hazelcast-docker/pull/774 - Use internal snapshot maven repository for getting OSS snapshot distribution zip - Use internal snapshot docker registry for OSS snapshot images Fixes https://hazelcast.atlassian.net/browse/DI-169 --- .github/scripts/assert.sh/assert.sh | 2 +- .github/scripts/ee-build.functions_tests.sh | 11 +- .github/scripts/oss-build.functions.sh | 4 +- .github/workflows/build-pr.yml | 100 ++++++++++-------- .../workflows/oss_latest_snapshot_push.yml | 23 ++-- .../scheduled_vulnerability_scan.yml | 3 +- .github/workflows/vulnerability_scan.yml | 3 +- .../vulnerability_scan_subworkflow.yml | 19 ++++ hazelcast-enterprise/Dockerfile | 2 +- hazelcast-oss/Dockerfile | 2 +- 10 files changed, 106 insertions(+), 63 deletions(-) diff --git a/.github/scripts/assert.sh/assert.sh b/.github/scripts/assert.sh/assert.sh index 401ea869..a77ce4ea 100644 --- a/.github/scripts/assert.sh/assert.sh +++ b/.github/scripts/assert.sh/assert.sh @@ -178,7 +178,7 @@ assert_contain() { fi if [ -z "${haystack##*$needle*}" ]; then - log_success "Array [$(join_by "," $haystack)] contains $needle :: $msg" || true + log_success "[$(join_by "," $haystack)] contains $needle :: $msg" || true return 0 else [ "${#msg}" -gt 0 ] && log_failure "Array [$(join_by "," $haystack)] doesn't contain $needle :: $msg" || true diff --git a/.github/scripts/ee-build.functions_tests.sh b/.github/scripts/ee-build.functions_tests.sh index d288304f..a9da9b2b 100755 --- a/.github/scripts/ee-build.functions_tests.sh +++ b/.github/scripts/ee-build.functions_tests.sh @@ -36,9 +36,18 @@ function assert_get_hz_dist_zip { assert_eq "$expected_url" "$actual_url" "Expected URL for variant \"$hz_variant\", version \"$hz_version\"" || TESTS_RESULT=$? } +function assert_get_hz_dist_zip_for_snapshot { + local hz_variant=$1 + local hz_version=$2 + local expected_url=$3 + local actual_url=$(get_hz_dist_zip "$hz_variant" "$hz_version") + assert_contain "$actual_url" "$expected_url" "Expected URL for variant \"$hz_variant\", version \"$hz_version\" should contain $expected_url" || TESTS_RESULT=$? +} + log_header "Tests for get_hz_dist_zip" assert_get_hz_dist_zip slim 5.4.0 https://repository.hazelcast.com/release/com/hazelcast/hazelcast-enterprise-distribution/5.4.0/hazelcast-enterprise-distribution-5.4.0-slim.zip assert_get_hz_dist_zip "" 5.4.0 https://repository.hazelcast.com/release/com/hazelcast/hazelcast-enterprise-distribution/5.4.0/hazelcast-enterprise-distribution-5.4.0.zip -assert_get_hz_dist_zip "" 5.4.0-SNAPSHOT https://repository.hazelcast.com/snapshot/com/hazelcast/hazelcast-enterprise-distribution/5.4.0-SNAPSHOT/hazelcast-enterprise-distribution-5.4.0-20240301.103418-1664.zip +CURRENT_SNAPSHOT_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' "$SCRIPT_DIR/../../hazelcast-enterprise/Dockerfile") +assert_get_hz_dist_zip_for_snapshot "" "$CURRENT_SNAPSHOT_VERSION" "https://repository.hazelcast.com/snapshot/com/hazelcast/hazelcast-enterprise-distribution/$CURRENT_SNAPSHOT_VERSION/hazelcast-enterprise-distribution-${CURRENT_SNAPSHOT_VERSION%-SNAPSHOT}" assert_eq 0 "$TESTS_RESULT" "All tests should pass" diff --git a/.github/scripts/oss-build.functions.sh b/.github/scripts/oss-build.functions.sh index ffb4a5af..76a1f3c3 100644 --- a/.github/scripts/oss-build.functions.sh +++ b/.github/scripts/oss-build.functions.sh @@ -11,9 +11,7 @@ function get_hz_dist_zip() { if [[ "${hz_version}" == *"SNAPSHOT"* ]] then - # DI-95 - Do not rely on the OSS distribution zip in the hazelcast-docker PR builder - # https://hazelcast.atlassian.net/browse/DI-95 - url="$(aws s3 presign "s3://hazelcast/distribution-snapshot/hazelcast-${hz_version}${suffix}.zip" --expires-in 600)" + url="https://${HZ_SNAPSHOT_INTERNAL_USERNAME}:${HZ_SNAPSHOT_INTERNAL_PASSWORD}@repository.hazelcast.com/snapshot-internal/com/hazelcast/hazelcast-distribution/${hz_version}/hazelcast-distribution-${hz_version}${suffix}.zip" else url="https://repo1.maven.org/maven2/com/hazelcast/hazelcast-distribution/${hz_version}/hazelcast-distribution-${hz_version}${suffix}.zip" fi diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 976f8f2e..623dac03 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -11,9 +11,13 @@ env: docker_log_file_ee: docker-hazelcast-ee-test.log jobs: - build-pr: + prepare: runs-on: ubuntu-latest - name: Build with default JDK + name: Prepare environment + outputs: + HZ_VERSION_OSS: ${{ steps.get_oss_vars.outputs.HZ_VERSION_OSS }} + HZ_VERSION_EE: ${{ steps.get_ee_vars.outputs.HZ_VERSION_EE }} + HAZELCAST_EE_ZIP_URL: ${{ steps.get_ee_vars.outputs.HAZELCAST_EE_ZIP_URL }} steps: - name: Checkout Code uses: actions/checkout@v4 @@ -30,35 +34,52 @@ jobs: - name: Install xmllint uses: ./.github/actions/install-xmllint - - name: Set up Docker - uses: docker/setup-buildx-action@v3 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: 'us-east-1' - - name: Test scripts run: | .github/scripts/test_scripts.sh - - name: Get OSS dist ZIP URL + - name: Setup OSS variables + id: get_oss_vars run: | - . .github/scripts/oss-build.functions.sh - HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-oss/Dockerfile) - echo "HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" ${HZ_VERSION})" >> $GITHUB_ENV + HZ_VERSION_OSS=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-oss/Dockerfile) + echo "HZ_VERSION_OSS=$HZ_VERSION_OSS" >> $GITHUB_OUTPUT + + - name: Setup EE variables + id: get_ee_vars + run: | + HZ_VERSION_EE=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-enterprise/Dockerfile) + . .github/scripts/ee-build.functions.sh + echo "HZ_VERSION_EE=$HZ_VERSION_EE" >> $GITHUB_OUTPUT + echo "HAZELCAST_EE_ZIP_URL=$(get_hz_dist_zip "" ${HZ_VERSION_EE})" >> $GITHUB_OUTPUT + + build-pr: + runs-on: ubuntu-latest + name: Build with default JDK + needs: [ prepare ] + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 - name: Build OSS image run: | DOCKER_PATH=hazelcast-oss - # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version - HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile) - + HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_OSS }}" + + # duplicated block as GH doesn't support passing sensitive data between jobs + . .github/scripts/oss-build.functions.sh + export HZ_SNAPSHOT_INTERNAL_PASSWORD=${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }} + export HZ_SNAPSHOT_INTERNAL_USERNAME=${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }} + HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}") + + curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output $DOCKER_PATH/hazelcast-distribution.zip; + docker buildx build --load \ --build-arg HZ_VERSION=${HZ_VERSION} \ - --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \ --tag hazelcast-oss:test \ ${DOCKER_PATH} @@ -67,21 +88,14 @@ jobs: run: | .github/scripts/simple-smoke-test.sh hazelcast-oss:test ${{ env.test_container_name_oss }} oss - - name: Get EE dist ZIP URL - run: | - HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-enterprise/Dockerfile) - . .github/scripts/ee-build.functions.sh - echo "HAZELCAST_EE_ZIP_URL=$(get_hz_dist_zip "" ${HZ_VERSION})" >> $GITHUB_ENV - - name: Build Test EE image run: | DOCKER_PATH=hazelcast-enterprise - # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version - HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile) - + HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_EE }}" + curl --fail --silent --show-error --location "${{ needs.prepare.outputs.HAZELCAST_EE_ZIP_URL }}" --output $DOCKER_PATH/hazelcast-enterprise-distribution.zip; + docker buildx build --load \ --build-arg HZ_VERSION=${HZ_VERSION} \ - --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_EE_ZIP_URL \ --tag hazelcast-ee:test \ ${DOCKER_PATH} @@ -111,7 +125,7 @@ jobs: build-pr-custom-jdk: runs-on: ubuntu-latest - needs: jdks + needs: [ jdks, prepare ] name: Build with jdk-${{ matrix.jdk }} strategy: fail-fast: false @@ -129,23 +143,22 @@ jobs: - name: Set up Docker uses: docker/setup-buildx-action@v3 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: 'us-east-1' - - name: Build OSS image run: | DOCKER_PATH=hazelcast-oss - # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version - HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile) + HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_OSS }}" + + # duplicated block as GH doesn't support passing sensitive data between jobs + . .github/scripts/oss-build.functions.sh + export HZ_SNAPSHOT_INTERNAL_PASSWORD=${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }} + export HZ_SNAPSHOT_INTERNAL_USERNAME=${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }} + HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}") + + curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output $DOCKER_PATH/hazelcast-distribution.zip; docker buildx build --load \ --build-arg JDK_VERSION=${{ matrix.jdk }} \ --build-arg HZ_VERSION=${HZ_VERSION} \ - --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \ --tag hazelcast-oss:test \ ${DOCKER_PATH} @@ -157,13 +170,12 @@ jobs: - name: Build Test EE image run: | DOCKER_PATH=hazelcast-enterprise - # Extract from Dockerfile to avoid duplicate hardcoding of the latest SNAPSHOT version - HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' ${DOCKER_PATH}/Dockerfile) + HZ_VERSION="${{ needs.prepare.outputs.HZ_VERSION_EE }}" + curl --fail --silent --show-error --location "${{ needs.prepare.outputs.HAZELCAST_EE_ZIP_URL }}" --output $DOCKER_PATH/hazelcast-enterprise-distribution.zip; docker buildx build --load \ --build-arg JDK_VERSION=${{ matrix.jdk }} \ --build-arg HZ_VERSION=${HZ_VERSION} \ - --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_EE_ZIP_URL \ --tag hazelcast-ee:test \ ${DOCKER_PATH} diff --git a/.github/workflows/oss_latest_snapshot_push.yml b/.github/workflows/oss_latest_snapshot_push.yml index bfc4b847..22b5b419 100644 --- a/.github/workflows/oss_latest_snapshot_push.yml +++ b/.github/workflows/oss_latest_snapshot_push.yml @@ -28,7 +28,14 @@ jobs: - variant: slim - variant: '' env: - DOCKER_ORG: hazelcast + DOCKER_REGISTRY: ${{ secrets.HZ_SNAPSHOT_INTERNAL_DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.JFROG_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.JFROG_PASSWORD }} + + # required by OSS get_hz_dist_zip function + HZ_SNAPSHOT_INTERNAL_USERNAME: ${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }} + HZ_SNAPSHOT_INTERNAL_PASSWORD: ${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }} + HZ_VERSION: ${{ github.event.inputs.HZ_VERSION }} steps: - name: Compute Suffix @@ -62,16 +69,17 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: 'us-east-1' - - name: Get OSS dist ZIP URL + - name: Get OSS dist ZIP run: | . .github/scripts/oss-build.functions.sh - echo "HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "${{ matrix.variant }}" "${HZ_VERSION}")" >> $GITHUB_ENV + HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "${{ matrix.variant }}" "${HZ_VERSION}") + + curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output hazelcast-oss/hazelcast-distribution.zip; - name: Build Test OSS image run: | docker buildx build --load \ --build-arg JDK_VERSION=${{ matrix.jdk }} \ - --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \ --tag hazelcast-oss:test \ hazelcast-oss @@ -95,8 +103,8 @@ jobs: path: | ${{ env.DOCKER_LOG_FILE_OSS }} - - name: Login to Docker Hub - run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + - name: Login to Docker Registry + run: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin $DOCKER_REGISTRY - name: Build/Push OSS image run: | @@ -110,7 +118,7 @@ jobs: fi DOCKER_DIR=hazelcast-oss - IMAGE_NAME=${{ env.DOCKER_ORG }}/hazelcast + IMAGE_NAME=$DOCKER_REGISTRY/docker/hazelcast/hazelcast DEFAULT_JDK="$(get_default_jdk $DOCKER_DIR)" TAGS_TO_PUSH=$(augment_with_suffixed_tags "${VERSIONS[*]}" "${{ env.SUFFIX }}" "${{ matrix.jdk }}" "$DEFAULT_JDK") @@ -125,7 +133,6 @@ jobs: docker buildx build --push \ --build-arg JDK_VERSION=${{ matrix.jdk }} \ --label hazelcast.revision=${{ github.event.inputs.HZ_REVISION }} \ - --build-arg HAZELCAST_ZIP_URL=$HAZELCAST_OSS_ZIP_URL \ $TAGS_ARG \ --platform=${PLATFORMS} $DOCKER_DIR - name: Slack notification diff --git a/.github/workflows/scheduled_vulnerability_scan.yml b/.github/workflows/scheduled_vulnerability_scan.yml index e9d7c59f..ea5f8ed7 100644 --- a/.github/workflows/scheduled_vulnerability_scan.yml +++ b/.github/workflows/scheduled_vulnerability_scan.yml @@ -15,5 +15,4 @@ jobs: uses: ./.github/workflows/vulnerability_scan_subworkflow.yml with: ref: ${{ matrix.ref }} - secrets: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + secrets: inherit diff --git a/.github/workflows/vulnerability_scan.yml b/.github/workflows/vulnerability_scan.yml index 7dbcb55c..47655c7d 100644 --- a/.github/workflows/vulnerability_scan.yml +++ b/.github/workflows/vulnerability_scan.yml @@ -12,6 +12,5 @@ jobs: uses: ./.github/workflows/vulnerability_scan_subworkflow.yml with: ref: ${{ github.ref }} - secrets: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/vulnerability_scan_subworkflow.yml b/.github/workflows/vulnerability_scan_subworkflow.yml index 51e7b42c..af543312 100644 --- a/.github/workflows/vulnerability_scan_subworkflow.yml +++ b/.github/workflows/vulnerability_scan_subworkflow.yml @@ -29,6 +29,15 @@ jobs: - name: Install xmllint uses: ./.github/actions/install-xmllint + - name: Get OSS dist ZIP + run: | + . .github/scripts/oss-build.functions.sh + HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-oss/Dockerfile) + export HZ_SNAPSHOT_INTERNAL_PASSWORD=${{ secrets.HZ_SNAPSHOT_INTERNAL_PASSWORD }} + export HZ_SNAPSHOT_INTERNAL_USERNAME=${{ secrets.HZ_SNAPSHOT_INTERNAL_USERNAME }} + HAZELCAST_OSS_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}") + curl --fail --silent --show-error --location "$HAZELCAST_OSS_ZIP_URL" --output hazelcast-oss/hazelcast-distribution.zip; + - name: Build OSS image run: | docker build -t hazelcast/oss:${{ github.sha }} hazelcast-oss @@ -70,6 +79,16 @@ jobs: with: ref: ${{ inputs.ref }} + - name: Install xmllint + uses: ./.github/actions/install-xmllint + + - name: Get EE dist ZIP + run: | + . .github/scripts/ee-build.functions.sh + HZ_VERSION=$(awk -F '=' '/^ARG HZ_VERSION=/ {print $2}' hazelcast-enterprise/Dockerfile) + HAZELCAST_EE_ZIP_URL=$(get_hz_dist_zip "" "${HZ_VERSION}") + curl --fail --silent --show-error --location "$HAZELCAST_EE_ZIP_URL" --output hazelcast-enterprise/hazelcast-enterprise-distribution.zip; + - name: Build EE image run: | docker build -t hazelcast/ee:${{ github.sha }} hazelcast-enterprise diff --git a/hazelcast-enterprise/Dockerfile b/hazelcast-enterprise/Dockerfile index 0f8467ca..2846e668 100644 --- a/hazelcast-enterprise/Dockerfile +++ b/hazelcast-enterprise/Dockerfile @@ -50,7 +50,7 @@ RUN echo "Installing new packages" \ fi; \ echo "Downloading Hazelcast distribution zip from ${HAZELCAST_ZIP_URL}..."; \ mkdir --parents ${HZ_HOME}; \ - curl -sfL ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-enterprise-distribution.zip; \ + curl --fail --silent --show-error --location ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-enterprise-distribution.zip; \ else \ echo "Using local hazelcast-enterprise-distribution.zip"; \ fi \ diff --git a/hazelcast-oss/Dockerfile b/hazelcast-oss/Dockerfile index dd8132a1..8566bc6a 100644 --- a/hazelcast-oss/Dockerfile +++ b/hazelcast-oss/Dockerfile @@ -39,7 +39,7 @@ RUN echo "Upgrading APK packages" \ fi; \ echo "Downloading Hazelcast distribution zip from ${HAZELCAST_ZIP_URL}..."; \ mkdir --parents ${HZ_HOME}; \ - curl -sfL ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-distribution.zip; \ + curl --fail --silent --show-error --location ${HAZELCAST_ZIP_URL} --output ${HZ_HOME}/hazelcast-distribution.zip; \ else \ echo "Using local hazelcast-distribution.zip"; \ fi \