From 67cea41515a56bdf0c3a1187e2e05d03a75b33e8 Mon Sep 17 00:00:00 2001 From: KrishVora2912 Date: Mon, 22 Jul 2024 16:55:20 +0530 Subject: [PATCH] adding doi pr changes --- docker/common.py | 23 +++++++++ docker/docker_build_test.py | 7 ++- docker/docker_release.py | 5 +- docker/jvm/Dockerfile | 47 ++++++++++--------- docker/native/Dockerfile | 16 +++++-- .../prepare_docker_official_image_source.py | 3 ++ docker/version_keys.py | 4 ++ 7 files changed, 75 insertions(+), 30 deletions(-) create mode 100644 docker/version_keys.py diff --git a/docker/common.py b/docker/common.py index aaab1eae1ff18..6b0163b82bd14 100644 --- a/docker/common.py +++ b/docker/common.py @@ -19,7 +19,30 @@ import tempfile import os from distutils.dir_util import copy_tree +from version_keys import version_keys import shutil +import sys +import re + +def get_gpg_key(kafka_version): + """ + Retrieves the GPG key for the specified kafka version, if it exists, from docker/version_keys.py. + """ + gpg_key = version_keys.get(kafka_version) + if gpg_key is not None: + return gpg_key + else: + print(f"No GPG Key data exists for kafka version {kafka_version}.") + print("Please ensure an entry corresponding to it exists under docker/version_keys.py") + sys.exit(1) + +def get_kafka_version_from_url(kafka_url): + match = re.search("\d+\.\d+\.\d+", kafka_url) + if match: + return match.group(0) + else: + print(f"No pattern found matching x.x.x in {kafka_url}. No version number extracted") + sys.exit(1) def execute(command): if subprocess.run(command).returncode != 0: diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 793148573f395..906b88772206f 100755 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -37,13 +37,16 @@ from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests -from common import execute, build_docker_image_runner +from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url import tempfile import os +import re +import sys def build_docker_image(image, tag, kafka_url, image_type): image = f'{image}:{tag}' - build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR", image_type) + kafka_version = get_kafka_version_from_url(kafka_url) + build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} $DOCKER_DIR", image_type) def run_docker_tests(image, tag, kafka_url, image_type): temp_dir_path = tempfile.mkdtemp() diff --git a/docker/docker_release.py b/docker/docker_release.py index c92459a86eb17..44e8b0ebcc027 100755 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -38,12 +38,13 @@ from datetime import date import argparse -from common import execute, build_docker_image_runner +from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url def build_push(image, kafka_url, image_type): try: create_builder() - build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ + kafka_version = get_kafka_version_from_url(kafka_url) + build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} --push \ --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR", image_type) except: raise SystemError("Docker image push failed") diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 72e35e63c0473..cf3047c874708 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -18,27 +18,30 @@ FROM eclipse-temurin:21-jre-alpine AS build-jsa -USER root - # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments ARG kafka_url +ARG GPG_KEY COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ - apk update ; \ - apk upgrade ; \ apk add --no-cache wget gcompat gpg gpg-agent procps bash; \ - mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ - tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ - wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ - gpg --import KEYS; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ + hkp://keys.openpgp.org) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ gpg --batch --verify kafka.tgz.asc kafka.tgz -# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command -RUN /etc/kafka/docker/jsa_launch +RUN mkdir opt/kafka; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ + # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command + /etc/kafka/docker/jsa_launch FROM eclipse-temurin:21-jre-alpine @@ -46,11 +49,10 @@ FROM eclipse-temurin:21-jre-alpine # exposed ports EXPOSE 9092 -USER root - # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments ARG kafka_url ARG build_date +ARG GPG_KEY LABEL org.label-schema.name="kafka" \ @@ -60,16 +62,20 @@ LABEL org.label-schema.name="kafka" \ maintainer="Apache Kafka" RUN set -eux ; \ - apk update ; \ - apk upgrade ; \ apk add --no-cache wget gcompat gpg gpg-agent procps bash; \ - mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ - tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ - wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ - gpg --import KEYS; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ + hkp://keys.openpgp.org) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + mkdir opt/kafka; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \ mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ @@ -79,9 +85,8 @@ RUN set -eux ; \ cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \ cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \ cp /opt/kafka/config/kraft/server.properties /etc/kafka/docker/server.properties; \ - rm kafka.tgz kafka.tgz.asc KEYS; \ - apk del wget gpg gpg-agent; \ - apk cache clean; + rm kafka.tgz kafka.tgz.asc; \ + apk del wget gpg gpg-agent; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa diff --git a/docker/native/Dockerfile b/docker/native/Dockerfile index 36104b76edca2..c2eab9e3d9789 100644 --- a/docker/native/Dockerfile +++ b/docker/native/Dockerfile @@ -16,6 +16,7 @@ FROM ghcr.io/graalvm/graalvm-community:21 AS build-native-image ARG kafka_url +ARG GPG_KEY WORKDIR /app @@ -33,10 +34,16 @@ RUN mkdir $KAFKA_DIR; \ microdnf install wget; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ - tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \ - wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ - gpg --import KEYS; \ + for server in ha.pool.sks-keyservers.net $(shuf -e \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ + hkp://keys.openpgp.org) ; do \ + gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \ + done && \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \ rm kafka.tgz ; \ # Build the native-binary of the apache kafka using graalVM native-image. /app/native_command.sh $NATIVE_IMAGE_PATH $NATIVE_CONFIGS_DIR $KAFKA_LIBS_DIR $TARGET_PATH @@ -54,8 +61,7 @@ LABEL org.label-schema.name="kafka" \ org.label-schema.vcs-url="https://github.com/apache/kafka" \ maintainer="Apache Kafka" -RUN apk update ; \ - apk add --no-cache gcompat ; \ +RUN apk add --no-cache gcompat ; \ apk add --no-cache bash ; \ mkdir -p /etc/kafka/docker /mnt/shared/config /opt/kafka/config /etc/kafka/secrets ; \ adduser -h /home/appuser -D --shell /bin/bash appuser ; \ diff --git a/docker/prepare_docker_official_image_source.py b/docker/prepare_docker_official_image_source.py index 25d57c53e0fd8..6d841325397c8 100644 --- a/docker/prepare_docker_official_image_source.py +++ b/docker/prepare_docker_official_image_source.py @@ -34,6 +34,7 @@ from datetime import date import argparse from distutils.dir_util import copy_tree +from common import get_gpg_key import os import shutil import re @@ -45,6 +46,8 @@ def remove_args_and_hardcode_values(file_path, kafka_version, kafka_url): filedata = filedata.replace("ARG kafka_url", f"ENV kafka_url {kafka_url}") filedata = filedata.replace( "ARG build_date", f"ENV build_date {str(date.today())}") + filedata = filedata.replace( + "ARG GPG_KEY", f"ENV GPG_KEY {get_gpg_key(kafka_version)}") original_comment = re.compile(r"# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments") updated_comment = f"# Get Kafka from https://archive.apache.org/dist/kafka, url passed as env var, for version {kafka_version}" filedata = original_comment.sub(updated_comment, filedata) diff --git a/docker/version_keys.py b/docker/version_keys.py new file mode 100644 index 0000000000000..c3cf09f8da33a --- /dev/null +++ b/docker/version_keys.py @@ -0,0 +1,4 @@ +version_keys = { + '3.7.0': 'DE0D9D12FB1360DA', + '3.7.1': '39AB5531A7FCB08E', +} \ No newline at end of file