adding doi pr changes

KrishVora2912 · Jul 22, 2024 · 67cea41 · 67cea41
1 parent 90b779b
commit 67cea41
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 30 deletions.
diff --git a/docker/common.py b/docker/common.py
@@ -19,7 +19,30 @@
 import tempfile
 import os
 from distutils.dir_util import copy_tree
+from version_keys import version_keys
 import shutil
+import sys
+import re
+
+def get_gpg_key(kafka_version):
+    """
+    Retrieves the GPG key for the specified kafka version, if it exists, from docker/version_keys.py.
+    """
+    gpg_key = version_keys.get(kafka_version)
+    if gpg_key is not None:
+        return gpg_key
+    else:
+        print(f"No GPG Key data exists for kafka version {kafka_version}.")
+        print("Please ensure an entry corresponding to it exists under docker/version_keys.py")
+        sys.exit(1)
+
+def get_kafka_version_from_url(kafka_url):
+    match = re.search("\d+\.\d+\.\d+", kafka_url)
+    if match:
+        return match.group(0)
+    else:
+        print(f"No pattern found matching x.x.x in {kafka_url}. No version number extracted")
+        sys.exit(1)
 
 def execute(command):
     if subprocess.run(command).returncode != 0:

diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py
@@ -37,13 +37,16 @@
 from distutils.dir_util import copy_tree
 import shutil
 from test.docker_sanity_test import run_tests
-from common import execute, build_docker_image_runner
+from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url
 import tempfile
 import os
+import re
+import sys
 
 def build_docker_image(image, tag, kafka_url, image_type):
     image = f'{image}:{tag}'
-    build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR", image_type)
+    kafka_version = get_kafka_version_from_url(kafka_url)
+    build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} $DOCKER_DIR", image_type)
 
 def run_docker_tests(image, tag, kafka_url, image_type):
     temp_dir_path = tempfile.mkdtemp()

diff --git a/docker/docker_release.py b/docker/docker_release.py
@@ -38,12 +38,13 @@
 from datetime import date
 import argparse
 
-from common import execute, build_docker_image_runner
+from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url
 
 def build_push(image, kafka_url, image_type):
     try:
         create_builder()
-        build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \
+        kafka_version = get_kafka_version_from_url(kafka_url)
+        build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} --push \
               --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR", image_type)
     except:
         raise SystemError("Docker image push failed")

diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile
@@ -18,39 +18,41 @@
 
 FROM eclipse-temurin:21-jre-alpine AS build-jsa
 
-USER root
-
 # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
 ARG kafka_url
+ARG GPG_KEY
 
 COPY jsa_launch /etc/kafka/docker/jsa_launch
 
 RUN set -eux ; \
-    apk update ; \
-    apk upgrade ; \
     apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
-    mkdir opt/kafka; \
     wget -nv -O kafka.tgz "$kafka_url"; \
     wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
-    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
-    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
-    gpg --import KEYS; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                          hkp://p80.pool.sks-keyservers.net:80 \
+                          keyserver.ubuntu.com \
+                          hkp://keyserver.ubuntu.com:80 \
+                          pgp.mit.edu \
+                          hkp://keys.openpgp.org) ; do \
+      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+    done && \
     gpg --batch --verify kafka.tgz.asc kafka.tgz
 
-# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
-RUN /etc/kafka/docker/jsa_launch
+RUN mkdir opt/kafka; \
+    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
+    # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
+    /etc/kafka/docker/jsa_launch
 
 
 FROM eclipse-temurin:21-jre-alpine
 
 # exposed ports
 EXPOSE 9092
 
-USER root
-
 # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
 ARG kafka_url
 ARG build_date
+ARG GPG_KEY
 
 
 LABEL org.label-schema.name="kafka" \
@@ -60,16 +62,20 @@ LABEL org.label-schema.name="kafka" \
       maintainer="Apache Kafka"
 
 RUN set -eux ; \
-    apk update ; \
-    apk upgrade ; \
     apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
-    mkdir opt/kafka; \
     wget -nv -O kafka.tgz "$kafka_url"; \
     wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
-    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
-    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
-    gpg --import KEYS; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                          hkp://p80.pool.sks-keyservers.net:80 \
+                          keyserver.ubuntu.com \
+                          hkp://keyserver.ubuntu.com:80 \
+                          pgp.mit.edu \
+                          hkp://keys.openpgp.org) ; do \
+      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+    done && \
     gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    mkdir opt/kafka; \
+    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
     mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \
     mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \
     adduser -h /home/appuser -D --shell /bin/bash appuser; \
@@ -79,9 +85,8 @@ RUN set -eux ; \
     cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \
     cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \
     cp /opt/kafka/config/kraft/server.properties /etc/kafka/docker/server.properties; \
-    rm kafka.tgz kafka.tgz.asc KEYS; \
-    apk del wget gpg gpg-agent; \
-    apk cache clean;
+    rm kafka.tgz kafka.tgz.asc; \
+    apk del wget gpg gpg-agent; 
 
 COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa
 COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa

diff --git a/docker/native/Dockerfile b/docker/native/Dockerfile
@@ -16,6 +16,7 @@
 FROM ghcr.io/graalvm/graalvm-community:21 AS build-native-image
 
 ARG kafka_url
+ARG GPG_KEY
 
 WORKDIR /app
 
@@ -33,10 +34,16 @@ RUN mkdir $KAFKA_DIR; \
     microdnf install wget; \
     wget -nv -O kafka.tgz "$KAFKA_URL"; \
     wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
-    tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \
-    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
-    gpg --import KEYS; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                          hkp://p80.pool.sks-keyservers.net:80 \
+                          keyserver.ubuntu.com \
+                          hkp://keyserver.ubuntu.com:80 \
+                          pgp.mit.edu \
+                          hkp://keys.openpgp.org) ; do \
+      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+    done && \
     gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \
     rm kafka.tgz ; \
 # Build the native-binary of the apache kafka using graalVM native-image.
     /app/native_command.sh $NATIVE_IMAGE_PATH $NATIVE_CONFIGS_DIR $KAFKA_LIBS_DIR $TARGET_PATH
@@ -54,8 +61,7 @@ LABEL org.label-schema.name="kafka" \
       org.label-schema.vcs-url="https://github.com/apache/kafka" \
       maintainer="Apache Kafka"
 
-RUN apk update ; \
-    apk add --no-cache gcompat ; \
+RUN apk add --no-cache gcompat ; \
     apk add --no-cache bash ; \
     mkdir -p /etc/kafka/docker /mnt/shared/config /opt/kafka/config /etc/kafka/secrets ; \
     adduser -h /home/appuser -D --shell /bin/bash appuser ; \

diff --git a/docker/prepare_docker_official_image_source.py b/docker/prepare_docker_official_image_source.py
@@ -34,6 +34,7 @@
 from datetime import date
 import argparse
 from distutils.dir_util import copy_tree
+from common import get_gpg_key
 import os
 import shutil
 import re
@@ -45,6 +46,8 @@ def remove_args_and_hardcode_values(file_path, kafka_version, kafka_url):
     filedata = filedata.replace("ARG kafka_url", f"ENV kafka_url {kafka_url}")
     filedata = filedata.replace(
         "ARG build_date", f"ENV build_date {str(date.today())}")
+    filedata = filedata.replace(
+        "ARG GPG_KEY", f"ENV GPG_KEY {get_gpg_key(kafka_version)}")
     original_comment = re.compile(r"# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments")
     updated_comment = f"# Get Kafka from https://archive.apache.org/dist/kafka, url passed as env var, for version {kafka_version}"
     filedata = original_comment.sub(updated_comment, filedata)

diff --git a/docker/version_keys.py b/docker/version_keys.py
@@ -0,0 +1,4 @@
+version_keys = {
+    '3.7.0': 'DE0D9D12FB1360DA',
+    '3.7.1': '39AB5531A7FCB08E',
+}