KAFKA-16373: KIP-1028: Addressing Docker Official Images PR Comments for JVM, Native and Docker Official Images

docker/common.py

@@ -19,7 +19,44 @@
 import tempfile
 import os
 from distutils.dir_util import copy_tree
+import json
 import shutil
+import sys
+import re
+
+def load_version_gpg_keys():
+    '''
+    Loads the version-specific GPG keys from the 'version_gpg_keys.json' file.
+    '''
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    json_file = os.path.join(script_dir, 'version_gpg_keys.json')
+    with open(json_file, 'r') as f:
+        version_gpg_keys = json.load(f)
+    return version_gpg_keys
+
+def get_gpg_key(kafka_version):
+    """
+    Retrieves the GPG key for the specified kafka version, if it exists, from docker/version_gpg_keys.py.
+    """
+    version_gpg_keys = load_version_gpg_keys()
+    gpg_key = version_gpg_keys.get(kafka_version)
+    if gpg_key is not None:
+        return gpg_key
+    else:
+        print(f"No GPG Key data exists for kafka version {kafka_version}.")
+        print("Please ensure an entry corresponding to it exists under docker/version_gpg_keys.py")
+        sys.exit(1)
+
+def get_kafka_version_from_url(kafka_url):
+    """
+    Retrives the major.minor.patch (x.x.x) version from the given Kafka URL.
+    """
+    match = re.search("\d+\.\d+\.\d+", kafka_url)
+    if match:
+        return match.group(0)
+    else:
+        print(f"No pattern found matching x.x.x in {kafka_url}. No version number extracted")
+        sys.exit(1)
 
 def execute(command):
     if subprocess.run(command).returncode != 0:

docker/docker_build_test.py

@@ -37,13 +37,16 @@
 from distutils.dir_util import copy_tree
 import shutil
 from test.docker_sanity_test import run_tests
-from common import execute, build_docker_image_runner
+from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url
 import tempfile
 import os
+import re
+import sys
 
 def build_docker_image(image, tag, kafka_url, image_type):
     image = f'{image}:{tag}'
-    build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR", image_type)
+    kafka_version = get_kafka_version_from_url(kafka_url)
+    build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} $DOCKER_DIR", image_type)
 
 def run_docker_tests(image, tag, kafka_url, image_type):
     temp_dir_path = tempfile.mkdtemp()

docker/docker_release.py

@@ -38,12 +38,13 @@
 from datetime import date
 import argparse
 
-from common import execute, build_docker_image_runner
+from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url
 
 def build_push(image, kafka_url, image_type):
     try:
         create_builder()
-        build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \
+        kafka_version = get_kafka_version_from_url(kafka_url)
+        build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} --push \
               --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR", image_type)
     except:
         raise SystemError("Docker image push failed")

docker/jvm/Dockerfile

@@ -18,58 +18,56 @@
 
 FROM eclipse-temurin:21-jre-alpine AS build-jsa
 
-USER root
-
 # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
 ARG kafka_url
+ARG GPG_KEY
 
 COPY jsa_launch /etc/kafka/docker/jsa_launch
 
 RUN set -eux ; \
-    apk update ; \
-    apk upgrade ; \
     apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
-    mkdir opt/kafka; \
     wget -nv -O kafka.tgz "$kafka_url"; \
     wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    gpg --batch --keyserver hkp://keys.openpgp.org --recv-keys "$GPG_KEY" || \
+    gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$GPG_KEY" ; \
+    gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" kafka.tgz.asc; \
+    mkdir opt/kafka; \
     tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
-    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
-    gpg --import KEYS; \
-    gpg --batch --verify kafka.tgz.asc kafka.tgz
-
-# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
-RUN /etc/kafka/docker/jsa_launch
+    # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
+    /etc/kafka/docker/jsa_launch
 
 
 FROM eclipse-temurin:21-jre-alpine
 
 # exposed ports
 EXPOSE 9092
 
-USER root
-
 # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
 ARG kafka_url
 ARG build_date
+ARG GPG_KEY
 
-
-LABEL org.label-schema.name="kafka" \
-      org.label-schema.description="Apache Kafka" \
-      org.label-schema.build-date="${build_date}" \
-      org.label-schema.vcs-url="https://github.com/apache/kafka" \
+LABEL org.opencontainers.image.title="kafka" \
+      org.opencontainers.image.description="Apache Kafka" \
+      org.opencontainers.image.created="${build_date}" \
+      org.opencontainers.image.source="https://github.com/apache/kafka" \
       maintainer="Apache Kafka"
 
 RUN set -eux ; \
-    apk update ; \
-    apk upgrade ; \
     apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
-    mkdir opt/kafka; \
     wget -nv -O kafka.tgz "$kafka_url"; \
     wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
-    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
-    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
-    gpg --import KEYS; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    gpg --batch --keyserver hkp://keys.openpgp.org --recv-keys "$GPG_KEY" || \
+    gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$GPG_KEY" ; \
     gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" kafka.tgz.asc; \
+    mkdir opt/kafka; \
+    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
     mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \
     mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \
     adduser -h /home/appuser -D --shell /bin/bash appuser; \
@@ -79,9 +77,8 @@ RUN set -eux ; \
     cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \
     cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \
     cp /opt/kafka/config/kraft/server.properties /etc/kafka/docker/server.properties; \
-    rm kafka.tgz kafka.tgz.asc KEYS; \
-    apk del wget gpg gpg-agent; \
-    apk cache clean;
+    rm kafka.tgz; \
+    apk del wget gpg gpg-agent; 
 
 COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa
 COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa

docker/native/Dockerfile

@@ -16,6 +16,7 @@
 FROM ghcr.io/graalvm/graalvm-community:21 AS build-native-image
 
 ARG kafka_url
+ARG GPG_KEY
 
 WORKDIR /app
 
@@ -33,10 +34,13 @@ RUN mkdir $KAFKA_DIR; \
     microdnf install wget; \
     wget -nv -O kafka.tgz "$KAFKA_URL"; \
     wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
-    tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \
-    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
-    gpg --import KEYS; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    gpg --batch --keyserver hkp://keys.openpgp.org --recv-keys "$GPG_KEY" || \
+    gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$GPG_KEY" ; \
     gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" kafka.tgz.asc; \
+    tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \
     rm kafka.tgz ; \
 # Build the native-binary of the apache kafka using graalVM native-image.
     /app/native_command.sh $NATIVE_IMAGE_PATH $NATIVE_CONFIGS_DIR $KAFKA_LIBS_DIR $TARGET_PATH
@@ -48,14 +52,13 @@ EXPOSE 9092
 
 ARG build_date
 
-LABEL org.label-schema.name="kafka" \
-      org.label-schema.description="Apache Kafka" \
-      org.label-schema.build-date="${build_date}" \
-      org.label-schema.vcs-url="https://github.com/apache/kafka" \
+LABEL org.opencontainers.image.title="kafka" \
+      org.opencontainers.image.description="Apache Kafka" \
+      org.opencontainers.image.created="${build_date}" \
+      org.opencontainers.image.source="https://github.com/apache/kafka" \
       maintainer="Apache Kafka"
 
-RUN apk update ; \
-    apk add --no-cache gcompat ; \
+RUN apk add --no-cache gcompat ; \
     apk add --no-cache bash ; \
     mkdir -p /etc/kafka/docker /mnt/shared/config /opt/kafka/config /etc/kafka/secrets ; \
     adduser -h /home/appuser -D --shell /bin/bash appuser ; \

docker/prepare_docker_official_image_source.py

@@ -34,6 +34,7 @@
 from datetime import date
 import argparse
 from distutils.dir_util import copy_tree
+from common import get_gpg_key
 import os
 import shutil
 import re
@@ -42,9 +43,15 @@
 def remove_args_and_hardcode_values(file_path, kafka_version, kafka_url):
     with open(file_path, 'r') as file:
         filedata = file.read()
+    # Remove any line containing "ARG GPG_KEY"
+    lines = filedata.splitlines()
+    lines = [line for line in lines if "ARG GPG_KEY" not in line]
+    filedata = "\n".join(lines)
     filedata = filedata.replace("ARG kafka_url", f"ENV kafka_url {kafka_url}")
     filedata = filedata.replace(
         "ARG build_date", f"ENV build_date {str(date.today())}")
+    # Replace "$GPG_KEY" with the actual GPG key
+    filedata = filedata.replace('"$GPG_KEY"', get_gpg_key(kafka_version))
     original_comment = re.compile(r"# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments")
     updated_comment = f"# Get Kafka from https://archive.apache.org/dist/kafka, url passed as env var, for version {kafka_version}"
     filedata = original_comment.sub(updated_comment, filedata)

docker/test/docker_sanity_test.py

@@ -41,10 +41,10 @@ def update_file(self, filename, old_string, new_string):
     def start_compose(self, filename) -> None:
         self.update_file(filename, "image: {$IMAGE}", f"image: {self.IMAGE}")
         self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", "{$DIR}", self.FIXTURES_DIR)
-        subprocess.run(["docker-compose", "-f", filename, "up", "-d"])
+        subprocess.run(["docker", "compose", "-f", filename, "up", "-d"])
 
     def destroy_compose(self, filename) -> None:
-        subprocess.run(["docker-compose", "-f", filename, "down"])
+        subprocess.run(["docker", "compose", "-f", filename, "down"])
         self.update_file(filename, f"image: {self.IMAGE}", "image: {$IMAGE}")
         self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", self.FIXTURES_DIR, "{$DIR}")
 

docker/version_gpg_keys.json

@@ -0,0 +1,5 @@
+{
+    "3.7.0": "7C38C2F6E7DF40E527C7C996DE0D9D12FB1360DA",
+    "3.7.1": "4687E2BC1319B57B321D6F0E39AB5531A7FCB08E",
+    "3.8.0": "CF9500821E9557AEB04E026C05EEA67F87749E61"
+}