diff --git a/docker/common.py b/docker/common.py index aaab1eae1ff18..5794e1713639b 100644 --- a/docker/common.py +++ b/docker/common.py @@ -19,7 +19,44 @@ import tempfile import os from distutils.dir_util import copy_tree +import json import shutil +import sys +import re + +def load_version_gpg_keys(): + ''' + Loads the version-specific GPG keys from the 'version_gpg_keys.json' file. + ''' + script_dir = os.path.dirname(os.path.abspath(__file__)) + json_file = os.path.join(script_dir, 'version_gpg_keys.json') + with open(json_file, 'r') as f: + version_gpg_keys = json.load(f) + return version_gpg_keys + +def get_gpg_key(kafka_version): + """ + Retrieves the GPG key for the specified kafka version, if it exists, from docker/version_gpg_keys.py. + """ + version_gpg_keys = load_version_gpg_keys() + gpg_key = version_gpg_keys.get(kafka_version) + if gpg_key is not None: + return gpg_key + else: + print(f"No GPG Key data exists for kafka version {kafka_version}.") + print("Please ensure an entry corresponding to it exists under docker/version_gpg_keys.py") + sys.exit(1) + +def get_kafka_version_from_url(kafka_url): + """ + Retrives the major.minor.patch (x.x.x) version from the given Kafka URL. + """ + match = re.search("\d+\.\d+\.\d+", kafka_url) + if match: + return match.group(0) + else: + print(f"No pattern found matching x.x.x in {kafka_url}. No version number extracted") + sys.exit(1) def execute(command): if subprocess.run(command).returncode != 0: diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 793148573f395..906b88772206f 100755 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -37,13 +37,16 @@ from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests -from common import execute, build_docker_image_runner +from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url import tempfile import os +import re +import sys def build_docker_image(image, tag, kafka_url, image_type): image = f'{image}:{tag}' - build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR", image_type) + kafka_version = get_kafka_version_from_url(kafka_url) + build_docker_image_runner(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} $DOCKER_DIR", image_type) def run_docker_tests(image, tag, kafka_url, image_type): temp_dir_path = tempfile.mkdtemp() diff --git a/docker/docker_release.py b/docker/docker_release.py index c92459a86eb17..44e8b0ebcc027 100755 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -38,12 +38,13 @@ from datetime import date import argparse -from common import execute, build_docker_image_runner +from common import execute, build_docker_image_runner, get_gpg_key, get_kafka_version_from_url def build_push(image, kafka_url, image_type): try: create_builder() - build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ + kafka_version = get_kafka_version_from_url(kafka_url) + build_docker_image_runner(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --build-arg GPG_KEY={get_gpg_key(kafka_version)} --push \ --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR", image_type) except: raise SystemError("Docker image push failed") diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 72e35e63c0473..0609217439bde 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -18,27 +18,26 @@ FROM eclipse-temurin:21-jre-alpine AS build-jsa -USER root - # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments ARG kafka_url +ARG GPG_KEY COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ - apk update ; \ - apk upgrade ; \ apk add --no-cache wget gcompat gpg gpg-agent procps bash; \ - mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkp://keys.openpgp.org --recv-keys "$GPG_KEY" || \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$GPG_KEY" ; \ + gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" kafka.tgz.asc; \ + mkdir opt/kafka; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ - wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ - gpg --import KEYS; \ - gpg --batch --verify kafka.tgz.asc kafka.tgz - -# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command -RUN /etc/kafka/docker/jsa_launch + # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command + /etc/kafka/docker/jsa_launch FROM eclipse-temurin:21-jre-alpine @@ -46,30 +45,29 @@ FROM eclipse-temurin:21-jre-alpine # exposed ports EXPOSE 9092 -USER root - # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments ARG kafka_url ARG build_date +ARG GPG_KEY - -LABEL org.label-schema.name="kafka" \ - org.label-schema.description="Apache Kafka" \ - org.label-schema.build-date="${build_date}" \ - org.label-schema.vcs-url="https://github.com/apache/kafka" \ +LABEL org.opencontainers.image.title="kafka" \ + org.opencontainers.image.description="Apache Kafka" \ + org.opencontainers.image.created="${build_date}" \ + org.opencontainers.image.source="https://github.com/apache/kafka" \ maintainer="Apache Kafka" RUN set -eux ; \ - apk update ; \ - apk upgrade ; \ apk add --no-cache wget gcompat gpg gpg-agent procps bash; \ - mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ - tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ - wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ - gpg --import KEYS; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkp://keys.openpgp.org --recv-keys "$GPG_KEY" || \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$GPG_KEY" ; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" kafka.tgz.asc; \ + mkdir opt/kafka; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \ mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ @@ -79,9 +77,8 @@ RUN set -eux ; \ cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \ cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \ cp /opt/kafka/config/kraft/server.properties /etc/kafka/docker/server.properties; \ - rm kafka.tgz kafka.tgz.asc KEYS; \ - apk del wget gpg gpg-agent; \ - apk cache clean; + rm kafka.tgz; \ + apk del wget gpg gpg-agent; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa diff --git a/docker/native/Dockerfile b/docker/native/Dockerfile index 36104b76edca2..e2b681027d713 100644 --- a/docker/native/Dockerfile +++ b/docker/native/Dockerfile @@ -16,6 +16,7 @@ FROM ghcr.io/graalvm/graalvm-community:21 AS build-native-image ARG kafka_url +ARG GPG_KEY WORKDIR /app @@ -33,10 +34,13 @@ RUN mkdir $KAFKA_DIR; \ microdnf install wget; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ - tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \ - wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ - gpg --import KEYS; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkp://keys.openpgp.org --recv-keys "$GPG_KEY" || \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$GPG_KEY" ; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" kafka.tgz.asc; \ + tar xfz kafka.tgz -C $KAFKA_DIR --strip-components 1; \ rm kafka.tgz ; \ # Build the native-binary of the apache kafka using graalVM native-image. /app/native_command.sh $NATIVE_IMAGE_PATH $NATIVE_CONFIGS_DIR $KAFKA_LIBS_DIR $TARGET_PATH @@ -48,14 +52,13 @@ EXPOSE 9092 ARG build_date -LABEL org.label-schema.name="kafka" \ - org.label-schema.description="Apache Kafka" \ - org.label-schema.build-date="${build_date}" \ - org.label-schema.vcs-url="https://github.com/apache/kafka" \ +LABEL org.opencontainers.image.title="kafka" \ + org.opencontainers.image.description="Apache Kafka" \ + org.opencontainers.image.created="${build_date}" \ + org.opencontainers.image.source="https://github.com/apache/kafka" \ maintainer="Apache Kafka" -RUN apk update ; \ - apk add --no-cache gcompat ; \ +RUN apk add --no-cache gcompat ; \ apk add --no-cache bash ; \ mkdir -p /etc/kafka/docker /mnt/shared/config /opt/kafka/config /etc/kafka/secrets ; \ adduser -h /home/appuser -D --shell /bin/bash appuser ; \ diff --git a/docker/prepare_docker_official_image_source.py b/docker/prepare_docker_official_image_source.py index 25d57c53e0fd8..fb715d5b9ad3a 100644 --- a/docker/prepare_docker_official_image_source.py +++ b/docker/prepare_docker_official_image_source.py @@ -34,6 +34,7 @@ from datetime import date import argparse from distutils.dir_util import copy_tree +from common import get_gpg_key import os import shutil import re @@ -42,9 +43,15 @@ def remove_args_and_hardcode_values(file_path, kafka_version, kafka_url): with open(file_path, 'r') as file: filedata = file.read() + # Remove any line containing "ARG GPG_KEY" + lines = filedata.splitlines() + lines = [line for line in lines if "ARG GPG_KEY" not in line] + filedata = "\n".join(lines) filedata = filedata.replace("ARG kafka_url", f"ENV kafka_url {kafka_url}") filedata = filedata.replace( "ARG build_date", f"ENV build_date {str(date.today())}") + # Replace "$GPG_KEY" with the actual GPG key + filedata = filedata.replace('"$GPG_KEY"', get_gpg_key(kafka_version)) original_comment = re.compile(r"# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments") updated_comment = f"# Get Kafka from https://archive.apache.org/dist/kafka, url passed as env var, for version {kafka_version}" filedata = original_comment.sub(updated_comment, filedata) diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index d2135fb029507..59ed6fcc9b41f 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -41,10 +41,10 @@ def update_file(self, filename, old_string, new_string): def start_compose(self, filename) -> None: self.update_file(filename, "image: {$IMAGE}", f"image: {self.IMAGE}") self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", "{$DIR}", self.FIXTURES_DIR) - subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) + subprocess.run(["docker", "compose", "-f", filename, "up", "-d"]) def destroy_compose(self, filename) -> None: - subprocess.run(["docker-compose", "-f", filename, "down"]) + subprocess.run(["docker", "compose", "-f", filename, "down"]) self.update_file(filename, f"image: {self.IMAGE}", "image: {$IMAGE}") self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", self.FIXTURES_DIR, "{$DIR}") diff --git a/docker/version_gpg_keys.json b/docker/version_gpg_keys.json new file mode 100644 index 0000000000000..c8524f34c7240 --- /dev/null +++ b/docker/version_gpg_keys.json @@ -0,0 +1,5 @@ +{ + "3.7.0": "7C38C2F6E7DF40E527C7C996DE0D9D12FB1360DA", + "3.7.1": "4687E2BC1319B57B321D6F0E39AB5531A7FCB08E", + "3.8.0": "CF9500821E9557AEB04E026C05EEA67F87749E61" +} \ No newline at end of file