From a611f2569e7ea1d5d490bcf28f5a7dc1638184a6 Mon Sep 17 00:00:00 2001 From: stephanemaarek Date: Tue, 20 May 2025 19:13:54 +0100 Subject: [PATCH 1/4] Add Kafka KRaft mode support with Conduktor and JMX integration --- conduktor-kafka-single.yml | 57 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 conduktor-kafka-single.yml diff --git a/conduktor-kafka-single.yml b/conduktor-kafka-single.yml new file mode 100644 index 0000000..d1b9cda --- /dev/null +++ b/conduktor-kafka-single.yml @@ -0,0 +1,57 @@ +services: + postgresql: + image: postgres:14 + hostname: postgresql + volumes: + - pg_data:/var/lib/postgresql/data + environment: + POSTGRES_DB: "conduktor-console" + POSTGRES_USER: "conduktor" + POSTGRES_PASSWORD: "some_password" + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" + conduktor-console: + image: conduktor/conduktor-console:1.34.0 + ports: + - "8080:8080" + volumes: + - conduktor_data:/var/conduktor + environment: + CDK_DATABASE_URL: "postgresql://conduktor:some_password@postgresql:5432/conduktor-console" + CDK_CLUSTERS_0_ID: "default" + CDK_CLUSTERS_0_NAME: "My Local Kafka Cluster" + CDK_CLUSTERS_0_COLOR: "#0013E7" + CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "PLAINTEXT://kafka1:19092" + # CONSOLE_JAVA_OPTS: "-XX:UseSVE=0" + CDK_CLUSTERS_0_JMXPORT: "9999" + CDK_CLUSTERS_0_JMXHOST: "kafka1" + kafka1: + image: confluentinc/cp-kafka:7.8.0 + hostname: kafka1 + container_name: kafka1 + ports: + - "9092:9092" + - "29092:29092" + - "9999:9999" # JMX Port + - "9099:9099" # Internal inter-broker communication + environment: + environment: + CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka1:9099" + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:9092,DOCKER://0.0.0.0:29092,CONTROLLER://0.0.0.0:9099 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_JMX_PORT: 9999 + KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} + KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" +volumes: + pg_data: {} + conduktor_data: {} \ No newline at end of file From a34bca81848ec4fb3db01538442cf5a1a5246bb1 Mon Sep 17 00:00:00 2001 From: stephanemaarek Date: Tue, 20 May 2025 19:17:12 +0100 Subject: [PATCH 2/4] updated confluent cpvversion --- conduktor-kafka-single.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conduktor-kafka-single.yml b/conduktor-kafka-single.yml index d1b9cda..20d03ae 100644 --- a/conduktor-kafka-single.yml +++ b/conduktor-kafka-single.yml @@ -25,7 +25,7 @@ services: CDK_CLUSTERS_0_JMXPORT: "9999" CDK_CLUSTERS_0_JMXHOST: "kafka1" kafka1: - image: confluentinc/cp-kafka:7.8.0 + image: confluentinc/cp-kafka:7.9.1 hostname: kafka1 container_name: kafka1 ports: From 6f6043400dfc2d249319febcb88c6e92e467ac1a Mon Sep 17 00:00:00 2001 From: stephanemaarek Date: Wed, 13 Aug 2025 11:23:46 +0100 Subject: [PATCH 3/4] updated Kafka version and README --- README.md | 14 ++++++++++++++ conduktor-kafka-single.yml | 8 ++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4bea24a..f52f5b7 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,20 @@ docker compose -f full-stack.yml up docker compose -f full-stack.yml down ``` +## Single Kafka (KRaft mode) + Conduktor + +This configuration fits most development requirements. + + - Kafka will be available at `$DOCKER_HOST_IP:9092` + - Conduktor will be available at : `$DOCKER_HOST_IP:8080` + +Run with: +``` +docker compose -f conduktor-kafka-single.yml up +docker compose -f conduktor-kafka-single.yml down +``` + + ## Single Zookeeper / Single Kafka This configuration fits most development requirements. diff --git a/conduktor-kafka-single.yml b/conduktor-kafka-single.yml index 20d03ae..5c0637a 100644 --- a/conduktor-kafka-single.yml +++ b/conduktor-kafka-single.yml @@ -10,7 +10,7 @@ services: POSTGRES_PASSWORD: "some_password" POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" conduktor-console: - image: conduktor/conduktor-console:1.34.0 + image: conduktor/conduktor-console:1.36.2 ports: - "8080:8080" volumes: @@ -22,10 +22,8 @@ services: CDK_CLUSTERS_0_COLOR: "#0013E7" CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "PLAINTEXT://kafka1:19092" # CONSOLE_JAVA_OPTS: "-XX:UseSVE=0" - CDK_CLUSTERS_0_JMXPORT: "9999" - CDK_CLUSTERS_0_JMXHOST: "kafka1" kafka1: - image: confluentinc/cp-kafka:7.9.1 + image: confluentinc/cp-kafka:8.0.0 hostname: kafka1 container_name: kafka1 ports: @@ -48,8 +46,6 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_JMX_PORT: 9999 - KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" volumes: From 0f1319d5942fcea462d05006bb4f88a1e3d70709 Mon Sep 17 00:00:00 2001 From: stephanemaarek Date: Wed, 13 Aug 2025 11:27:37 +0100 Subject: [PATCH 4/4] added a test --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dd05287..e0424a3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,3 +52,7 @@ jobs: - name: run test full-stack 8 run: ./test.sh full-stack.yml 8 + + - name: run test conduktor-kafka-single 3 + run: ./test.sh conduktor-kafka-single.yml 3 +