feat(airflow): Add apache-kafka and update queue/messaging libs (#1313)

* add apache-kafka and update queue/messaging libs

* changelog

* Update CHANGELOG.md

Co-authored-by: Sebastian Bernauer <[email protected]>

---------

Co-authored-by: Sebastian Bernauer <[email protected]>

Author: adwk67

CHANGELOG.md

@@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file.
 - hbase: Add `2.6.3` ([#1296]).
 - airflow,superset: Build from source ([#1304]).
 - kafka: Add updated JMX config `server.yaml` for KRaft ([#1308]).
+- airflow: Add apache-kafka provider plus dependencies to `3.0.6` image ([#1313]).
 
 ### Changed
 
@@ -110,6 +111,7 @@ All notable changes to this project will be documented in this file.
 [#1308]: https://github.com/stackabletech/docker-images/pull/1308
 [#1309]: https://github.com/stackabletech/docker-images/pull/1309
 [#1311]: https://github.com/stackabletech/docker-images/pull/1311
+[#1313]: https://github.com/stackabletech/docker-images/pull/1313
 [#1314]: https://github.com/stackabletech/docker-images/pull/1314
 [#1315]: https://github.com/stackabletech/docker-images/pull/1315
 [#1316]: https://github.com/stackabletech/docker-images/pull/1316

airflow/boil-config.toml

@@ -58,6 +58,6 @@ s3fs-version = "2024.9.0"
 cyclonedx-bom-version = "6.0.0"
 tini-version = "0.19.0"
 uv-version = "0.7.8"
-airflow-extras = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino"
+airflow-extras = "amazon,apache-kafka,async,celery,cncf-kubernetes,common-messaging,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino"
 opa-auth-manager = "airflow-3"
 nodejs-version = "20"

airflow/stackable/constraints/3.0.6/constraints-python3.12.txt

@@ -109,7 +109,10 @@ apache-airflow-providers-apache-hdfs==4.10.2
 apache-airflow-providers-apache-hive==9.1.2
 apache-airflow-providers-apache-iceberg==1.3.2
 apache-airflow-providers-apache-impala==1.7.2
-apache-airflow-providers-apache-kafka==1.10.2
+# Stackable patch:
+# The original version 1.10.2 has sync/async issues when using event scheduling
+# with the triggerer. The fix could not be traced to a specific change/commit.
+apache-airflow-providers-apache-kafka==1.10.4
 apache-airflow-providers-apache-kylin==3.9.2
 apache-airflow-providers-apache-livy==4.4.2
 apache-airflow-providers-apache-pig==4.7.2
@@ -125,7 +128,12 @@ apache-airflow-providers-cncf-kubernetes==10.7.0
 apache-airflow-providers-cohere==1.5.3
 apache-airflow-providers-common-compat==1.7.3
 apache-airflow-providers-common-io==1.6.2
-apache-airflow-providers-common-messaging==1.0.5
+# Stackable patch:
+# Update from previous version 1.0.5 due to refactoring and compatability with
+# apache-airflow-providers-apache-kafka==1.10.4 (see above).
+# See https://github.com/apache/airflow/commit/1790ec41eeaff6e33f78b39ad50b2bcd98107e67
+# See https://github.com/apache/airflow/commit/1790ec41eeaff6e33f78b39ad50b2bcd98107e67
+apache-airflow-providers-common-messaging==2.0.0
 apache-airflow-providers-common-sql==1.27.5
 apache-airflow-providers-databricks==7.7.1
 apache-airflow-providers-datadog==3.9.2