From c9d20a121ff6419575ded0a23854cbad2f365d83 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 9 Dec 2021 00:26:41 +0100 Subject: [PATCH 1/8] Upgrade Kafka client library to 3.0.0. Also update other dependencies on which Kafka client relies on. --- build.gradle | 10 +++++----- lib/logstash-input-kafka_jars.rb | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index 33ebdd4..fd57461 100644 --- a/build.gradle +++ b/build.gradle @@ -44,11 +44,11 @@ task wrapper(type: Wrapper) { } dependencies { - compile 'org.apache.kafka:kafka-clients:2.3.0' - compile 'com.github.luben:zstd-jni:1.4.2-1' - compile 'org.slf4j:slf4j-api:1.7.26' - compile 'org.lz4:lz4-java:1.6.0' - compile 'org.xerial.snappy:snappy-java:1.1.7.3' + compile 'org.apache.kafka:kafka-clients:3.0.0' + compile 'com.github.luben:zstd-jni:1.5.0-2' + compile 'org.slf4j:slf4j-api:1.7.30' + compile 'org.lz4:lz4-java:1.7.1' + compile 'org.xerial.snappy:snappy-java:1.1.8.1' } task generateGemJarRequiresFile { diff --git a/lib/logstash-input-kafka_jars.rb b/lib/logstash-input-kafka_jars.rb index ccd564e..82b601d 100644 --- a/lib/logstash-input-kafka_jars.rb +++ b/lib/logstash-input-kafka_jars.rb @@ -1,8 +1,8 @@ # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT. require 'jar_dependencies' -require_jar('org.apache.kafka', 'kafka-clients', '2.1.0') -require_jar('com.github.luben', 'zstd-jni', '1.3.7-3') -require_jar('org.slf4j', 'slf4j-api', '1.7.25') -require_jar('org.lz4', 'lz4-java', '1.5.0') -require_jar('org.xerial.snappy', 'snappy-java', '1.1.7.2') +require_jar('org.apache.kafka', 'kafka-clients', '3.0.0') +require_jar('com.github.luben', 'zstd-jni', '1.5.0-2') +require_jar('org.slf4j', 'slf4j-api', '1.7.30') +require_jar('org.lz4', 'lz4-java', '1.7.1') +require_jar('org.xerial.snappy', 'snappy-java', '1.1.8.1') From 0f63921b599f65e2b364e3b9a8166cbafa94080d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 14 Dec 2021 12:02:09 +0100 Subject: [PATCH 2/8] Fix gradle file so it works with newer versions of gradle. --- build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.gradle b/build.gradle index fd57461..20355b4 100644 --- a/build.gradle +++ b/build.gradle @@ -39,10 +39,6 @@ repositories { mavenCentral() } -task wrapper(type: Wrapper) { - gradleVersion = '4.0' -} - dependencies { compile 'org.apache.kafka:kafka-clients:3.0.0' compile 'com.github.luben:zstd-jni:1.5.0-2' From 2e5f721943a12e5f63c9de3dca5c609b1fb9d04b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 14 Dec 2021 12:02:44 +0100 Subject: [PATCH 3/8] Bump version. --- logstash-input-kafka.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-input-kafka.gemspec b/logstash-input-kafka.gemspec index 795be69..952cad3 100644 --- a/logstash-input-kafka.gemspec +++ b/logstash-input-kafka.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-input-kafka' - s.version = '9.1.0' + s.version = '9.2.0' s.licenses = ['Apache-2.0'] s.summary = "Reads events from a Kafka topic" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 51e4a4163e3b5e85ffcb8da566466d2305194cf8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 14 Dec 2021 12:03:40 +0100 Subject: [PATCH 4/8] Use latest gradle version since using old version build fails under JDK 11. --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 8de2fda..3db2d00 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.0-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-all.zip From feffd15d707bf317c8a4731486273eaf8f55a345 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 7 Jan 2022 11:14:48 +0100 Subject: [PATCH 5/8] Use kafka-clients v2.8.1. --- build.gradle | 2 +- lib/logstash-input-kafka_jars.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 20355b4..67ff1ac 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ repositories { } dependencies { - compile 'org.apache.kafka:kafka-clients:3.0.0' + compile 'org.apache.kafka:kafka-clients:2.8.1' compile 'com.github.luben:zstd-jni:1.5.0-2' compile 'org.slf4j:slf4j-api:1.7.30' compile 'org.lz4:lz4-java:1.7.1' diff --git a/lib/logstash-input-kafka_jars.rb b/lib/logstash-input-kafka_jars.rb index 82b601d..dae1fe7 100644 --- a/lib/logstash-input-kafka_jars.rb +++ b/lib/logstash-input-kafka_jars.rb @@ -1,7 +1,7 @@ # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT. require 'jar_dependencies' -require_jar('org.apache.kafka', 'kafka-clients', '3.0.0') +require_jar('org.apache.kafka', 'kafka-clients', '2.8.1') require_jar('com.github.luben', 'zstd-jni', '1.5.0-2') require_jar('org.slf4j', 'slf4j-api', '1.7.30') require_jar('org.lz4', 'lz4-java', '1.7.1') From fc98f87c868884a0c03c6d3cb8844f4650b595f6 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 7 Jan 2022 11:46:54 +0100 Subject: [PATCH 6/8] Add a comment. --- gradle/wrapper/gradle-wrapper.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3db2d00..724e193 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists +# NOTE: For building this gem under JDK 17 we need patched ruby-maven gaven and +# gradle >= 7.3 distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-all.zip From ac040b11d7d0dc23b6e13fd616d3a47780ced195 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 7 Jan 2022 13:08:33 +0100 Subject: [PATCH 7/8] Add gradle file for Gradle >= 7.3. --- build.gradle.gradle73 | 86 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 build.gradle.gradle73 diff --git a/build.gradle.gradle73 b/build.gradle.gradle73 new file mode 100644 index 0000000..464b7de --- /dev/null +++ b/build.gradle.gradle73 @@ -0,0 +1,86 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +apply plugin: "java" +apply plugin: 'maven-publish' +apply plugin: "distribution" +apply plugin: "idea" + +group "org.logstash.inputs" + +sourceCompatibility = JavaVersion.VERSION_1_8 + +buildscript { + repositories { + mavenCentral() + jcenter() + } + +} + +repositories { + mavenCentral() +} + +dependencies { + compileOnly 'org.apache.kafka:kafka-clients:2.8.1' + compileOnly 'com.github.luben:zstd-jni:1.5.0-2' + compileOnly 'org.slf4j:slf4j-api:1.7.30' + compileOnly 'org.lz4:lz4-java:1.7.1' + compileOnly 'org.xerial.snappy:snappy-java:1.1.8.1' + +} + +configurations { + compileOnlyClasspath { + // https://discuss.gradle.org/t/what-is-a-configuration-which-cant-be-directly-resolved/30721 + extendsFrom compileOnly + canBeConsumed false + canBeResolved true + } +} + +task generateGemJarRequiresFile { + doLast { + File jars_file = file('lib/logstash-input-kafka_jars.rb') + jars_file.newWriter().withWriter { w -> + w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n" + w << "require \'jar_dependencies\'\n" + configurations.runtimeClasspath.allDependencies.each { + w << "require_jar(\'${it.group}\', \'${it.name}\', \'${it.version}\')\n" + } + } + } +} + +task vendor { + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + configurations.compileOnlyClasspath.allDependencies.each { dep -> + File f = configurations.compileOnlyClasspath.filter { it.absolutePath.contains("${dep.group}/${dep.name}/${dep.version}") }.singleFile + String groupPath = dep.group.replaceAll('\\.', '/') + File newJarFile = file("${vendorPathPrefix}/${groupPath}/${dep.name}/${dep.version}/${dep.name}-${dep.version}.jar") + newJarFile.mkdirs() + Files.copy(f.toPath(), newJarFile.toPath(), REPLACE_EXISTING) + } + } +} + +vendor.dependsOn(generateGemJarRequiresFile) From ad3fa7cbe356298ba8b857a46c142f7c69ecd70a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 7 Jan 2022 13:10:31 +0100 Subject: [PATCH 8/8] Add properties file for gradle 7.3. --- gradle/wrapper/gradle-wrapper.properties.gradle73 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 gradle/wrapper/gradle-wrapper.properties.gradle73 diff --git a/gradle/wrapper/gradle-wrapper.properties.gradle73 b/gradle/wrapper/gradle-wrapper.properties.gradle73 new file mode 100644 index 0000000..f276eb9 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties.gradle73 @@ -0,0 +1,6 @@ +#Wed Jun 21 11:39:16 CEST 2017 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-all.zip