Skip to content

Commit ea0816e

Browse files
ferenc-csakyferenc-csaky
authored
Update to Flink 2.1 (#1650)
1 parent 8842a0b commit ea0816e

File tree

128 files changed

+946
-1149
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+946
-1149
lines changed

‎pom.xml‎

Lines changed: 66 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -97,94 +97,94 @@
9797
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
9898

9999
<!-- Flink runner props -->
100-
<flinkrunner.version>0.8.2</flinkrunner.version>
101-
<flinkrunner.image.repo>docker.io</flinkrunner.image.repo>
100+
<flinkrunner.image.repo>ghcr.io</flinkrunner.image.repo>
101+
<flinkrunner.version>0.9-SNAPSHOT</flinkrunner.version>
102102

103103
<!-- Params -->
104104
<docker.image.tag>local</docker.image.tag>
105105
<snapshots.update>false</snapshots.update>
106106

107+
<!-- Flink versions -->
108+
<flink.version>2.1.0</flink.version>
109+
<flink.cdc.version>3.5.0</flink.cdc.version>
110+
<flink.connector.jdbc.version>4.0.0-2.0</flink.connector.jdbc.version>
111+
<flink.connector.kafka.version>4.0.1-2.0</flink.connector.kafka.version>
112+
<flink-client.version>1.1.4</flink-client.version>
113+
107114
<!-- Dependency versions -->
108-
<mockito.version>4.8.0</mockito.version>
109-
<jackson.version>2.20.0</jackson.version>
110-
<vertx.version>5.0.4</vertx.version>
111115
<agroal.version>2.8</agroal.version>
112-
<kafka.version>3.4.0</kafka.version>
113-
<flink.version>1.19.3</flink.version>
114-
<slf4j.version>1.7.36</slf4j.version>
115-
<postgres.version>42.7.8</postgres.version>
116-
<log4j.version>2.25.2</log4j.version>
117-
<scala.version>2.12</scala.version>
118-
<guava.version>33.5.0-jre</guava.version>
119-
<commonsio.version>2.20.0</commonsio.version>
120-
<commons-lang3.version>3.19.0</commons-lang3.version>
116+
<antlr.version>4.13.2</antlr.version>
117+
<assertj.version>3.27.6</assertj.version>
118+
<autoservice.version>1.1.1</autoservice.version>
119+
<avro.version>1.12.1</avro.version>
120+
<awaitility.version>4.3.0</awaitility.version>
121+
<aws-sdk-bom.version>2.35.10</aws-sdk-bom.version>
122+
<calcite.version>1.40.0</calcite.version>
123+
<commons-bean.version>1.11.0</commons-bean.version>
124+
<commons-collections.version>4.5.0</commons-collections.version>
121125
<!-- hadoop shades in commons-config 2.1.1, so upgrade path here is very narrow -->
122126
<commons-config.version>2.3</commons-config.version>
123-
<commons-collections.version>4.5.0</commons-collections.version>
127+
<commons-csv.version>1.9.0</commons-csv.version>
124128
<commons-exec.version>1.5.0</commons-exec.version>
125-
<httpcomponents.version>4.5.14</httpcomponents.version>
126-
<commons-bean.version>1.11.0</commons-bean.version>
127-
<h2.version>2.4.240</h2.version>
128-
<opencsv.version>5.12.0</opencsv.version>
129-
<javax.validation.version>2.0.1.Final</javax.validation.version>
130-
<glassfish.version>3.0.0</glassfish.version>
131-
<reactive-streams.version>1.0.4</reactive-streams.version>
132-
<testcontainers.version>1.21.3</testcontainers.version>
133-
<scram.version>2.1</scram.version>
134-
<protobuf.version>4.33.0</protobuf.version>
135-
<picocli.version>4.7.7</picocli.version>
136129
<commonscodec.version>1.19.0</commonscodec.version>
137-
<guice.version>5.1.0</guice.version>
138-
<autoservice.version>1.1.1</autoservice.version>
139-
<antlr.version>4.13.2</antlr.version>
140-
<guava.version>33.5.0-jre</guava.version>
141-
<lombok.version>1.18.42</lombok.version>
142-
<junit.jupiter.version>5.13.4</junit.jupiter.version>
143-
<calcite.version>1.27.0</calcite.version>
144-
<graphql-java.version>19.2</graphql-java.version>
145-
<graphql-java-extended-scalars.version>22.0</graphql-java-extended-scalars.version>
146-
<projectreactor.version>3.7.12</projectreactor.version>
147-
<avro.version>1.12.1</avro.version>
130+
<commonsio.version>2.20.0</commonsio.version>
131+
<commons-lang3.version>3.19.0</commons-lang3.version>
132+
<duckdb.version>1.4.1.0</duckdb.version>
133+
<feign.version>13.6</feign.version>
148134
<freemarker.version>2.3.34</freemarker.version>
149-
<mustache.version>0.9.14</mustache.version>
150-
<jsonschema.version>1.5.9</jsonschema.version>
151-
<iceberg.version>1.9.2</iceberg.version>
135+
<glassfish.version>3.0.0</glassfish.version>
136+
<graphql-java-extended-scalars.version>22.0</graphql-java-extended-scalars.version>
137+
<graphql-java.version>19.2</graphql-java.version>
138+
<graphql-micrometer.version>1.0.1</graphql-micrometer.version>
139+
<guava.version>33.5.0-jre</guava.version>
140+
<guice-multibindings.version>4.2.3</guice-multibindings.version>
141+
<guice.version>7.0.0</guice.version>
142+
<h2.version>2.4.240</h2.version>
152143
<!-- Do not touch Hadoop version unless it has been updated on the Flink side.
153144
Otherwise, it might break the Flink filesystem integration. -->
154145
<hadoop.version>3.3.4</hadoop.version>
146+
<httpcomponents.version>4.5.14</httpcomponents.version>
147+
<iceberg.version>1.10.0</iceberg.version>
148+
<jackson.version>2.20.0</jackson.version>
155149
<jacoco.version>0.8.14</jacoco.version>
156-
<jjwt.version>0.13.0</jjwt.version>
157-
<assertj.version>3.27.6</assertj.version>
158-
<awaitility.version>4.3.0</awaitility.version>
159150
<jakarta.annotation.version>3.0.0</jakarta.annotation.version>
160-
<graphql-micrometer.version>1.0.1</graphql-micrometer.version>
151+
<javax.validation.version>2.0.1.Final</javax.validation.version>
152+
<jjwt.version>0.13.0</jjwt.version>
153+
<jsonschema.version>1.5.9</jsonschema.version>
154+
<junit.jupiter.version>5.13.4</junit.jupiter.version>
155+
<kafka.version>3.4.0</kafka.version>
156+
<log4j.version>2.25.2</log4j.version>
157+
<lombok.version>1.18.42</lombok.version>
158+
<mcp-sdk.version>0.14.1</mcp-sdk.version>
161159
<micrometer.version>1.15.5</micrometer.version>
162-
<calcite-core.version>1.40.0</calcite-core.version>
163-
<duckdb.version>1.4.1.0</duckdb.version>
164-
<snowflake-jdbc.version>3.27.0</snowflake-jdbc.version>
160+
<mockito.version>4.8.0</mockito.version>
161+
<mustache.version>0.9.14</mustache.version>
162+
<openai-gpt3-java.version>0.18.2</openai-gpt3-java.version>
163+
<opencsv.version>5.12.0</opencsv.version>
164+
<picocli.version>4.7.7</picocli.version>
165+
<postgres.version>42.7.8</postgres.version>
166+
<projectreactor.version>3.7.12</projectreactor.version>
167+
<protobuf.version>4.33.0</protobuf.version>
168+
<reactive-streams.version>1.0.4</reactive-streams.version>
165169
<reflections.version>0.10.2</reflections.version>
166-
<swagger-core.version>2.2.39</swagger-core.version>
167-
<commons-csv.version>1.9.0</commons-csv.version>
168-
<guice.version>7.0.0</guice.version>
169-
<guice-multibindings.version>4.2.3</guice-multibindings.version>
170-
<flink-connector-postgres-cdc.version>3.4.0</flink-connector-postgres-cdc.version>
171-
<zookeeper.version>3.9.4</zookeeper.version>
170+
<scala.version>2.12</scala.version>
171+
<scram.version>2.1</scram.version>
172+
<slf4j.version>1.7.36</slf4j.version>
173+
<snowflake-jdbc.version>3.27.0</snowflake-jdbc.version>
172174
<snakeyaml.version>2.5</snakeyaml.version>
173-
<aws-sdk-bom.version>2.35.10</aws-sdk-bom.version>
174-
<openai-gpt3-java.version>0.18.2</openai-gpt3-java.version>
175-
<flink-client.version>1.1.4</flink-client.version>
176-
<mcp-sdk.version>0.14.1</mcp-sdk.version>
177-
<feign.version>13.6</feign.version>
175+
<swagger-core.version>2.2.39</swagger-core.version>
176+
<testcontainers.version>1.21.3</testcontainers.version>
177+
<vertx.version>5.0.4</vertx.version>
178178

179179
<!-- Plugin versions -->
180+
<dockerfile-maven-plugin.version>1.4.13</dockerfile-maven-plugin.version>
181+
<easy-jacoco-maven-plugin.version>0.1.4</easy-jacoco-maven-plugin.version>
182+
<git-code-format-maven-plugin.version>5.4</git-code-format-maven-plugin.version>
183+
<license-maven-plugin.version>5.0.0</license-maven-plugin.version>
184+
<maven-gpg-plugin.version>3.2.8</maven-gpg-plugin.version>
180185
<maven-javadoc-plugin.version>3.12.0</maven-javadoc-plugin.version>
181186
<maven-source-plugin.version>3.3.1</maven-source-plugin.version>
182-
<maven-gpg-plugin.version>3.2.8</maven-gpg-plugin.version>
183-
<license-maven-plugin.version>5.0.0</license-maven-plugin.version>
184-
<git-code-format-maven-plugin.version>5.4</git-code-format-maven-plugin.version>
185187
<sortpom.plugin.version>4.0.0</sortpom.plugin.version>
186-
<dockerfile-maven-plugin.version>1.4.13</dockerfile-maven-plugin.version>
187-
<easy-jacoco-maven-plugin.version>0.1.4</easy-jacoco-maven-plugin.version>
188188
</properties>
189189

190190
<dependencyManagement>
@@ -479,13 +479,13 @@
479479
</dependency>
480480
<dependency>
481481
<groupId>org.apache.flink</groupId>
482-
<artifactId>flink-connector-jdbc</artifactId>
483-
<version>3.2.0-1.19</version>
482+
<artifactId>flink-connector-jdbc-postgres</artifactId>
483+
<version>${flink.connector.jdbc.version}</version>
484484
</dependency>
485485
<dependency>
486486
<groupId>org.apache.flink</groupId>
487487
<artifactId>flink-connector-kafka</artifactId>
488-
<version>3.2.0-1.19</version>
488+
<version>${flink.connector.kafka.version}</version>
489489
</dependency>
490490

491491
<!-- calcite -->

‎sqrl-cli/pom.xml‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -106,16 +106,16 @@
106106
</dependency>
107107
<dependency>
108108
<groupId>org.apache.flink</groupId>
109-
<artifactId>flink-connector-kafka</artifactId>
109+
<artifactId>flink-connector-jdbc-postgres</artifactId>
110110
</dependency>
111111
<dependency>
112112
<groupId>org.apache.flink</groupId>
113-
<artifactId>flink-connector-jdbc</artifactId>
113+
<artifactId>flink-connector-kafka</artifactId>
114114
</dependency>
115115
<dependency>
116116
<groupId>org.apache.flink</groupId>
117117
<artifactId>flink-connector-postgres-cdc</artifactId>
118-
<version>${flink-connector-postgres-cdc.version}</version>
118+
<version>${flink.cdc.version}</version>
119119
</dependency>
120120

121121
<!-- Kafka dependencies -->
@@ -188,11 +188,6 @@
188188
<artifactId>postgresql</artifactId>
189189
<scope>compile</scope>
190190
</dependency>
191-
<dependency>
192-
<groupId>org.apache.zookeeper</groupId>
193-
<artifactId>zookeeper</artifactId>
194-
<version>${zookeeper.version}</version>
195-
</dependency>
196191

197192
<!-- Hadoop dependencies -->
198193
<dependency>
@@ -269,12 +264,7 @@
269264
</dependency>
270265
<dependency>
271266
<groupId>org.apache.iceberg</groupId>
272-
<artifactId>iceberg-flink-runtime-1.19</artifactId>
273-
<version>${iceberg.version}</version>
274-
</dependency>
275-
<dependency>
276-
<groupId>org.apache.iceberg</groupId>
277-
<artifactId>iceberg-hive-metastore</artifactId>
267+
<artifactId>iceberg-flink-runtime-2.0</artifactId>
278268
<version>${iceberg.version}</version>
279269
</dependency>
280270

‎sqrl-cli/src/main/java/com/datasqrl/cli/DatasqrlRun.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@
6363
import org.apache.flink.configuration.CheckpointingOptions;
6464
import org.apache.flink.configuration.Configuration;
6565
import org.apache.flink.configuration.ExecutionOptions;
66+
import org.apache.flink.configuration.StateRecoveryOptions;
6667
import org.apache.flink.core.execution.SavepointFormatType;
67-
import org.apache.flink.runtime.jobgraph.SavepointConfigOptions;
6868
import org.apache.flink.table.api.TableResult;
6969
import org.apache.kafka.clients.admin.AdminClient;
7070
import org.apache.kafka.clients.admin.AdminClientConfig;
@@ -196,7 +196,7 @@ private TableResult runFlinkJob() {
196196
.ifPresent(
197197
sp -> {
198198
log.info("Trying to restore from savepoint: {}", sp);
199-
flinkConfig.set(SavepointConfigOptions.SAVEPOINT_PATH, sp);
199+
flinkConfig.set(StateRecoveryOptions.SAVEPOINT_PATH, sp);
200200
});
201201

202202
var runner = new SqrlRunner(execMode, flinkConfig, resolver, sqlFile, planFile, udfPath);

‎sqrl-planner/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@
137137
</dependency>
138138
<dependency>
139139
<groupId>org.apache.flink</groupId>
140-
<artifactId>flink-connector-jdbc</artifactId>
140+
<artifactId>flink-connector-jdbc-postgres</artifactId>
141141
</dependency>
142142
<dependency>
143143
<groupId>org.apache.flink</groupId>

‎sqrl-planner/src/main/java/com/datasqrl/calcite/OperatorRuleTransformer.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import com.datasqrl.calcite.function.OperatorRuleTransform;
1919
import com.datasqrl.canonicalizer.Name;
20+
import com.datasqrl.plan.rules.SqrlRelMetadataProvider;
2021
import com.datasqrl.util.ServiceLoaderDiscovery;
2122
import java.util.HashMap;
2223
import java.util.List;
@@ -65,7 +66,7 @@ public RelNode convert(RelNode relNode) {
6566
.collect(Collectors.toList());
6667
// Apply the rules to relnode
6768
relNode =
68-
Programs.hep(rules, false, null)
69+
Programs.hep(rules, false, SqrlRelMetadataProvider.INSTANCE)
6970
.run(null, relNode, relNode.getTraitSet(), List.of(), List.of());
7071

7172
return relNode;

‎sqrl-planner/src/main/java/com/datasqrl/calcite/SqrlConfigurations.java‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,7 @@
1616
package com.datasqrl.calcite;
1717

1818
import java.util.function.UnaryOperator;
19-
import org.apache.calcite.avatica.util.Quoting;
2019
import org.apache.calcite.sql.SqlWriterConfig;
21-
import org.apache.calcite.sql.parser.SqlParser;
22-
import org.apache.calcite.sql.validate.SqlConformanceEnum;
23-
import org.apache.calcite.sql.validate.SqlValidator;
24-
import org.apache.calcite.sql2rel.SqlToRelConverter;
2520

2621
public class SqrlConfigurations {
2722

@@ -32,25 +27,4 @@ public class SqrlConfigurations {
3227
.withUpdateSetListNewline(false)
3328
.withIndentation(1)
3429
.withSelectFolding(null);
35-
36-
public static final SqlToRelConverter.Config sqlToRelConverterConfig =
37-
SqlToRelConverter.config()
38-
.withExpand(false)
39-
.withDecorrelationEnabled(false)
40-
.withTrimUnusedFields(false);
41-
42-
public static SqlValidator.Config sqlValidatorConfig =
43-
SqlValidator.Config.DEFAULT
44-
.withCallRewrite(true)
45-
.withIdentifierExpansion(false)
46-
.withColumnReferenceExpansion(true)
47-
.withTypeCoercionEnabled(true) // must be true to allow null literals
48-
.withLenientOperatorLookup(false)
49-
.withSqlConformance(SqrlConformance.INSTANCE);
50-
51-
public static SqlParser.Config calciteParserConfig =
52-
SqlParser.config()
53-
.withCaseSensitive(true)
54-
.withConformance(SqlConformanceEnum.DEFAULT)
55-
.withQuoting(Quoting.DOUBLE_QUOTE);
5630
}

0 commit comments

Comments
 (0)