Skip to content

Commit 2d0309c

Browse files
darenwktdarenwkt
committed
[FLINK-37688] Implement Amazon CloudWatch Metric Sink Connector
1 parent 9d6746b commit 2d0309c

File tree

42 files changed

+4292
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+4292
-0
lines changed

‎flink-connector-aws/flink-connector-cloudwatch/archunit-violations/70dddb8b-045a-45a9-80f2-a32ad375d87f

Whitespace-only changes.

‎flink-connector-aws/flink-connector-cloudwatch/archunit-violations/9577f915-0c2c-4536-a06d-561accc2e4b6

Whitespace-only changes.

‎flink-connector-aws/flink-connector-cloudwatch/archunit-violations/stored.rules

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#
2+
#Wed Apr 16 17:34:20 BST 2025
3+
ITCASE\ tests\ should\ use\ a\ MiniCluster\ resource\ or\ extension=70dddb8b-045a-45a9-80f2-a32ad375d87f
4+
Tests\ inheriting\ from\ AbstractTestBase\ should\ have\ name\ ending\ with\ ITCase=9577f915-0c2c-4536-a06d-561accc2e4b6

‎flink-connector-aws/flink-connector-cloudwatch/pom.xml

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
23+
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<parent>
27+
<groupId>org.apache.flink</groupId>
28+
<artifactId>flink-connector-aws-parent</artifactId>
29+
<version>5.1-SNAPSHOT</version>
30+
</parent>
31+
32+
<artifactId>flink-connector-cloudwatch</artifactId>
33+
<name>Flink : Connectors : AWS : Amazon Cloudwatch</name>
34+
<packaging>jar</packaging>
35+
36+
<properties>
37+
<aws.sdkv2.version>2.31.18</aws.sdkv2.version>
38+
<flink.version>1.19.0</flink.version>
39+
</properties>
40+
41+
<dependencies>
42+
<dependency>
43+
<groupId>org.apache.flink</groupId>
44+
<artifactId>flink-streaming-java</artifactId>
45+
<version>${flink.version}</version>
46+
<scope>provided</scope>
47+
</dependency>
48+
49+
<dependency>
50+
<groupId>org.apache.flink</groupId>
51+
<artifactId>flink-connector-aws-base</artifactId>
52+
<version>${project.version}</version>
53+
</dependency>
54+
55+
<dependency>
56+
<groupId>software.amazon.awssdk</groupId>
57+
<artifactId>cloudwatch</artifactId>
58+
</dependency>
59+
60+
<dependency>
61+
<groupId>software.amazon.awssdk</groupId>
62+
<artifactId>netty-nio-client</artifactId>
63+
</dependency>
64+
65+
<!-- Test dependencies -->
66+
<dependency>
67+
<groupId>org.apache.flink</groupId>
68+
<artifactId>flink-test-utils</artifactId>
69+
<version>${flink.version}</version>
70+
<scope>test</scope>
71+
</dependency>
72+
<dependency>
73+
<groupId>org.apache.flink</groupId>
74+
<artifactId>flink-connector-test-utils</artifactId>
75+
<version>${flink.version}</version>
76+
<scope>test</scope>
77+
</dependency>
78+
79+
<dependency>
80+
<groupId>org.apache.flink</groupId>
81+
<artifactId>flink-connector-aws-base</artifactId>
82+
<version>${project.version}</version>
83+
<type>test-jar</type>
84+
<scope>test</scope>
85+
</dependency>
86+
87+
<dependency>
88+
<groupId>org.apache.flink</groupId>
89+
<artifactId>flink-connector-base</artifactId>
90+
<version>${flink.version}</version>
91+
<type>test-jar</type>
92+
<scope>test</scope>
93+
</dependency>
94+
95+
<dependency>
96+
<groupId>org.testcontainers</groupId>
97+
<artifactId>testcontainers</artifactId>
98+
<scope>test</scope>
99+
</dependency>
100+
101+
<!-- Table API dependencies-->
102+
<dependency>
103+
<groupId>org.apache.flink</groupId>
104+
<artifactId>flink-table-common</artifactId>
105+
<version>${flink.version}</version>
106+
<scope>provided</scope>
107+
</dependency>
108+
109+
<dependency>
110+
<groupId>org.apache.flink</groupId>
111+
<artifactId>flink-table-runtime</artifactId>
112+
<version>${flink.version}</version>
113+
<scope>test</scope>
114+
</dependency>
115+
116+
<dependency>
117+
<groupId>org.apache.flink</groupId>
118+
<artifactId>flink-table-common</artifactId>
119+
<version>${flink.version}</version>
120+
<scope>test</scope>
121+
<type>test-jar</type>
122+
</dependency>
123+
124+
<!-- ArchUnit test dependencies -->
125+
126+
<dependency>
127+
<groupId>org.apache.flink</groupId>
128+
<artifactId>flink-architecture-tests-test</artifactId>
129+
<scope>test</scope>
130+
</dependency>
131+
132+
<dependency>
133+
<groupId>software.amazon.awssdk</groupId>
134+
<artifactId>s3</artifactId>
135+
<scope>test</scope>
136+
</dependency>
137+
138+
<dependency>
139+
<groupId>software.amazon.awssdk</groupId>
140+
<artifactId>bom</artifactId>
141+
<version>${aws.sdkv2.version}</version>
142+
<type>pom</type>
143+
<scope>import</scope>
144+
</dependency>
145+
</dependencies>
146+
147+
<dependencyManagement>
148+
<dependencies>
149+
150+
<!-- Overridden aws-sdk dependency to older version to temporarily fix 'not able to create sqs localstack error with newer version'-->
151+
<dependency>
152+
<groupId>software.amazon.awssdk</groupId>
153+
<artifactId>bom</artifactId>
154+
<version>${aws.sdkv2.version}</version>
155+
<type>pom</type>
156+
<scope>import</scope>
157+
</dependency>
158+
</dependencies>
159+
</dependencyManagement>
160+
161+
<build>
162+
<plugins>
163+
<plugin>
164+
<groupId>org.apache.maven.plugins</groupId>
165+
<artifactId>maven-jar-plugin</artifactId>
166+
<executions>
167+
<execution>
168+
<goals>
169+
<goal>test-jar</goal>
170+
</goals>
171+
</execution>
172+
</executions>
173+
</plugin>
174+
175+
<plugin>
176+
<groupId>org.apache.maven.plugins</groupId>
177+
<artifactId>maven-shade-plugin</artifactId>
178+
<executions>
179+
<execution>
180+
<id>shade-flink</id>
181+
<phase>package</phase>
182+
<goals>
183+
<goal>shade</goal>
184+
</goals>
185+
<configuration>
186+
<artifactSet combine.children="append">
187+
<includes>
188+
<include>org.apache.flink:flink-connector-aws-base:*</include>
189+
<include>com.amazonaws:*</include>
190+
<include>software.amazon.awssdk:*</include>
191+
<include>org.reactivestreams:*</include>
192+
<include>io.netty:*</include>
193+
<include>com.typesafe.netty:*</include>
194+
</includes>
195+
</artifactSet>
196+
<relocations combine.children="override">
197+
<!-- Do not relocate guava because it is exposed in the Kinesis API (KinesisProducer#addUserRecord).
198+
Users may be using other affected API's, so relocations may break user-code -->
199+
<relocation>
200+
<pattern>org.apache.flink.connector.aws</pattern>
201+
<shadedPattern>
202+
org.apache.flink.cloudwatch.shaded.org.apache.flink.connector.aws
203+
</shadedPattern>
204+
</relocation>
205+
<relocation>
206+
<pattern>com.amazonaws</pattern>
207+
<shadedPattern>org.apache.flink.cloudwatch.shaded.com.amazonaws</shadedPattern>
208+
</relocation>
209+
<relocation>
210+
<pattern>io.netty</pattern>
211+
<shadedPattern>org.apache.flink.cloudwatch.shaded.io.netty</shadedPattern>
212+
</relocation>
213+
<relocation>
214+
<pattern>com.typesafe.netty</pattern>
215+
<shadedPattern>org.apache.flink.cloudwatch.shaded.com.typesafe.netty</shadedPattern>
216+
</relocation>
217+
<relocation>
218+
<pattern>software.amazon</pattern>
219+
<shadedPattern>org.apache.flink.cloudwatch.shaded.software.amazon</shadedPattern>
220+
</relocation>
221+
<relocation>
222+
<pattern>org.reactivestreams</pattern>
223+
<shadedPattern>org.apache.flink.cloudwatch.shaded.org.reactivestreams</shadedPattern>
224+
</relocation>
225+
</relocations>
226+
<filters>
227+
<filter>
228+
<artifact>*:*</artifact>
229+
<excludes>
230+
<exclude>.gitkeep</exclude>
231+
</excludes>
232+
</filter>
233+
<filter>
234+
<artifact>org.apache.flink:flink-connector-aws-base:*</artifact>
235+
<excludes>
236+
<exclude>profile</exclude>
237+
</excludes>
238+
</filter>
239+
</filters>
240+
</configuration>
241+
</execution>
242+
</executions>
243+
</plugin>
244+
</plugins>
245+
</build>
246+
</project>

‎flink-connector-aws/flink-connector-cloudwatch/src/main/java/org/apache/flink/connector/cloudwatch/sink/CloudWatchConfigConstants.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.flink.connector.cloudwatch.sink;
19+
20+
import org.apache.flink.annotation.PublicEvolving;
21+
import org.apache.flink.configuration.ConfigOption;
22+
import org.apache.flink.configuration.ConfigOptions;
23+
24+
/** Defaults for {@link CloudWatchSinkWriter}. */
25+
@PublicEvolving
26+
public class CloudWatchConfigConstants {
27+
28+
public static final ConfigOption<String> BASE_CLOUDWATCH_USER_AGENT_PREFIX_FORMAT =
29+
ConfigOptions.key("Apache Flink %s (%s) CloudWatch Connector")
30+
.stringType()
31+
.noDefaultValue()
32+
.withDescription("CloudWatch useragent prefix format.");
33+
34+
public static final ConfigOption<String> CLOUDWATCH_CLIENT_USER_AGENT_PREFIX =
35+
ConfigOptions.key("aws.cloudwatch.client.user-agent-prefix")
36+
.stringType()
37+
.noDefaultValue()
38+
.withDescription("CloudWatch identifier for user agent prefix.");
39+
}

‎flink-connector-aws/flink-connector-cloudwatch/src/main/java/org/apache/flink/connector/cloudwatch/sink/CloudWatchExceptionClassifiers.java

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.flink.connector.cloudwatch.sink;
19+
20+
import org.apache.flink.annotation.Internal;
21+
import org.apache.flink.connector.aws.sink.throwable.AWSExceptionClassifierUtil;
22+
import org.apache.flink.connector.base.sink.throwable.FatalExceptionClassifier;
23+
24+
import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException;
25+
import software.amazon.awssdk.services.cloudwatch.model.ResourceNotFoundException;
26+
27+
/**
28+
* Class containing set of {@link FatalExceptionClassifier} for {@link
29+
* software.amazon.awssdk.services.cloudwatch.model.CloudWatchException}.
30+
*/
31+
@Internal
32+
public class CloudWatchExceptionClassifiers {
33+
34+
public static FatalExceptionClassifier getNotAuthorizedExceptionClassifier() {
35+
return AWSExceptionClassifierUtil.withAWSServiceErrorCode(
36+
CloudWatchException.class,
37+
"NotAuthorized",
38+
err ->
39+
new CloudWatchSinkException(
40+
"Encountered non-recoverable exception: NotAuthorized", err));
41+
}
42+
43+
public static FatalExceptionClassifier getAccessDeniedExceptionClassifier() {
44+
return AWSExceptionClassifierUtil.withAWSServiceErrorCode(
45+
CloudWatchException.class,
46+
"AccessDeniedException",
47+
err ->
48+
new CloudWatchSinkException(
49+
"Encountered non-recoverable exception: AccessDeniedException",
50+
err));
51+
}
52+
53+
public static FatalExceptionClassifier getResourceNotFoundExceptionClassifier() {
54+
return FatalExceptionClassifier.withRootCauseOfType(
55+
ResourceNotFoundException.class,
56+
err ->
57+
new CloudWatchSinkException(
58+
"Encountered non-recoverable exception relating to not being able to find the specified resources",
59+
err));
60+
}
61+
}

0 commit comments

Comments
 (0)