Skip to content

Commit 40984bf

Browse files
germanosiniliax
germanosin
and
iliax
authored
BE: Metrics collection / storage / quering (#1208)
Co-authored-by: iliax <[email protected]>
1 parent bac16cc commit 40984bf

File tree

108 files changed

+4368
-1077
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+4368
-1077
lines changed

‎api/build.gradle

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ dependencies {
3434

3535
implementation libs.apache.avro
3636
implementation libs.apache.commons
37+
implementation libs.apache.commons.text
3738
implementation libs.apache.commons.pool2
3839
implementation libs.apache.datasketches
3940

@@ -81,6 +82,13 @@ dependencies {
8182
because("CVE Fix: It is excluded above because of a vulnerability")
8283
}
8384

85+
implementation libs.prometheus.metrics.core
86+
implementation libs.prometheus.metrics.textformats
87+
implementation (libs.prometheus.metrics.exporter.pushgateway) {
88+
exclude group: 'com.google.protobuf', module: 'protobuf-java' because("PushGW lib pulls protobuf-java 4.x, which is incompatible with protobuf-java 3.x used by various dependencies of this project.")
89+
}
90+
implementation libs.snappy
91+
8492
// Annotation processors
8593
implementation libs.lombok
8694
implementation libs.mapstruct
@@ -107,11 +115,11 @@ dependencies {
107115

108116
testImplementation libs.okhttp3
109117
testImplementation libs.okhttp3.mockwebserver
118+
testImplementation libs.prometheus.metrics.core
110119
}
111120

112121
generateGrammarSource {
113122
maxHeapSize = "64m"
114-
arguments += ["-package", "ksql"]
115123
}
116124

117125
tasks.withType(JavaCompile) {
@@ -133,6 +141,7 @@ sourceSets {
133141

134142
tasks.withType(Checkstyle).configureEach {
135143
exclude '**/ksql/**'
144+
exclude '**/promql/**'
136145
}
137146

138147
checkstyle {

‎api/src/main/antlr/ksql/KsqlGrammar.g4

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
grammar KsqlGrammar;
22

3+
@header {package ksql;}
4+
5+
36
tokens {
47
DELIMITER
58
}

‎api/src/main/antlr/promql/PromQL.g4

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
grammar PromQL;
2+
3+
@header {package promql;}
4+
5+
options {
6+
caseInsensitive = true;
7+
}
8+
9+
expression: vectorOperation EOF;
10+
11+
// Binary operations are ordered by precedence
12+
13+
// Unary operations have the same precedence as multiplications
14+
15+
vectorOperation
16+
: <assoc=right> vectorOperation powOp vectorOperation
17+
| <assoc=right> vectorOperation subqueryOp
18+
| unaryOp vectorOperation
19+
| vectorOperation multOp vectorOperation
20+
| vectorOperation addOp vectorOperation
21+
| vectorOperation compareOp vectorOperation
22+
| vectorOperation andUnlessOp vectorOperation
23+
| vectorOperation orOp vectorOperation
24+
| vectorOperation vectorMatchOp vectorOperation
25+
| vectorOperation AT vectorOperation
26+
| vector
27+
;
28+
29+
// Operators
30+
31+
unaryOp: (ADD | SUB);
32+
powOp: POW grouping?;
33+
multOp: (MULT | DIV | MOD) grouping?;
34+
addOp: (ADD | SUB) grouping?;
35+
compareOp: (DEQ | NE | GT | LT | GE | LE) BOOL? grouping?;
36+
andUnlessOp: (AND | UNLESS) grouping?;
37+
orOp: OR grouping?;
38+
vectorMatchOp: (ON | UNLESS) grouping?;
39+
subqueryOp: SUBQUERY_RANGE offsetOp?;
40+
offsetOp: OFFSET DURATION;
41+
42+
vector
43+
: function_
44+
| aggregation
45+
| instantSelector
46+
| matrixSelector
47+
| offset
48+
| literal
49+
| parens
50+
;
51+
52+
parens: LEFT_PAREN vectorOperation RIGHT_PAREN;
53+
54+
// Selectors
55+
56+
instantSelector
57+
: METRIC_NAME (LEFT_BRACE labelMatcherList? RIGHT_BRACE)?
58+
| LEFT_BRACE labelMatcherList RIGHT_BRACE
59+
;
60+
61+
labelMatcher: labelName labelMatcherOperator STRING;
62+
labelMatcherOperator: EQ | NE | RE | NRE;
63+
labelMatcherList: labelMatcher (COMMA labelMatcher)* COMMA?;
64+
65+
matrixSelector: instantSelector TIME_RANGE;
66+
67+
offset
68+
: instantSelector OFFSET DURATION
69+
| matrixSelector OFFSET DURATION
70+
;
71+
72+
// Functions
73+
74+
function_: FUNCTION LEFT_PAREN (parameter (COMMA parameter)*)? RIGHT_PAREN;
75+
76+
parameter: literal | vectorOperation;
77+
parameterList: LEFT_PAREN (parameter (COMMA parameter)*)? RIGHT_PAREN;
78+
79+
// Aggregations
80+
81+
aggregation
82+
: AGGREGATION_OPERATOR parameterList
83+
| AGGREGATION_OPERATOR (by | without) parameterList
84+
| AGGREGATION_OPERATOR parameterList ( by | without)
85+
;
86+
by: BY labelNameList;
87+
without: WITHOUT labelNameList;
88+
89+
// Vector one-to-one/one-to-many joins
90+
91+
grouping: (on_ | ignoring) (groupLeft | groupRight)?;
92+
on_: ON labelNameList;
93+
ignoring: IGNORING labelNameList;
94+
groupLeft: GROUP_LEFT labelNameList?;
95+
groupRight: GROUP_RIGHT labelNameList?;
96+
97+
// Label names
98+
99+
labelName: keyword | METRIC_NAME | LABEL_NAME;
100+
labelNameList: LEFT_PAREN (labelName (COMMA labelName)*)? RIGHT_PAREN;
101+
102+
keyword
103+
: AND
104+
| OR
105+
| UNLESS
106+
| BY
107+
| WITHOUT
108+
| ON
109+
| IGNORING
110+
| GROUP_LEFT
111+
| GROUP_RIGHT
112+
| OFFSET
113+
| BOOL
114+
| AGGREGATION_OPERATOR
115+
| FUNCTION
116+
;
117+
118+
literal: NUMBER | STRING;
119+
120+
fragment NUMERAL: [0-9]+ ('.' [0-9]+)?;
121+
122+
fragment SCIENTIFIC_NUMBER
123+
: NUMERAL ('e' [-+]? NUMERAL)?
124+
;
125+
126+
NUMBER
127+
: NUMERAL
128+
| SCIENTIFIC_NUMBER;
129+
130+
STRING
131+
: '\'' (~('\'' | '\\') | '\\' .)* '\''
132+
| '"' (~('"' | '\\') | '\\' .)* '"'
133+
;
134+
135+
// Binary operators
136+
137+
ADD: '+';
138+
SUB: '-';
139+
MULT: '*';
140+
DIV: '/';
141+
MOD: '%';
142+
POW: '^';
143+
144+
AND: 'and';
145+
OR: 'or';
146+
UNLESS: 'unless';
147+
148+
// Comparison operators
149+
150+
EQ: '=';
151+
DEQ: '==';
152+
NE: '!=';
153+
GT: '>';
154+
LT: '<';
155+
GE: '>=';
156+
LE: '<=';
157+
RE: '=~';
158+
NRE: '!~';
159+
160+
// Aggregation modifiers
161+
162+
BY: 'by';
163+
WITHOUT: 'without';
164+
165+
// Join modifiers
166+
167+
ON: 'on';
168+
IGNORING: 'ignoring';
169+
GROUP_LEFT: 'group_left';
170+
GROUP_RIGHT: 'group_right';
171+
172+
OFFSET: 'offset';
173+
174+
BOOL: 'bool';
175+
176+
AGGREGATION_OPERATOR
177+
: 'sum'
178+
| 'min'
179+
| 'max'
180+
| 'avg'
181+
| 'group'
182+
| 'stddev'
183+
| 'stdvar'
184+
| 'count'
185+
| 'count_values'
186+
| 'bottomk'
187+
| 'topk'
188+
| 'quantile'
189+
;
190+
191+
FUNCTION
192+
: 'abs'
193+
| 'absent'
194+
| 'absent_over_time'
195+
| 'ceil'
196+
| 'changes'
197+
| 'clamp_max'
198+
| 'clamp_min'
199+
| 'day_of_month'
200+
| 'day_of_week'
201+
| 'days_in_month'
202+
| 'delta'
203+
| 'deriv'
204+
| 'exp'
205+
| 'floor'
206+
| 'histogram_quantile'
207+
| 'holt_winters'
208+
| 'hour'
209+
| 'idelta'
210+
| 'increase'
211+
| 'irate'
212+
| 'label_join'
213+
| 'label_replace'
214+
| 'ln'
215+
| 'log2'
216+
| 'log10'
217+
| 'minute'
218+
| 'month'
219+
| 'predict_linear'
220+
| 'rate'
221+
| 'resets'
222+
| 'round'
223+
| 'scalar'
224+
| 'sort'
225+
| 'sort_desc'
226+
| 'sqrt'
227+
| 'time'
228+
| 'timestamp'
229+
| 'vector'
230+
| 'year'
231+
| 'avg_over_time'
232+
| 'min_over_time'
233+
| 'max_over_time'
234+
| 'sum_over_time'
235+
| 'count_over_time'
236+
| 'quantile_over_time'
237+
| 'stddev_over_time'
238+
| 'stdvar_over_time'
239+
| 'last_over_time'
240+
| 'acos'
241+
| 'acosh'
242+
| 'asin'
243+
| 'asinh'
244+
| 'atan'
245+
| 'atanh'
246+
| 'cos'
247+
| 'cosh'
248+
| 'sin'
249+
| 'sinh'
250+
| 'tan'
251+
| 'tanh'
252+
| 'deg'
253+
| 'pi'
254+
| 'rad'
255+
;
256+
257+
LEFT_BRACE: '{';
258+
RIGHT_BRACE: '}';
259+
260+
LEFT_PAREN: '(';
261+
RIGHT_PAREN: ')';
262+
263+
LEFT_BRACKET: '[';
264+
RIGHT_BRACKET: ']';
265+
266+
COMMA: ',';
267+
268+
AT: '@';
269+
270+
SUBQUERY_RANGE
271+
: LEFT_BRACKET DURATION ':' DURATION? RIGHT_BRACKET;
272+
273+
TIME_RANGE
274+
: LEFT_BRACKET DURATION RIGHT_BRACKET;
275+
276+
// The proper order (longest to the shortest) must be validated after parsing
277+
DURATION: ([0-9]+ ('ms' | [smhdwy]))+;
278+
279+
METRIC_NAME: [a-z_:] [a-z0-9_:]*;
280+
LABEL_NAME: [a-z_] [a-z0-9_]*;
281+
282+
283+
284+
WS: [\r\t\n ]+ -> channel(HIDDEN);
285+
SL_COMMENT
286+
: '#' .*? '\n' -> channel(HIDDEN)
287+
;

‎api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package io.kafbat.ui.client;
22

3+
import static org.apache.commons.lang3.Strings.CI;
4+
35
import com.fasterxml.jackson.annotation.JsonProperty;
46
import io.kafbat.ui.config.ClustersProperties;
57
import io.kafbat.ui.connect.ApiClient;
@@ -22,7 +24,6 @@
2224
import java.util.Objects;
2325
import javax.annotation.Nullable;
2426
import lombok.extern.slf4j.Slf4j;
25-
import org.apache.commons.lang3.StringUtils;
2627
import org.springframework.http.ResponseEntity;
2728
import org.springframework.util.unit.DataSize;
2829
import org.springframework.web.client.RestClientException;
@@ -58,7 +59,7 @@ private static Retry conflictCodeRetry() {
5859

5960
if (e instanceof WebClientResponseException.InternalServerError exception) {
6061
final var errorMessage = getMessage(exception);
61-
return StringUtils.equals(errorMessage,
62+
return CI.equals(errorMessage,
6263
// From https://github.com/apache/kafka/blob/dfc07e0e0c6e737a56a5402644265f634402b864/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java#L2340
6364
"Request cannot be completed because a rebalance is expected");
6465
}

0 commit comments

Comments
 (0)