From 8b7b32d71faf80aa21d099439b4c3ea2cbe83abb Mon Sep 17 00:00:00 2001 From: German Osin Date: Tue, 13 May 2025 23:01:20 +0200 Subject: [PATCH 01/29] metrics implementation --- api/build.gradle | 19 +- .../{antlr4 => antlr}/ksql/KsqlGrammar.g4 | 3 + api/src/main/antlr/promql/PromQL.g4 | 287 ++++++++++++++ .../kafbat/ui/config/ClustersProperties.java | 36 +- .../auth/AbstractAuthSecurityConfig.java | 1 + .../ui/controller/GraphsController.java | 80 ++++ .../PrometheusExposeController.java | 32 ++ .../io/kafbat/ui/mapper/ClusterMapper.java | 45 ++- .../ui/mapper/DescribeLogDirsMapper.java | 24 +- .../io/kafbat/ui/model/InternalBroker.java | 6 +- .../ui/model/InternalBrokerDiskUsage.java | 11 - .../kafbat/ui/model/InternalClusterState.java | 31 +- .../kafbat/ui/model/InternalLogDirStats.java | 60 ++- .../ui/model/InternalPartitionsOffsets.java | 8 + .../io/kafbat/ui/model/InternalTopic.java | 32 +- .../java/io/kafbat/ui/model/KafkaCluster.java | 6 +- .../main/java/io/kafbat/ui/model/Metrics.java | 45 ++- .../io/kafbat/ui/model/MetricsConfig.java | 22 - .../ui/model/MetricsScrapeProperties.java | 46 +++ .../ui/model/PartitionDistributionStats.java | 11 +- .../java/io/kafbat/ui/model/Statistics.java | 16 +- .../io/kafbat/ui/service/BrokerService.java | 12 +- .../ui/service/KafkaClusterFactory.java | 84 ++-- .../io/kafbat/ui/service/MessagesService.java | 8 +- .../ui/service/ReactiveAdminClient.java | 6 +- .../io/kafbat/ui/service/StatisticsCache.java | 29 +- .../kafbat/ui/service/StatisticsService.java | 65 ++- .../io/kafbat/ui/service/TopicsService.java | 54 +-- .../ui/service/graphs/GraphDescription.java | 25 ++ .../ui/service/graphs/GraphDescriptions.java | 74 ++++ .../ui/service/graphs/GraphsService.java | 95 +++++ .../service/graphs/PromQueryLangGrammar.java | 35 ++ .../ui/service/graphs/PromQueryTemplate.java | 51 +++ .../integration/odd/TopicsExporter.java | 22 +- .../ui/service/metrics/MetricsCollector.java | 68 ---- .../ui/service/metrics/MetricsRetriever.java | 9 - .../PrometheusEndpointMetricsParser.java | 46 --- .../metrics/PrometheusMetricsRetriever.java | 70 ---- .../kafbat/ui/service/metrics/RawMetric.java | 64 ++- .../ui/service/metrics/SummarizedMetrics.java | 101 +++++ .../ui/service/metrics/WellKnownMetrics.java | 70 ---- .../metrics/prometheus/PrometheusExpose.java | 95 +++++ .../metrics/scrape/IoRatesMetricsScanner.java | 89 +++++ .../metrics/scrape/MetricsScrapping.java | 92 +++++ .../scrape/PerBrokerScrapedMetrics.java | 18 + .../metrics/scrape/ScrapedClusterState.java | 198 +++++++++ .../scrape/inferred/InferredMetrics.java | 23 ++ .../inferred/InferredMetricsScraper.java | 231 +++++++++++ .../{ => scrape/jmx}/JmxMetricsFormatter.java | 7 +- .../{ => scrape/jmx}/JmxMetricsRetriever.java | 63 ++- .../metrics/scrape/jmx/JmxMetricsScraper.java | 35 ++ .../{ => scrape/jmx}/JmxSslSocketFactory.java | 5 +- .../prometheus/PrometheusEndpointParser.java | 375 ++++++++++++++++++ .../PrometheusMetricsRetriever.java | 53 +++ .../scrape/prometheus/PrometheusScraper.java | 29 ++ .../ui/service/metrics/sink/KafkaSink.java | 78 ++++ .../ui/service/metrics/sink/MetricsSink.java | 55 +++ .../sink/PrometheusPushGatewaySink.java | 50 +++ .../sink/PrometheusRemoteWriteSink.java | 80 ++++ .../ui/util/KafkaServicesValidation.java | 14 + .../java/io/kafbat/ui/util/MetricsUtils.java | 107 +++++ .../io/kafbat/ui/util/ReactiveFailover.java | 6 - .../io/kafbat/ui/util/SslPropertiesUtil.java | 23 ++ .../ui/container/PrometheusContainer.java | 19 + .../model/PartitionDistributionStatsTest.java | 35 +- .../service/TopicsServicePaginationTest.java | 16 +- .../integration/odd/TopicsExporterTest.java | 99 +++-- .../ui/service/ksql/KsqlApiClientTest.java | 2 + .../metrics/JmxMetricsFormatterTest.java | 1 + .../ui/service/metrics/MetricsUtils.java | 57 +++ .../PrometheusEndpointMetricsParserTest.java | 31 -- .../PrometheusMetricsRetrieverTest.java | 97 ----- .../service/metrics/WellKnownMetricsTest.java | 93 ----- .../prometheus/PrometheusExposeTest.java | 67 ++++ .../scrape/IoRatesMetricsScannerTest.java | 75 ++++ .../scrape/PrometheusEndpointParserTest.java | 198 +++++++++ .../inferred/InferredMetricsScraperTest.java | 120 ++++++ contract/build.gradle | 30 +- .../proto/prometheus/gogoproto/gogo.proto | 133 +++++++ .../src/main/proto/prometheus/remote.proto | 88 ++++ .../src/main/proto/prometheus/types.proto | 187 +++++++++ .../main/resources/swagger/kafbat-ui-api.yaml | 193 +++++++++ .../swagger/prometheus-query-api.yaml | 363 +++++++++++++++++ documentation/compose/kafbat-ui.yaml | 13 + documentation/compose/scripts/prometheus.yaml | 14 + gradle/libs.versions.toml | 15 +- 86 files changed, 4551 insertions(+), 930 deletions(-) rename api/src/main/{antlr4 => antlr}/ksql/KsqlGrammar.g4 (99%) create mode 100644 api/src/main/antlr/promql/PromQL.g4 create mode 100644 api/src/main/java/io/kafbat/ui/controller/GraphsController.java create mode 100644 api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java delete mode 100644 api/src/main/java/io/kafbat/ui/model/InternalBrokerDiskUsage.java delete mode 100644 api/src/main/java/io/kafbat/ui/model/MetricsConfig.java create mode 100644 api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java create mode 100644 api/src/main/java/io/kafbat/ui/service/graphs/GraphDescription.java create mode 100644 api/src/main/java/io/kafbat/ui/service/graphs/GraphDescriptions.java create mode 100644 api/src/main/java/io/kafbat/ui/service/graphs/GraphsService.java create mode 100644 api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java create mode 100644 api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/MetricsCollector.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/MetricsRetriever.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParser.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetriever.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/WellKnownMetrics.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java rename api/src/main/java/io/kafbat/ui/service/metrics/{ => scrape/jmx}/JmxMetricsFormatter.java (93%) rename api/src/main/java/io/kafbat/ui/service/metrics/{ => scrape/jmx}/JmxMetricsRetriever.java (58%) create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java rename api/src/main/java/io/kafbat/ui/service/metrics/{ => scrape/jmx}/JmxSslSocketFactory.java (97%) create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java create mode 100644 api/src/main/java/io/kafbat/ui/util/MetricsUtils.java create mode 100644 api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java create mode 100644 api/src/test/java/io/kafbat/ui/container/PrometheusContainer.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java delete mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParserTest.java delete mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetrieverTest.java delete mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/WellKnownMetricsTest.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java create mode 100644 contract/src/main/proto/prometheus/gogoproto/gogo.proto create mode 100644 contract/src/main/proto/prometheus/remote.proto create mode 100644 contract/src/main/proto/prometheus/types.proto create mode 100644 contract/src/main/resources/swagger/prometheus-query-api.yaml create mode 100644 documentation/compose/scripts/prometheus.yaml diff --git a/api/build.gradle b/api/build.gradle index 563da52ab..8f55783c1 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -71,6 +71,11 @@ dependencies { because("CVE Fix: It is excluded above because of a vulnerability") } + implementation libs.prometheus.metrics.core + implementation libs.prometheus.metrics.textformats + implementation libs.prometheus.metrics.exporter.pushgateway + implementation libs.snappy + // Annotation processors implementation libs.lombok implementation libs.mapstruct @@ -97,26 +102,16 @@ dependencies { testImplementation libs.okhttp3 testImplementation libs.okhttp3.mockwebserver + testImplementation libs.prometheus.metrics.core } generateGrammarSource { maxHeapSize = "64m" - arguments += ["-package", "ksql"] -} - -sourceSets { - main { - antlr { - srcDirs = ["src/main/antlr4"] - } - java { - srcDirs += generateGrammarSource.outputDirectory - } - } } tasks.withType(Checkstyle).configureEach { exclude '**/ksql/**' + exclude '**/promql/**' } checkstyle { diff --git a/api/src/main/antlr4/ksql/KsqlGrammar.g4 b/api/src/main/antlr/ksql/KsqlGrammar.g4 similarity index 99% rename from api/src/main/antlr4/ksql/KsqlGrammar.g4 rename to api/src/main/antlr/ksql/KsqlGrammar.g4 index 2fcd623e3..e46da04ba 100644 --- a/api/src/main/antlr4/ksql/KsqlGrammar.g4 +++ b/api/src/main/antlr/ksql/KsqlGrammar.g4 @@ -1,5 +1,8 @@ grammar KsqlGrammar; +@header {package ksql;} + + tokens { DELIMITER } diff --git a/api/src/main/antlr/promql/PromQL.g4 b/api/src/main/antlr/promql/PromQL.g4 new file mode 100644 index 000000000..a405f9ce8 --- /dev/null +++ b/api/src/main/antlr/promql/PromQL.g4 @@ -0,0 +1,287 @@ +grammar PromQL; + +@header {package promql;} + +options { + caseInsensitive = true; +} + +expression: vectorOperation EOF; + +// Binary operations are ordered by precedence + +// Unary operations have the same precedence as multiplications + +vectorOperation + : vectorOperation powOp vectorOperation + | vectorOperation subqueryOp + | unaryOp vectorOperation + | vectorOperation multOp vectorOperation + | vectorOperation addOp vectorOperation + | vectorOperation compareOp vectorOperation + | vectorOperation andUnlessOp vectorOperation + | vectorOperation orOp vectorOperation + | vectorOperation vectorMatchOp vectorOperation + | vectorOperation AT vectorOperation + | vector + ; + +// Operators + +unaryOp: (ADD | SUB); +powOp: POW grouping?; +multOp: (MULT | DIV | MOD) grouping?; +addOp: (ADD | SUB) grouping?; +compareOp: (DEQ | NE | GT | LT | GE | LE) BOOL? grouping?; +andUnlessOp: (AND | UNLESS) grouping?; +orOp: OR grouping?; +vectorMatchOp: (ON | UNLESS) grouping?; +subqueryOp: SUBQUERY_RANGE offsetOp?; +offsetOp: OFFSET DURATION; + +vector + : function_ + | aggregation + | instantSelector + | matrixSelector + | offset + | literal + | parens + ; + +parens: LEFT_PAREN vectorOperation RIGHT_PAREN; + +// Selectors + +instantSelector + : METRIC_NAME (LEFT_BRACE labelMatcherList? RIGHT_BRACE)? + | LEFT_BRACE labelMatcherList RIGHT_BRACE + ; + +labelMatcher: labelName labelMatcherOperator STRING; +labelMatcherOperator: EQ | NE | RE | NRE; +labelMatcherList: labelMatcher (COMMA labelMatcher)* COMMA?; + +matrixSelector: instantSelector TIME_RANGE; + +offset + : instantSelector OFFSET DURATION + | matrixSelector OFFSET DURATION + ; + +// Functions + +function_: FUNCTION LEFT_PAREN (parameter (COMMA parameter)*)? RIGHT_PAREN; + +parameter: literal | vectorOperation; +parameterList: LEFT_PAREN (parameter (COMMA parameter)*)? RIGHT_PAREN; + +// Aggregations + +aggregation + : AGGREGATION_OPERATOR parameterList + | AGGREGATION_OPERATOR (by | without) parameterList + | AGGREGATION_OPERATOR parameterList ( by | without) + ; +by: BY labelNameList; +without: WITHOUT labelNameList; + +// Vector one-to-one/one-to-many joins + +grouping: (on_ | ignoring) (groupLeft | groupRight)?; +on_: ON labelNameList; +ignoring: IGNORING labelNameList; +groupLeft: GROUP_LEFT labelNameList?; +groupRight: GROUP_RIGHT labelNameList?; + +// Label names + +labelName: keyword | METRIC_NAME | LABEL_NAME; +labelNameList: LEFT_PAREN (labelName (COMMA labelName)*)? RIGHT_PAREN; + +keyword + : AND + | OR + | UNLESS + | BY + | WITHOUT + | ON + | IGNORING + | GROUP_LEFT + | GROUP_RIGHT + | OFFSET + | BOOL + | AGGREGATION_OPERATOR + | FUNCTION + ; + +literal: NUMBER | STRING; + +fragment NUMERAL: [0-9]+ ('.' [0-9]+)?; + +fragment SCIENTIFIC_NUMBER + : NUMERAL ('e' [-+]? NUMERAL)? + ; + +NUMBER + : NUMERAL + | SCIENTIFIC_NUMBER; + +STRING + : '\'' (~('\'' | '\\') | '\\' .)* '\'' + | '"' (~('"' | '\\') | '\\' .)* '"' + ; + +// Binary operators + +ADD: '+'; +SUB: '-'; +MULT: '*'; +DIV: '/'; +MOD: '%'; +POW: '^'; + +AND: 'and'; +OR: 'or'; +UNLESS: 'unless'; + +// Comparison operators + +EQ: '='; +DEQ: '=='; +NE: '!='; +GT: '>'; +LT: '<'; +GE: '>='; +LE: '<='; +RE: '=~'; +NRE: '!~'; + +// Aggregation modifiers + +BY: 'by'; +WITHOUT: 'without'; + +// Join modifiers + +ON: 'on'; +IGNORING: 'ignoring'; +GROUP_LEFT: 'group_left'; +GROUP_RIGHT: 'group_right'; + +OFFSET: 'offset'; + +BOOL: 'bool'; + +AGGREGATION_OPERATOR + : 'sum' + | 'min' + | 'max' + | 'avg' + | 'group' + | 'stddev' + | 'stdvar' + | 'count' + | 'count_values' + | 'bottomk' + | 'topk' + | 'quantile' + ; + +FUNCTION + : 'abs' + | 'absent' + | 'absent_over_time' + | 'ceil' + | 'changes' + | 'clamp_max' + | 'clamp_min' + | 'day_of_month' + | 'day_of_week' + | 'days_in_month' + | 'delta' + | 'deriv' + | 'exp' + | 'floor' + | 'histogram_quantile' + | 'holt_winters' + | 'hour' + | 'idelta' + | 'increase' + | 'irate' + | 'label_join' + | 'label_replace' + | 'ln' + | 'log2' + | 'log10' + | 'minute' + | 'month' + | 'predict_linear' + | 'rate' + | 'resets' + | 'round' + | 'scalar' + | 'sort' + | 'sort_desc' + | 'sqrt' + | 'time' + | 'timestamp' + | 'vector' + | 'year' + | 'avg_over_time' + | 'min_over_time' + | 'max_over_time' + | 'sum_over_time' + | 'count_over_time' + | 'quantile_over_time' + | 'stddev_over_time' + | 'stdvar_over_time' + | 'last_over_time' + | 'acos' + | 'acosh' + | 'asin' + | 'asinh' + | 'atan' + | 'atanh' + | 'cos' + | 'cosh' + | 'sin' + | 'sinh' + | 'tan' + | 'tanh' + | 'deg' + | 'pi' + | 'rad' + ; + +LEFT_BRACE: '{'; +RIGHT_BRACE: '}'; + +LEFT_PAREN: '('; +RIGHT_PAREN: ')'; + +LEFT_BRACKET: '['; +RIGHT_BRACKET: ']'; + +COMMA: ','; + +AT: '@'; + +SUBQUERY_RANGE + : LEFT_BRACKET DURATION ':' DURATION? RIGHT_BRACKET; + +TIME_RANGE + : LEFT_BRACKET DURATION RIGHT_BRACKET; + +// The proper order (longest to the shortest) must be validated after parsing +DURATION: ([0-9]+ ('ms' | [smhdwy]))+; + +METRIC_NAME: [a-z_:] [a-z0-9_:]*; +LABEL_NAME: [a-z_] [a-z0-9_]*; + + + +WS: [\r\t\n ]+ -> channel(HIDDEN); +SL_COMMENT + : '#' .*? '\n' -> channel(HIDDEN) + ; diff --git a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java index 23be86369..1a5e2c6e9 100644 --- a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java +++ b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java @@ -1,6 +1,7 @@ package io.kafbat.ui.config; -import io.kafbat.ui.model.MetricsConfig; +import static io.kafbat.ui.model.MetricsScrapeProperties.JMX_METRICS_TYPE; + import jakarta.annotation.PostConstruct; import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; @@ -59,7 +60,7 @@ public static class Cluster { String defaultKeySerde; String defaultValueSerde; - MetricsConfigData metrics; + MetricsConfig metrics; Map properties; Map consumerProperties; Map producerProperties; @@ -81,8 +82,8 @@ public static class PollingProperties { } @Data - @ToString(exclude = "password") - public static class MetricsConfigData { + @ToString(exclude = {"password", "keystorePassword"}) + public static class MetricsConfig { String type; Integer port; Boolean ssl; @@ -90,6 +91,31 @@ public static class MetricsConfigData { String password; String keystoreLocation; String keystorePassword; + + Boolean prometheusExpose; + MetricsStorage store; + } + + @Data + public static class MetricsStorage { + PrometheusStorage prometheus; + KafkaMetricsStorage kafka; + } + + @Data + public static class KafkaMetricsStorage { + String topic; + } + + @Data + @ToString(exclude = {"pushGatewayPassword"}) + public static class PrometheusStorage { + String url; + String pushGatewayUrl; + String pushGatewayUsername; + String pushGatewayPassword; + String pushGatewayJobName; + Boolean remoteWrite; } @Data @@ -195,7 +221,7 @@ public void validateAndSetDefaults() { private void setMetricsDefaults() { for (Cluster cluster : clusters) { if (cluster.getMetrics() != null && !StringUtils.hasText(cluster.getMetrics().getType())) { - cluster.getMetrics().setType(MetricsConfig.JMX_METRICS_TYPE); + cluster.getMetrics().setType(JMX_METRICS_TYPE); } } } diff --git a/api/src/main/java/io/kafbat/ui/config/auth/AbstractAuthSecurityConfig.java b/api/src/main/java/io/kafbat/ui/config/auth/AbstractAuthSecurityConfig.java index 265bac03f..9818db04f 100644 --- a/api/src/main/java/io/kafbat/ui/config/auth/AbstractAuthSecurityConfig.java +++ b/api/src/main/java/io/kafbat/ui/config/auth/AbstractAuthSecurityConfig.java @@ -24,6 +24,7 @@ protected AbstractAuthSecurityConfig() { "/static/**", "/resources/**", + "/metrics", /* ACTUATOR */ "/actuator/health/**", diff --git a/api/src/main/java/io/kafbat/ui/controller/GraphsController.java b/api/src/main/java/io/kafbat/ui/controller/GraphsController.java new file mode 100644 index 000000000..02a5421d4 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/controller/GraphsController.java @@ -0,0 +1,80 @@ +package io.kafbat.ui.controller; + +import io.kafbat.ui.api.GraphsApi; +import io.kafbat.ui.model.GraphDataRequestDTO; +import io.kafbat.ui.model.GraphDescriptionDTO; +import io.kafbat.ui.model.GraphDescriptionsDTO; +import io.kafbat.ui.model.GraphParameterDTO; +import io.kafbat.ui.model.PrometheusApiQueryResponseDTO; +import io.kafbat.ui.model.rbac.AccessContext; +import io.kafbat.ui.prometheus.model.QueryResponse; +import io.kafbat.ui.service.graphs.GraphDescription; +import io.kafbat.ui.service.graphs.GraphsService; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +@RestController +@RequiredArgsConstructor +public class GraphsController extends AbstractController implements GraphsApi { + + private static final PrometheusApiMapper MAPPER = Mappers.getMapper(PrometheusApiMapper.class); + + @Mapper + interface PrometheusApiMapper { + PrometheusApiQueryResponseDTO fromClientResponse(QueryResponse resp); + } + + private final GraphsService graphsService; + + @Override + public Mono> getGraphData(String clusterName, + Mono graphDataRequestDto, + ServerWebExchange exchange) { + var context = AccessContext.builder() + .cluster(clusterName) + .operationName("getGraphData") + .build(); + + return accessControlService.validateAccess(context) + .then( + graphDataRequestDto.flatMap(req -> + graphsService.getGraphData( + getCluster(clusterName), + req.getId(), + Optional.ofNullable(req.getFrom()).map(OffsetDateTime::toInstant).orElse(null), + Optional.ofNullable(req.getTo()).map(OffsetDateTime::toInstant).orElse(null), + req.getParameters() + ).map(MAPPER::fromClientResponse)) + .map(ResponseEntity::ok) + ).doOnEach(sig -> auditService.audit(context, sig)); + } + + @Override + public Mono> getGraphsList(String clusterName, + ServerWebExchange exchange) { + var context = AccessContext.builder() + .cluster(clusterName) + .operationName("getGraphsList") + .build(); + + var graphs = graphsService.getGraphs(getCluster(clusterName)); + return accessControlService.validateAccess(context).then( + Mono.just(ResponseEntity.ok(new GraphDescriptionsDTO().graphs(graphs.map(this::map).toList())))); + } + + private GraphDescriptionDTO map(GraphDescription graph) { + return new GraphDescriptionDTO() + .id(graph.id()) + .defaultPeriod(Optional.ofNullable(graph.defaultInterval()).map(Duration::toString).orElse(null)) + .type(graph.isRange() ? GraphDescriptionDTO.TypeEnum.RANGE : GraphDescriptionDTO.TypeEnum.INSTANT) + .parameters(graph.params().stream().map(GraphParameterDTO::new).toList()); + } +} diff --git a/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java b/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java new file mode 100644 index 000000000..97c5084b1 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java @@ -0,0 +1,32 @@ +package io.kafbat.ui.controller; + +import io.kafbat.ui.api.PrometheusExposeApi; +import io.kafbat.ui.model.KafkaCluster; +import io.kafbat.ui.service.StatisticsCache; +import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ServerWebExchange; +import reactor.core.publisher.Mono; + +@RestController +@RequiredArgsConstructor +public class PrometheusExposeController extends AbstractController implements PrometheusExposeApi { + + private final StatisticsCache statisticsCache; + + @Override + public Mono> getAllMetrics(ServerWebExchange exchange) { + return Mono.just( + PrometheusExpose.exposeAllMetrics( + clustersStorage.getKafkaClusters() + .stream() + .filter(KafkaCluster::isExposeMetricsViaPrometheusEndpoint) + .collect(Collectors.toMap(KafkaCluster::getName, c -> statisticsCache.get(c).getMetrics())) + ) + ); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java b/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java index 5dfd7c954..c31db42cf 100644 --- a/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java +++ b/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java @@ -1,9 +1,12 @@ package io.kafbat.ui.mapper; +import static io.kafbat.ui.util.MetricsUtils.readPointValue; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; + import io.kafbat.ui.config.ClustersProperties; import io.kafbat.ui.model.BrokerConfigDTO; import io.kafbat.ui.model.BrokerDTO; -import io.kafbat.ui.model.BrokerDiskUsageDTO; import io.kafbat.ui.model.BrokerMetricsDTO; import io.kafbat.ui.model.ClusterDTO; import io.kafbat.ui.model.ClusterFeature; @@ -14,7 +17,6 @@ import io.kafbat.ui.model.ConnectDTO; import io.kafbat.ui.model.InternalBroker; import io.kafbat.ui.model.InternalBrokerConfig; -import io.kafbat.ui.model.InternalBrokerDiskUsage; import io.kafbat.ui.model.InternalClusterState; import io.kafbat.ui.model.InternalPartition; import io.kafbat.ui.model.InternalReplica; @@ -31,9 +33,13 @@ import io.kafbat.ui.model.TopicDTO; import io.kafbat.ui.model.TopicDetailsDTO; import io.kafbat.ui.model.TopicProducerStateDTO; -import io.kafbat.ui.service.metrics.RawMetric; +import io.kafbat.ui.service.metrics.SummarizedMetrics; +import io.prometheus.metrics.model.snapshots.Label; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.math.BigDecimal; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.ProducerState; import org.apache.kafka.common.acl.AccessControlEntry; @@ -53,21 +59,27 @@ public interface ClusterMapper { ClusterStatsDTO toClusterStats(InternalClusterState clusterState); + @Deprecated default ClusterMetricsDTO toClusterMetrics(Metrics metrics) { return new ClusterMetricsDTO() - .items(metrics.getSummarizedMetrics().map(this::convert).toList()); + .items(convert(new SummarizedMetrics(metrics).asStream()).toList()); } - private MetricDTO convert(RawMetric rawMetric) { - return new MetricDTO() - .name(rawMetric.name()) - .labels(rawMetric.labels()) - .value(rawMetric.value()); + private Stream convert(Stream metrics) { + return metrics + .flatMap(m -> + m.getDataPoints().stream() + .map(p -> + new MetricDTO() + .name(m.getMetadata().getName()) + .labels(p.getLabels().stream().collect(toMap(Label::getName, Label::getValue))) + .value(BigDecimal.valueOf(readPointValue(p))) + ) + ); } - default BrokerMetricsDTO toBrokerMetrics(List metrics) { - return new BrokerMetricsDTO() - .metrics(metrics.stream().map(this::convert).toList()); + default BrokerMetricsDTO toBrokerMetrics(List metrics) { + return new BrokerMetricsDTO().metrics(convert(metrics.stream()).toList()); } @Mapping(target = "isSensitive", source = "sensitive") @@ -108,16 +120,9 @@ default ConfigSynonymDTO toConfigSynonym(ConfigEntry.ConfigSynonym config) { List toFeaturesEnum(List features); default List map(Map map) { - return map.values().stream().map(this::toPartition).toList(); + return map.values().stream().map(this::toPartition).collect(toList()); } - default BrokerDiskUsageDTO map(Integer id, InternalBrokerDiskUsage internalBrokerDiskUsage) { - final BrokerDiskUsageDTO brokerDiskUsage = new BrokerDiskUsageDTO(); - brokerDiskUsage.setBrokerId(id); - brokerDiskUsage.segmentCount((int) internalBrokerDiskUsage.getSegmentCount()); - brokerDiskUsage.segmentSize(internalBrokerDiskUsage.getSegmentSize()); - return brokerDiskUsage; - } default TopicProducerStateDTO map(int partition, ProducerState state) { return new TopicProducerStateDTO() diff --git a/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java b/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java index bccd3a66b..c4476405c 100644 --- a/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java +++ b/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java @@ -7,6 +7,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.kafka.clients.admin.LogDirDescription; +import org.apache.kafka.clients.admin.ReplicaInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.DescribeLogDirsResponse; @@ -16,7 +18,7 @@ public class DescribeLogDirsMapper { public List toBrokerLogDirsList( - Map> logDirsInfo) { + Map> logDirsInfo) { return logDirsInfo.entrySet().stream().map( mapEntry -> mapEntry.getValue().entrySet().stream() @@ -26,13 +28,13 @@ public List toBrokerLogDirsList( } private BrokersLogdirsDTO toBrokerLogDirs(Integer broker, String dirName, - DescribeLogDirsResponse.LogDirInfo logDirInfo) { + LogDirDescription logDirInfo) { BrokersLogdirsDTO result = new BrokersLogdirsDTO(); result.setName(dirName); - if (logDirInfo.error != null && logDirInfo.error != Errors.NONE) { - result.setError(logDirInfo.error.message()); + if (logDirInfo.error() != null) { + result.setError(logDirInfo.error().getMessage()); } - var topics = logDirInfo.replicaInfos.entrySet().stream() + var topics = logDirInfo.replicaInfos().entrySet().stream() .collect(Collectors.groupingBy(e -> e.getKey().topic())).entrySet().stream() .map(e -> toTopicLogDirs(broker, e.getKey(), e.getValue())) .toList(); @@ -40,9 +42,9 @@ private BrokersLogdirsDTO toBrokerLogDirs(Integer broker, String dirName, return result; } - private BrokerTopicLogdirsDTO toTopicLogDirs(Integer broker, String name, - List> partitions) { + private BrokerTopicLogdirsDTO toTopicLogDirs( + Integer broker, String name, + List> partitions) { BrokerTopicLogdirsDTO topic = new BrokerTopicLogdirsDTO(); topic.setName(name); topic.setPartitions( @@ -54,12 +56,12 @@ private BrokerTopicLogdirsDTO toTopicLogDirs(Integer broker, String name, } private BrokerTopicPartitionLogdirDTO topicPartitionLogDir(Integer broker, Integer partition, - DescribeLogDirsResponse.ReplicaInfo replicaInfo) { + ReplicaInfo replicaInfo) { BrokerTopicPartitionLogdirDTO logDir = new BrokerTopicPartitionLogdirDTO(); logDir.setBroker(broker); logDir.setPartition(partition); - logDir.setSize(replicaInfo.size); - logDir.setOffsetLag(replicaInfo.offsetLag); + logDir.setSize(replicaInfo.size()); + logDir.setOffsetLag(replicaInfo.offsetLag()); return logDir; } } diff --git a/api/src/main/java/io/kafbat/ui/model/InternalBroker.java b/api/src/main/java/io/kafbat/ui/model/InternalBroker.java index fd203c70f..e90cd5f7c 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalBroker.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalBroker.java @@ -21,12 +21,12 @@ public class InternalBroker { public InternalBroker(Node node, PartitionDistributionStats partitionDistribution, - Statistics statistics) { + Metrics metrics) { this.id = node.id(); this.host = node.host(); this.port = node.port(); - this.bytesInPerSec = statistics.getMetrics().getBrokerBytesInPerSec().get(node.id()); - this.bytesOutPerSec = statistics.getMetrics().getBrokerBytesOutPerSec().get(node.id()); + this.bytesInPerSec = metrics.getIoRates().brokerBytesInPerSec().get(node.id()); + this.bytesOutPerSec = metrics.getIoRates().brokerBytesOutPerSec().get(node.id()); this.partitionsLeader = partitionDistribution.getPartitionLeaders().get(node); this.partitions = partitionDistribution.getPartitionsCount().get(node); this.inSyncPartitions = partitionDistribution.getInSyncPartitions().get(node); diff --git a/api/src/main/java/io/kafbat/ui/model/InternalBrokerDiskUsage.java b/api/src/main/java/io/kafbat/ui/model/InternalBrokerDiskUsage.java deleted file mode 100644 index db48fab08..000000000 --- a/api/src/main/java/io/kafbat/ui/model/InternalBrokerDiskUsage.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.kafbat.ui.model; - -import lombok.Builder; -import lombok.Data; - -@Data -@Builder(toBuilder = true) -public class InternalBrokerDiskUsage { - private final long segmentCount; - private final long segmentSize; -} diff --git a/api/src/main/java/io/kafbat/ui/model/InternalClusterState.java b/api/src/main/java/io/kafbat/ui/model/InternalClusterState.java index 5f3c1f308..1c50fdad7 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalClusterState.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalClusterState.java @@ -36,39 +36,42 @@ public InternalClusterState(KafkaCluster cluster, Statistics statistics) { .message(e.getMessage()) .stackTrace(Throwables.getStackTraceAsString(e))) .orElse(null); - topicCount = statistics.getTopicDescriptions().size(); + topicCount = (int) statistics.topicDescriptions().count(); brokerCount = statistics.getClusterDescription().getNodes().size(); activeControllers = Optional.ofNullable(statistics.getClusterDescription().getController()) .map(Node::id) .orElse(null); version = statistics.getVersion(); - if (statistics.getLogDirInfo() != null) { - diskUsage = statistics.getLogDirInfo().getBrokerStats().entrySet().stream() - .map(e -> new BrokerDiskUsageDTO() - .brokerId(e.getKey()) - .segmentSize(e.getValue().getSegmentSize()) - .segmentCount(e.getValue().getSegmentsCount())) - .collect(Collectors.toList()); - } + diskUsage = statistics.getClusterState().getNodesStates().values().stream() + .filter(n -> n.segmentStats() != null) + .map(n -> new BrokerDiskUsageDTO() + .brokerId(n.id()) + .segmentSize(n.segmentStats().getSegmentSize()) + .segmentCount(n.segmentStats().getSegmentsCount())) + .collect(Collectors.toList()); features = statistics.getFeatures(); bytesInPerSec = statistics .getMetrics() - .getBrokerBytesInPerSec() - .values().stream() + .getIoRates() + .brokerBytesInPerSec() + .values() + .stream() .reduce(BigDecimal::add) .orElse(null); bytesOutPerSec = statistics .getMetrics() - .getBrokerBytesOutPerSec() - .values().stream() + .getIoRates() + .brokerBytesOutPerSec() + .values() + .stream() .reduce(BigDecimal::add) .orElse(null); - var partitionsStats = new PartitionsStats(statistics.getTopicDescriptions().values()); + var partitionsStats = new PartitionsStats(statistics.topicDescriptions().toList()); onlinePartitionCount = partitionsStats.getOnlinePartitionCount(); offlinePartitionCount = partitionsStats.getOfflinePartitionCount(); inSyncReplicasCount = partitionsStats.getInSyncReplicasCount(); diff --git a/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java b/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java index 64fc56c06..cb53ce527 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java @@ -1,15 +1,17 @@ package io.kafbat.ui.model; - import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summarizingLong; +import jakarta.annotation.Nullable; +import java.util.HashMap; import java.util.List; import java.util.LongSummaryStatistics; import java.util.Map; +import lombok.RequiredArgsConstructor; import lombok.Value; +import org.apache.kafka.clients.admin.LogDirDescription; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.requests.DescribeLogDirsResponse; import reactor.util.function.Tuple2; import reactor.util.function.Tuple3; import reactor.util.function.Tuples; @@ -18,30 +20,37 @@ public class InternalLogDirStats { @Value + @RequiredArgsConstructor public static class SegmentStats { - long segmentSize; - int segmentsCount; + Long segmentSize; + Integer segmentsCount; - public SegmentStats(LongSummaryStatistics s) { - segmentSize = s.getSum(); - segmentsCount = (int) s.getCount(); + private SegmentStats(LongSummaryStatistics s) { + this(s.getSum(), (int) s.getCount()); } } + public record LogDirSpaceStats(@Nullable Long totalBytes, + @Nullable Long usableBytes, + Map totalPerDir, + Map usablePerDir) { + } + Map partitionsStats; Map topicStats; Map brokerStats; + Map brokerDirsStats; public static InternalLogDirStats empty() { return new InternalLogDirStats(Map.of()); } - public InternalLogDirStats(Map> log) { + public InternalLogDirStats(Map> logsInfo) { final List> topicPartitions = - log.entrySet().stream().flatMap(b -> + logsInfo.entrySet().stream().flatMap(b -> b.getValue().entrySet().stream().flatMap(topicMap -> - topicMap.getValue().replicaInfos.entrySet().stream() - .map(e -> Tuples.of(b.getKey(), e.getKey(), e.getValue().size)) + topicMap.getValue().replicaInfos().entrySet().stream() + .map(e -> Tuples.of(b.getKey(), e.getKey(), e.getValue().size())) ) ).toList(); @@ -63,5 +72,34 @@ public InternalLogDirStats(Map calculateSpaceStats( + Map> logsInfo) { + + var stats = new HashMap(); + logsInfo.forEach((brokerId, logDirStats) -> { + Map totalBytes = new HashMap<>(); + Map usableBytes = new HashMap<>(); + logDirStats.forEach((logDir, descr) -> { + if (descr.error() == null) { + return; + } + descr.totalBytes().ifPresent(b -> totalBytes.merge(logDir, b, Long::sum)); + descr.usableBytes().ifPresent(b -> usableBytes.merge(logDir, b, Long::sum)); + }); + stats.put( + brokerId, + new LogDirSpaceStats( + totalBytes.isEmpty() ? null : totalBytes.values().stream().mapToLong(i -> i).sum(), + usableBytes.isEmpty() ? null : usableBytes.values().stream().mapToLong(i -> i).sum(), + totalBytes, + usableBytes + ) + ); + }); + return stats; } } diff --git a/api/src/main/java/io/kafbat/ui/model/InternalPartitionsOffsets.java b/api/src/main/java/io/kafbat/ui/model/InternalPartitionsOffsets.java index b4febb56d..57a9c3b82 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalPartitionsOffsets.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalPartitionsOffsets.java @@ -4,6 +4,7 @@ import com.google.common.collect.Table; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import lombok.Value; import org.apache.kafka.common.TopicPartition; @@ -29,4 +30,11 @@ public Optional get(String topic, int partition) { return Optional.ofNullable(offsets.get(topic, partition)); } + public Map topicOffsets(String topic, boolean earliest) { + return offsets.row(topic) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> earliest ? e.getValue().earliest : e.getValue().getLatest())); + } + } diff --git a/api/src/main/java/io/kafbat/ui/model/InternalTopic.java b/api/src/main/java/io/kafbat/ui/model/InternalTopic.java index 6aa3a0a1a..1e8c31b5b 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalTopic.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalTopic.java @@ -3,6 +3,7 @@ import java.math.BigDecimal; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.Builder; @@ -41,7 +42,8 @@ public static InternalTopic from(TopicDescription topicDescription, List configs, InternalPartitionsOffsets partitionsOffsets, Metrics metrics, - InternalLogDirStats logDirInfo, + @Nullable InternalLogDirStats.SegmentStats segmentStats, + @Nullable Map partitionsSegmentStats, @Nullable String internalTopicPrefix) { var topic = InternalTopic.builder(); @@ -78,13 +80,13 @@ public static InternalTopic from(TopicDescription topicDescription, partitionDto.offsetMax(offsets.getLatest()); }); - var segmentStats = - logDirInfo.getPartitionsStats().get( - new TopicPartition(topicDescription.name(), partition.partition())); - if (segmentStats != null) { - partitionDto.segmentCount(segmentStats.getSegmentsCount()); - partitionDto.segmentSize(segmentStats.getSegmentSize()); - } + Optional.ofNullable(partitionsSegmentStats) + .flatMap(s -> Optional.ofNullable(s.get(partition.partition()))) + .ifPresent(stats -> { + partitionDto.segmentCount(stats.getSegmentsCount()); + partitionDto.segmentSize(stats.getSegmentSize()); + }); + return partitionDto.build(); }) @@ -105,14 +107,14 @@ public static InternalTopic from(TopicDescription topicDescription, : topicDescription.partitions().get(0).replicas().size() ); - var segmentStats = logDirInfo.getTopicStats().get(topicDescription.name()); - if (segmentStats != null) { - topic.segmentCount(segmentStats.getSegmentsCount()); - topic.segmentSize(segmentStats.getSegmentSize()); - } + Optional.ofNullable(segmentStats) + .ifPresent(stats -> { + topic.segmentCount(stats.getSegmentsCount()); + topic.segmentSize(stats.getSegmentSize()); + }); - topic.bytesInPerSec(metrics.getTopicBytesInPerSec().get(topicDescription.name())); - topic.bytesOutPerSec(metrics.getTopicBytesOutPerSec().get(topicDescription.name())); + topic.bytesInPerSec(metrics.getIoRates().topicBytesInPerSec().get(topicDescription.name())); + topic.bytesOutPerSec(metrics.getIoRates().topicBytesOutPerSec().get(topicDescription.name())); topic.topicConfigs( configs.stream().map(InternalTopicConfig::from).collect(Collectors.toList())); diff --git a/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java b/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java index 6e2a00988..5281d07e8 100644 --- a/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java +++ b/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java @@ -3,8 +3,10 @@ import io.kafbat.ui.config.ClustersProperties; import io.kafbat.ui.connect.api.KafkaConnectClientApi; import io.kafbat.ui.emitter.PollingSettings; +import io.kafbat.ui.prometheus.api.PrometheusClientApi; import io.kafbat.ui.service.ksql.KsqlApiClient; import io.kafbat.ui.service.masking.DataMasking; +import io.kafbat.ui.service.metrics.scrape.MetricsScrapping; import io.kafbat.ui.sr.api.KafkaSrClientApi; import io.kafbat.ui.util.ReactiveFailover; import java.util.Map; @@ -27,10 +29,12 @@ public class KafkaCluster { private final Properties consumerProperties; private final Properties producerProperties; private final boolean readOnly; - private final MetricsConfig metricsConfig; + private final boolean exposeMetricsViaPrometheusEndpoint; private final DataMasking masking; private final PollingSettings pollingSettings; private final ReactiveFailover schemaRegistryClient; private final Map> connectsClients; private final ReactiveFailover ksqlClient; + private final MetricsScrapping metricsScrapping; + private final ReactiveFailover prometheusStorageClient; } diff --git a/api/src/main/java/io/kafbat/ui/model/Metrics.java b/api/src/main/java/io/kafbat/ui/model/Metrics.java index bb6d2ff0c..b21e523be 100644 --- a/api/src/main/java/io/kafbat/ui/model/Metrics.java +++ b/api/src/main/java/io/kafbat/ui/model/Metrics.java @@ -1,42 +1,45 @@ package io.kafbat.ui.model; -import static java.util.stream.Collectors.toMap; - -import io.kafbat.ui.service.metrics.RawMetric; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.math.BigDecimal; -import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.stream.Stream; import lombok.Builder; import lombok.Value; +import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; + + @Builder @Value public class Metrics { - Map brokerBytesInPerSec; - Map brokerBytesOutPerSec; - Map topicBytesInPerSec; - Map topicBytesOutPerSec; - Map> perBrokerMetrics; + IoRates ioRates; + InferredMetrics inferredMetrics; + Map> perBrokerScrapedMetrics; public static Metrics empty() { return Metrics.builder() - .brokerBytesInPerSec(Map.of()) - .brokerBytesOutPerSec(Map.of()) - .topicBytesInPerSec(Map.of()) - .topicBytesOutPerSec(Map.of()) - .perBrokerMetrics(Map.of()) + .ioRates(IoRates.empty()) + .perBrokerScrapedMetrics(Map.of()) + .inferredMetrics(InferredMetrics.empty()) .build(); } - public Stream getSummarizedMetrics() { - return perBrokerMetrics.values().stream() - .flatMap(Collection::stream) - .collect(toMap(RawMetric::identityKey, m -> m, (m1, m2) -> m1.copyWithValue(m1.value().add(m2.value())))) - .values() - .stream(); + @Builder + public record IoRates(Map brokerBytesInPerSec, + Map brokerBytesOutPerSec, + Map topicBytesInPerSec, + Map topicBytesOutPerSec) { + + static IoRates empty() { + return IoRates.builder() + .brokerBytesOutPerSec(Map.of()) + .brokerBytesInPerSec(Map.of()) + .topicBytesOutPerSec(Map.of()) + .topicBytesInPerSec(Map.of()) + .build(); + } } } diff --git a/api/src/main/java/io/kafbat/ui/model/MetricsConfig.java b/api/src/main/java/io/kafbat/ui/model/MetricsConfig.java deleted file mode 100644 index 46d036f92..000000000 --- a/api/src/main/java/io/kafbat/ui/model/MetricsConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.kafbat.ui.model; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; - -@Data -@Builder(toBuilder = true) -@AllArgsConstructor(access = AccessLevel.PRIVATE) -public class MetricsConfig { - public static final String JMX_METRICS_TYPE = "JMX"; - public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; - - private final String type; - private final Integer port; - private final boolean ssl; - private final String username; - private final String password; - private final String keystoreLocation; - private final String keystorePassword; -} diff --git a/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java b/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java new file mode 100644 index 000000000..41e0fc6fd --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java @@ -0,0 +1,46 @@ +package io.kafbat.ui.model; + +import static io.kafbat.ui.config.ClustersProperties.KeystoreConfig; +import static io.kafbat.ui.config.ClustersProperties.TruststoreConfig; + +import io.kafbat.ui.config.ClustersProperties; +import jakarta.annotation.Nullable; +import java.util.Objects; +import java.util.Optional; +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class MetricsScrapeProperties { + public static final String JMX_METRICS_TYPE = "JMX"; + public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; + + Integer port; + boolean ssl; + String username; + String password; + + @Nullable + KeystoreConfig keystoreConfig; + + @Nullable + TruststoreConfig truststoreConfig; + + public static MetricsScrapeProperties create(ClustersProperties.Cluster cluster) { + var metrics = Objects.requireNonNull(cluster.getMetrics()); + return MetricsScrapeProperties.builder() + .port(metrics.getPort()) + .ssl(Optional.ofNullable(metrics.getSsl()).orElse(false)) + .username(metrics.getUsername()) + .password(metrics.getPassword()) + .truststoreConfig(cluster.getSsl()) + .keystoreConfig( + metrics.getKeystoreLocation() != null + ? new KeystoreConfig(metrics.getKeystoreLocation(), metrics.getKeystorePassword()) + : null + ) + .build(); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/model/PartitionDistributionStats.java b/api/src/main/java/io/kafbat/ui/model/PartitionDistributionStats.java index 0f44a35e3..9eb77ea80 100644 --- a/api/src/main/java/io/kafbat/ui/model/PartitionDistributionStats.java +++ b/api/src/main/java/io/kafbat/ui/model/PartitionDistributionStats.java @@ -3,6 +3,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; import lombok.AccessLevel; @@ -29,15 +30,19 @@ public class PartitionDistributionStats { private final boolean skewCanBeCalculated; public static PartitionDistributionStats create(Statistics stats) { - return create(stats, MIN_PARTITIONS_FOR_SKEW_CALCULATION); + return create( + stats.topicDescriptions().toList(), + MIN_PARTITIONS_FOR_SKEW_CALCULATION + ); } - static PartitionDistributionStats create(Statistics stats, int minPartitionsForSkewCalculation) { + static PartitionDistributionStats create(List topicDescriptions, + int minPartitionsForSkewCalculation) { var partitionLeaders = new HashMap(); var partitionsReplicated = new HashMap(); var isr = new HashMap(); int partitionsCnt = 0; - for (TopicDescription td : stats.getTopicDescriptions().values()) { + for (TopicDescription td : topicDescriptions) { for (TopicPartitionInfo tp : td.partitions()) { partitionsCnt++; tp.replicas().forEach(r -> incr(partitionsReplicated, r)); diff --git a/api/src/main/java/io/kafbat/ui/model/Statistics.java b/api/src/main/java/io/kafbat/ui/model/Statistics.java index 43e6a26e0..5b6cd8f9f 100644 --- a/api/src/main/java/io/kafbat/ui/model/Statistics.java +++ b/api/src/main/java/io/kafbat/ui/model/Statistics.java @@ -1,12 +1,12 @@ package io.kafbat.ui.model; import io.kafbat.ui.service.ReactiveAdminClient; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; import java.util.List; -import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import lombok.Builder; import lombok.Value; -import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; @Value @@ -18,9 +18,7 @@ public class Statistics { List features; ReactiveAdminClient.ClusterDescription clusterDescription; Metrics metrics; - InternalLogDirStats logDirInfo; - Map topicDescriptions; - Map> topicConfigs; + ScrapedClusterState clusterState; public static Statistics empty() { return builder() @@ -30,9 +28,11 @@ public static Statistics empty() { .clusterDescription( new ReactiveAdminClient.ClusterDescription(null, null, List.of(), Set.of())) .metrics(Metrics.empty()) - .logDirInfo(InternalLogDirStats.empty()) - .topicDescriptions(Map.of()) - .topicConfigs(Map.of()) + .clusterState(ScrapedClusterState.empty()) .build(); } + + public Stream topicDescriptions() { + return clusterState.getTopicStates().values().stream().map(ScrapedClusterState.TopicState::description); + } } diff --git a/api/src/main/java/io/kafbat/ui/service/BrokerService.java b/api/src/main/java/io/kafbat/ui/service/BrokerService.java index 198685b93..c1ae5fb1d 100644 --- a/api/src/main/java/io/kafbat/ui/service/BrokerService.java +++ b/api/src/main/java/io/kafbat/ui/service/BrokerService.java @@ -11,7 +11,7 @@ import io.kafbat.ui.model.InternalBrokerConfig; import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.model.PartitionDistributionStats; -import io.kafbat.ui.service.metrics.RawMetric; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -20,13 +20,13 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.LogDirDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.LogDirNotFoundException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; -import org.apache.kafka.common.requests.DescribeLogDirsResponse; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -71,7 +71,7 @@ public Flux getBrokers(KafkaCluster cluster) { .get(cluster) .flatMap(ReactiveAdminClient::describeCluster) .map(description -> description.getNodes().stream() - .map(node -> new InternalBroker(node, partitionsDistribution, stats)) + .map(node -> new InternalBroker(node, partitionsDistribution, stats.getMetrics())) .collect(Collectors.toList())) .flatMapMany(Flux::fromIterable); } @@ -109,7 +109,7 @@ public Mono updateBrokerConfigByName(KafkaCluster cluster, .doOnError(e -> log.error("Unexpected error", e)); } - private Mono>> getClusterLogDirs( + private Mono>> getClusterLogDirs( KafkaCluster cluster, List reqBrokers) { return adminClientService.get(cluster) .flatMap(admin -> { @@ -138,8 +138,8 @@ public Flux getBrokerConfig(KafkaCluster cluster, Integer return getBrokersConfig(cluster, brokerId); } - public Mono> getBrokerMetrics(KafkaCluster cluster, Integer brokerId) { - return Mono.justOrEmpty(statisticsCache.get(cluster).getMetrics().getPerBrokerMetrics().get(brokerId)); + public Mono> getBrokerMetrics(KafkaCluster cluster, Integer brokerId) { + return Mono.justOrEmpty(statisticsCache.get(cluster).getMetrics().getPerBrokerScrapedMetrics().get(brokerId)); } } diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index f8c528f90..489f316f0 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -1,5 +1,11 @@ package io.kafbat.ui.service; +import static io.kafbat.ui.util.KafkaServicesValidation.validateClusterConnection; +import static io.kafbat.ui.util.KafkaServicesValidation.validateKsql; +import static io.kafbat.ui.util.KafkaServicesValidation.validatePrometheusStore; +import static io.kafbat.ui.util.KafkaServicesValidation.validateSchemaRegistry; +import static io.kafbat.ui.util.KafkaServicesValidation.validateTruststore; + import io.kafbat.ui.client.RetryingKafkaConnectClient; import io.kafbat.ui.config.ClustersProperties; import io.kafbat.ui.config.WebclientProperties; @@ -8,9 +14,10 @@ import io.kafbat.ui.model.ApplicationPropertyValidationDTO; import io.kafbat.ui.model.ClusterConfigValidationDTO; import io.kafbat.ui.model.KafkaCluster; -import io.kafbat.ui.model.MetricsConfig; +import io.kafbat.ui.prometheus.api.PrometheusClientApi; import io.kafbat.ui.service.ksql.KsqlApiClient; import io.kafbat.ui.service.masking.DataMasking; +import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; import io.kafbat.ui.sr.ApiClient; import io.kafbat.ui.sr.api.KafkaSrClientApi; import io.kafbat.ui.util.KafkaServicesValidation; @@ -23,9 +30,9 @@ import java.util.Optional; import java.util.Properties; import java.util.stream.Stream; -import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; import org.springframework.util.unit.DataSize; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Flux; @@ -42,14 +49,16 @@ public class KafkaClusterFactory { private final DataSize webClientMaxBuffSize; private final Duration responseTimeout; + private final JmxMetricsRetriever jmxMetricsRetriever; - public KafkaClusterFactory(WebclientProperties webclientProperties) { + public KafkaClusterFactory(WebclientProperties webclientProperties, JmxMetricsRetriever jmxMetricsRetriever) { this.webClientMaxBuffSize = Optional.ofNullable(webclientProperties.getMaxInMemoryBufferSize()) .map(DataSize::parse) .orElse(DEFAULT_WEBCLIENT_BUFFER); this.responseTimeout = Optional.ofNullable(webclientProperties.getResponseTimeoutMs()) .map(Duration::ofMillis) .orElse(DEFAULT_RESPONSE_TIMEOUT); + this.jmxMetricsRetriever = jmxMetricsRetriever; } public KafkaCluster create(ClustersProperties properties, @@ -62,6 +71,7 @@ public KafkaCluster create(ClustersProperties properties, builder.consumerProperties(convertProperties(clusterProperties.getConsumerProperties())); builder.producerProperties(convertProperties(clusterProperties.getProducerProperties())); builder.readOnly(clusterProperties.isReadOnly()); + builder.exposeMetricsViaPrometheusEndpoint(exposeMetricsViaPrometheusEndpoint(clusterProperties)); builder.masking(DataMasking.create(clusterProperties.getMasking())); builder.pollingSettings(PollingSettings.create(clusterProperties, properties)); @@ -74,8 +84,8 @@ public KafkaCluster create(ClustersProperties properties, if (ksqlConfigured(clusterProperties)) { builder.ksqlClient(ksqlClient(clusterProperties)); } - if (metricsConfigured(clusterProperties)) { - builder.metricsConfig(metricsConfigDataToMetricsConfig(clusterProperties.getMetrics())); + if (prometheusStorageConfigured(clusterProperties)) { + builder.prometheusStorageClient(prometheusStorageClient(clusterProperties)); } builder.originalProperties(clusterProperties); return builder.build(); @@ -83,7 +93,7 @@ public KafkaCluster create(ClustersProperties properties, public Mono validate(ClustersProperties.Cluster clusterProperties) { if (clusterProperties.getSsl() != null) { - Optional errMsg = KafkaServicesValidation.validateTruststore(clusterProperties.getSsl()); + Optional errMsg = validateTruststore(clusterProperties.getSsl()); if (errMsg.isPresent()) { return Mono.just(new ClusterConfigValidationDTO() .kafka(new ApplicationPropertyValidationDTO() @@ -93,40 +103,48 @@ public Mono validate(ClustersProperties.Cluster clus } return Mono.zip( - KafkaServicesValidation.validateClusterConnection( + validateClusterConnection( clusterProperties.getBootstrapServers(), convertProperties(clusterProperties.getProperties()), clusterProperties.getSsl() ), schemaRegistryConfigured(clusterProperties) - ? KafkaServicesValidation.validateSchemaRegistry( - () -> schemaRegistryClient(clusterProperties)).map(Optional::of) + ? validateSchemaRegistry(() -> schemaRegistryClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), ksqlConfigured(clusterProperties) - ? KafkaServicesValidation.validateKsql(() -> ksqlClient(clusterProperties)).map(Optional::of) + ? validateKsql(() -> ksqlClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), - connectClientsConfigured(clusterProperties) - ? + connectClientsConfigured(clusterProperties) ? Flux.fromIterable(clusterProperties.getKafkaConnect()) .flatMap(c -> KafkaServicesValidation.validateConnect(() -> connectClient(clusterProperties, c)) .map(r -> Tuples.of(c.getName(), r))) .collectMap(Tuple2::getT1, Tuple2::getT2) .map(Optional::of) - : - Mono.>>just(Optional.empty()) + : Mono.>>just(Optional.empty()), + + prometheusStorageConfigured(clusterProperties) + ? validatePrometheusStore(() -> prometheusStorageClient(clusterProperties)).map(Optional::of) + : Mono.>just(Optional.empty()) ).map(tuple -> { var validation = new ClusterConfigValidationDTO(); validation.kafka(tuple.getT1()); tuple.getT2().ifPresent(validation::schemaRegistry); tuple.getT3().ifPresent(validation::ksqldb); tuple.getT4().ifPresent(validation::kafkaConnects); + tuple.getT5().ifPresent(validation::prometheusStorage); return validation; }); } + private boolean exposeMetricsViaPrometheusEndpoint(ClustersProperties.Cluster clusterProperties) { + return Optional.ofNullable(clusterProperties.getMetrics()) + .map(m -> m.getPrometheusExpose() == null || m.getPrometheusExpose()) + .orElse(true); + } + private Properties convertProperties(Map propertiesMap) { Properties properties = new Properties(); if (propertiesMap != null) { @@ -135,6 +153,28 @@ private Properties convertProperties(Map propertiesMap) { return properties; } + private ReactiveFailover prometheusStorageClient(ClustersProperties.Cluster cluster) { + WebClient webClient = new WebClientConfigurator() + .configureSsl(cluster.getSsl(), null) + .configureBufferSize(webClientMaxBuffSize) + .build(); + return ReactiveFailover.create( + parseUrlList(cluster.getMetrics().getStore().getPrometheus().getUrl()), + url -> new PrometheusClientApi(new io.kafbat.ui.prometheus.ApiClient(webClient).setBasePath(url)), + ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, + "No live schemaRegistry instances available", + ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS + ); + } + + private boolean prometheusStorageConfigured(ClustersProperties.Cluster cluster) { + return Optional.ofNullable(cluster.getMetrics()) + .flatMap(m -> Optional.ofNullable(m.getStore())) + .flatMap(s -> Optional.ofNullable(s.getPrometheus())) + .map(p -> StringUtils.hasText(p.getUrl())) + .orElse(false); + } + private boolean connectClientsConfigured(ClustersProperties.Cluster clusterProperties) { return clusterProperties.getKafkaConnect() != null; } @@ -211,20 +251,4 @@ private boolean metricsConfigured(ClustersProperties.Cluster clusterProperties) return clusterProperties.getMetrics() != null; } - @Nullable - private MetricsConfig metricsConfigDataToMetricsConfig(ClustersProperties.MetricsConfigData metricsConfigData) { - if (metricsConfigData == null) { - return null; - } - MetricsConfig.MetricsConfigBuilder builder = MetricsConfig.builder(); - builder.type(metricsConfigData.getType()); - builder.port(metricsConfigData.getPort()); - builder.ssl(Optional.ofNullable(metricsConfigData.getSsl()).orElse(false)); - builder.username(metricsConfigData.getUsername()); - builder.password(metricsConfigData.getPassword()); - builder.keystoreLocation(metricsConfigData.getKeystoreLocation()); - builder.keystorePassword(metricsConfigData.getKeystorePassword()); - return builder.build(); - } - } diff --git a/api/src/main/java/io/kafbat/ui/service/MessagesService.java b/api/src/main/java/io/kafbat/ui/service/MessagesService.java index b33be8b76..8a8dfc587 100644 --- a/api/src/main/java/io/kafbat/ui/service/MessagesService.java +++ b/api/src/main/java/io/kafbat/ui/service/MessagesService.java @@ -24,6 +24,7 @@ import io.kafbat.ui.serdes.ConsumerRecordDeserializer; import io.kafbat.ui.serdes.ProducerRecordCreator; import io.kafbat.ui.util.KafkaClientSslPropertiesUtil; +import io.kafbat.ui.util.SslPropertiesUtil; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -198,8 +199,13 @@ private Mono sendMessageImpl(KafkaCluster cluster, public static KafkaProducer createProducer(KafkaCluster cluster, Map additionalProps) { + return createProducer(cluster.getOriginalProperties(), additionalProps); + } + + public static KafkaProducer createProducer(ClustersProperties.Cluster cluster, + Map additionalProps) { Properties properties = new Properties(); - KafkaClientSslPropertiesUtil.addKafkaSslProperties(cluster.getOriginalProperties().getSsl(), properties); + SslPropertiesUtil.addKafkaSslProperties(cluster.getSsl(), properties); properties.putAll(cluster.getProperties()); properties.putAll(cluster.getProducerProperties()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); diff --git a/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java b/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java index 6aea290c3..82f8658b2 100644 --- a/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java +++ b/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java @@ -52,6 +52,7 @@ import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsSpec; import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.ListTopicsOptions; +import org.apache.kafka.clients.admin.LogDirDescription; import org.apache.kafka.clients.admin.NewPartitionReassignment; import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; @@ -389,9 +390,8 @@ static Mono> toMonoWithExceptionFilter(Map> v ); } - public Mono>> describeLogDirs( - Collection brokerIds) { - return toMono(client.describeLogDirs(brokerIds).all()) + public Mono>> describeLogDirs(Collection brokerIds) { + return toMono(client.describeLogDirs(brokerIds).allDescriptions()) .onErrorResume(UnsupportedVersionException.class, th -> Mono.just(Map.of())) .onErrorResume(ClusterAuthorizationException.class, th -> Mono.just(Map.of())) .onErrorResume(th -> true, th -> { diff --git a/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java b/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java index 134b723f4..7b37e8400 100644 --- a/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java +++ b/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java @@ -1,5 +1,6 @@ package io.kafbat.ui.service; +import io.kafbat.ui.model.InternalPartitionsOffsets; import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.model.ServerStatusDTO; import io.kafbat.ui.model.Statistics; @@ -8,6 +9,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.springframework.stereotype.Component; @@ -28,38 +30,29 @@ public synchronized void replace(KafkaCluster c, Statistics stats) { public synchronized void update(KafkaCluster c, Map descriptions, - Map> configs) { - var metrics = get(c); - var updatedDescriptions = new HashMap<>(metrics.getTopicDescriptions()); - updatedDescriptions.putAll(descriptions); - var updatedConfigs = new HashMap<>(metrics.getTopicConfigs()); - updatedConfigs.putAll(configs); + Map> configs, + InternalPartitionsOffsets partitionsOffsets) { + var stats = get(c); replace( c, - metrics.toBuilder() - .topicDescriptions(updatedDescriptions) - .topicConfigs(updatedConfigs) + stats.toBuilder() + .clusterState(stats.getClusterState().updateTopics(descriptions, configs, partitionsOffsets)) .build() ); } public synchronized void onTopicDelete(KafkaCluster c, String topic) { - var metrics = get(c); - var updatedDescriptions = new HashMap<>(metrics.getTopicDescriptions()); - updatedDescriptions.remove(topic); - var updatedConfigs = new HashMap<>(metrics.getTopicConfigs()); - updatedConfigs.remove(topic); + var stats = get(c); replace( c, - metrics.toBuilder() - .topicDescriptions(updatedDescriptions) - .topicConfigs(updatedConfigs) + stats.toBuilder() + .clusterState(stats.getClusterState().topicDeleted(topic)) .build() ); } public Statistics get(KafkaCluster c) { - return Objects.requireNonNull(cache.get(c.getName()), "Unknown cluster metrics requested"); + return Objects.requireNonNull(cache.get(c.getName()), "Statistics for unknown cluster requested"); } } diff --git a/api/src/main/java/io/kafbat/ui/service/StatisticsService.java b/api/src/main/java/io/kafbat/ui/service/StatisticsService.java index 8adadf1ed..5946792f0 100644 --- a/api/src/main/java/io/kafbat/ui/service/StatisticsService.java +++ b/api/src/main/java/io/kafbat/ui/service/StatisticsService.java @@ -2,21 +2,13 @@ import static io.kafbat.ui.service.ReactiveAdminClient.ClusterDescription; -import io.kafbat.ui.model.ClusterFeature; -import io.kafbat.ui.model.InternalLogDirStats; import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.model.Metrics; import io.kafbat.ui.model.ServerStatusDTO; import io.kafbat.ui.model.Statistics; -import io.kafbat.ui.service.metrics.MetricsCollector; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.ConfigEntry; -import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.Node; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @@ -25,7 +17,6 @@ @Slf4j public class StatisticsService { - private final MetricsCollector metricsCollector; private final AdminClientService adminClientService; private final FeatureService featureService; private final StatisticsCache cache; @@ -36,44 +27,38 @@ public Mono updateCache(KafkaCluster c) { private Mono getStatistics(KafkaCluster cluster) { return adminClientService.get(cluster).flatMap(ac -> - ac.describeCluster().flatMap(description -> - ac.updateInternalStats(description.getController()).then( - Mono.zip( - List.of( - metricsCollector.getBrokerMetrics(cluster, description.getNodes()), - getLogDirInfo(description, ac), - featureService.getAvailableFeatures(ac, cluster, description), - loadTopicConfigs(cluster), - describeTopics(cluster)), - results -> - Statistics.builder() - .status(ServerStatusDTO.ONLINE) - .clusterDescription(description) - .version(ac.getVersion()) - .metrics((Metrics) results[0]) - .logDirInfo((InternalLogDirStats) results[1]) - .features((List) results[2]) - .topicConfigs((Map>) results[3]) - .topicDescriptions((Map) results[4]) - .build() - )))) + ac.describeCluster() + .flatMap(description -> + ac.updateInternalStats(description.getController()) + .then( + Mono.zip( + featureService.getAvailableFeatures(ac, cluster, description), + loadClusterState(description, ac) + ).flatMap(featuresAndState -> + scrapeMetrics(cluster, featuresAndState.getT2(), description) + .map(metrics -> + Statistics.builder() + .status(ServerStatusDTO.ONLINE) + .clusterDescription(description) + .version(ac.getVersion()) + .metrics(metrics) + .features(featuresAndState.getT1()) + .clusterState(featuresAndState.getT2()) + .build()))))) .doOnError(e -> log.error("Failed to collect cluster {} info", cluster.getName(), e)) .onErrorResume( e -> Mono.just(Statistics.empty().toBuilder().lastKafkaException(e).build())); } - private Mono getLogDirInfo(ClusterDescription desc, ReactiveAdminClient ac) { - var brokerIds = desc.getNodes().stream().map(Node::id).collect(Collectors.toSet()); - return ac.describeLogDirs(brokerIds).map(InternalLogDirStats::new); + private Mono loadClusterState(ClusterDescription clusterDescription, ReactiveAdminClient ac) { + return ScrapedClusterState.scrape(clusterDescription, ac); } - private Mono> describeTopics(KafkaCluster c) { - return adminClientService.get(c).flatMap(ReactiveAdminClient::describeTopics); - } - - private Mono>> loadTopicConfigs(KafkaCluster c) { - return adminClientService.get(c).flatMap(ReactiveAdminClient::getTopicsConfig); + private Mono scrapeMetrics(KafkaCluster cluster, + ScrapedClusterState clusterState, + ClusterDescription clusterDescription) { + return cluster.getMetricsScrapping().scrape(clusterState, clusterDescription.getNodes()); } } diff --git a/api/src/main/java/io/kafbat/ui/service/TopicsService.java b/api/src/main/java/io/kafbat/ui/service/TopicsService.java index 95ad7bc5a..47d3c4c3d 100644 --- a/api/src/main/java/io/kafbat/ui/service/TopicsService.java +++ b/api/src/main/java/io/kafbat/ui/service/TopicsService.java @@ -10,7 +10,6 @@ import io.kafbat.ui.exception.TopicRecreationException; import io.kafbat.ui.exception.ValidationException; import io.kafbat.ui.model.ClusterFeature; -import io.kafbat.ui.model.InternalLogDirStats; import io.kafbat.ui.model.InternalPartition; import io.kafbat.ui.model.InternalPartitionsOffsets; import io.kafbat.ui.model.InternalReplica; @@ -25,6 +24,8 @@ import io.kafbat.ui.model.Statistics; import io.kafbat.ui.model.TopicCreationDTO; import io.kafbat.ui.model.TopicUpdateDTO; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.TopicState; import java.time.Duration; import java.util.Collection; import java.util.Collections; @@ -72,20 +73,19 @@ public Mono> loadTopics(KafkaCluster c, List topics) return adminClientService.get(c) .flatMap(ac -> ac.describeTopics(topics).zipWith(ac.getTopicsConfig(topics, false), - (descriptions, configs) -> { - statisticsCache.update(c, descriptions, configs); - return getPartitionOffsets(descriptions, ac).map(offsets -> { - var metrics = statisticsCache.get(c); - return createList( - topics, - descriptions, - configs, - offsets, - metrics.getMetrics(), - metrics.getLogDirInfo() - ); - }); - })).flatMap(Function.identity()); + (descriptions, configs) -> + getPartitionOffsets(descriptions, ac).map(offsets -> { + statisticsCache.update(c, descriptions, configs, offsets); + var stats = statisticsCache.get(c); + return createList( + topics, + descriptions, + configs, + offsets, + stats.getMetrics(), + stats.getClusterState() + ); + }))).flatMap(Function.identity()); } private Mono loadTopic(KafkaCluster c, String topicName) { @@ -123,7 +123,7 @@ private List createList(List orderedNames, Map> configs, InternalPartitionsOffsets partitionsOffsets, Metrics metrics, - InternalLogDirStats logDirInfo) { + ScrapedClusterState clusterState) { return orderedNames.stream() .filter(descriptions::containsKey) .map(t -> InternalTopic.from( @@ -131,7 +131,8 @@ private List createList(List orderedNames, configs.getOrDefault(t, List.of()), partitionsOffsets, metrics, - logDirInfo, + Optional.ofNullable(clusterState.getTopicStates().get(t)).map(s -> s.segmentStats()).orElse(null), + Optional.ofNullable(clusterState.getTopicStates().get(t)).map(s -> s.partitionsSegmentStats()).orElse(null), clustersProperties.getInternalTopicPrefix() )) .collect(toList()); @@ -224,7 +225,8 @@ private Mono updateTopic(KafkaCluster cluster, .then(loadTopic(cluster, topicName))); } - public Mono updateTopic(KafkaCluster cl, String topicName, Mono topicUpdate) { + public Mono updateTopic(KafkaCluster cl, String topicName, + Mono topicUpdate) { return topicUpdate .flatMap(t -> updateTopic(cl, topicName, t)); } @@ -443,19 +445,21 @@ public Mono cloneTopic( public Mono> getTopicsForPagination(KafkaCluster cluster) { Statistics stats = statisticsCache.get(cluster); - return filterExisting(cluster, stats.getTopicDescriptions().keySet()) + Map topicStates = stats.getClusterState().getTopicStates(); + return filterExisting(cluster, topicStates.keySet()) .map(lst -> lst.stream() .map(topicName -> InternalTopic.from( - stats.getTopicDescriptions().get(topicName), - stats.getTopicConfigs().getOrDefault(topicName, List.of()), + topicStates.get(topicName).description(), + topicStates.get(topicName).configs(), InternalPartitionsOffsets.empty(), stats.getMetrics(), - stats.getLogDirInfo(), + Optional.ofNullable(topicStates.get(topicName)) + .map(TopicState::segmentStats).orElse(null), + Optional.ofNullable(topicStates.get(topicName)) + .map(TopicState::partitionsSegmentStats).orElse(null), clustersProperties.getInternalTopicPrefix() - )) - .collect(toList()) - ); + )).collect(toList())); } public Mono>> getActiveProducersState(KafkaCluster cluster, String topic) { diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/GraphDescription.java b/api/src/main/java/io/kafbat/ui/service/graphs/GraphDescription.java new file mode 100644 index 000000000..e11eab014 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/graphs/GraphDescription.java @@ -0,0 +1,25 @@ +package io.kafbat.ui.service.graphs; + +import java.time.Duration; +import java.util.Set; +import javax.annotation.Nullable; +import lombok.Builder; + +@Builder +public record GraphDescription(String id, + @Nullable Duration defaultInterval, //null for instant queries, set for range + String prometheusQuery, + Set params) { + + public static GraphDescriptionBuilder instant() { + return builder(); + } + + public static GraphDescriptionBuilder range(Duration defaultInterval) { + return builder().defaultInterval(defaultInterval); + } + + public boolean isRange() { + return defaultInterval != null; + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/GraphDescriptions.java b/api/src/main/java/io/kafbat/ui/service/graphs/GraphDescriptions.java new file mode 100644 index 000000000..4645739e6 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/graphs/GraphDescriptions.java @@ -0,0 +1,74 @@ +package io.kafbat.ui.service.graphs; + +import static java.util.stream.Collectors.toMap; + +import io.kafbat.ui.exception.ValidationException; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; +import org.springframework.stereotype.Component; + +@Component +class GraphDescriptions { + + private static final Duration DEFAULT_RANGE_DURATION = Duration.ofDays(7); + + private final Map graphsById; + + GraphDescriptions() { + validate(); + this.graphsById = PREDEFINED_GRAPHS.stream().collect(toMap(GraphDescription::id, d -> d)); + } + + Optional getById(String id) { + return Optional.ofNullable(graphsById.get(id)); + } + + Stream all() { + return graphsById.values().stream(); + } + + private void validate() { + Map errors = new HashMap<>(); + for (GraphDescription description : PREDEFINED_GRAPHS) { + new PromQueryTemplate(description) + .validateSyntax() + .ifPresent(err -> errors.put(description.id(), err)); + } + if (!errors.isEmpty()) { + throw new ValidationException("Error validating queries for following graphs: " + errors); + } + } + + private static final List PREDEFINED_GRAPHS = List.of( + + GraphDescription.range(DEFAULT_RANGE_DURATION) + .id("broker_bytes_disk_ts") + .prometheusQuery("broker_bytes_disk{cluster=\"${cluster}\"}") + .params(Set.of()) + .build(), + + GraphDescription.instant() + .id("broker_bytes_disk") + .prometheusQuery("broker_bytes_disk{cluster=\"${cluster}\"}") + .params(Set.of()) + .build(), + + GraphDescription.instant() + .id("kafka_topic_partition_current_offset") + .prometheusQuery("kafka_topic_partition_current_offset{cluster=\"${cluster}\"}") + .params(Set.of()) + .build(), + + GraphDescription.range(DEFAULT_RANGE_DURATION) + .id("kafka_topic_partition_current_offset_per_topic_ts") + .prometheusQuery("kafka_topic_partition_current_offset{cluster=\"${cluster}\",topic = \"${topic}\"}") + .params(Set.of("topic")) + .build() + ); + +} diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/GraphsService.java b/api/src/main/java/io/kafbat/ui/service/graphs/GraphsService.java new file mode 100644 index 000000000..451feb1da --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/graphs/GraphsService.java @@ -0,0 +1,95 @@ +package io.kafbat.ui.service.graphs; + +import com.google.common.base.Preconditions; +import io.kafbat.ui.exception.NotFoundException; +import io.kafbat.ui.exception.ValidationException; +import io.kafbat.ui.model.KafkaCluster; +import io.kafbat.ui.prometheus.api.PrometheusClientApi; +import io.kafbat.ui.prometheus.model.QueryResponse; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +@Component +@RequiredArgsConstructor +public class GraphsService { + + private static final int TARGET_MATRIX_DATA_POINTS = 200; + + private final GraphDescriptions graphDescriptions; + + public Mono getGraphData(KafkaCluster cluster, + String id, + @Nullable Instant from, + @Nullable Instant to, + @Nullable Map params) { + + var graph = graphDescriptions.getById(id) + .orElseThrow(() -> new NotFoundException("No graph found with id = " + id)); + + var promClient = cluster.getPrometheusStorageClient(); + if (promClient == null) { + throw new ValidationException("Prometheus not configured for cluster"); + } + String preparedQuery = prepareQuery(graph, cluster.getName(), params); + return cluster.getPrometheusStorageClient() + .mono(client -> { + if (graph.isRange()) { + return queryRange(client, preparedQuery, graph.defaultInterval(), from, to); + } + return queryInstant(client, preparedQuery); + }); + } + + private Mono queryRange(PrometheusClientApi c, + String preparedQuery, + Duration defaultPeriod, + @Nullable Instant from, + @Nullable Instant to) { + if (from == null) { + from = Instant.now().minus(defaultPeriod); + } + if (to == null) { + to = Instant.now(); + } + Preconditions.checkArgument(to.isAfter(from)); + return c.queryRange( + preparedQuery, + String.valueOf(from.getEpochSecond()), + String.valueOf(to.getEpochSecond()), + calculateStepSize(from, to), + null + ); + } + + private String calculateStepSize(Instant from, Instant to) { + long intervalInSecs = to.getEpochSecond() - from.getEpochSecond(); + if (intervalInSecs <= TARGET_MATRIX_DATA_POINTS) { + return intervalInSecs + "s"; + } + int step = ((int) (((double) intervalInSecs) / TARGET_MATRIX_DATA_POINTS)); + return step + "s"; + } + + private Mono queryInstant(PrometheusClientApi c, String preparedQuery) { + return c.query(preparedQuery, null, null); + } + + private String prepareQuery(GraphDescription d, String clusterName, @Nullable Map params) { + return new PromQueryTemplate(d).getQuery(clusterName, Optional.ofNullable(params).orElse(Map.of())); + } + + public Stream getGraphs(KafkaCluster cluster) { + if (cluster.getPrometheusStorageClient() == null) { + return Stream.empty(); + } + return graphDescriptions.all(); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java new file mode 100644 index 000000000..466bd7e19 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java @@ -0,0 +1,35 @@ +package io.kafbat.ui.service.graphs; + +import java.util.Optional; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.misc.ParseCancellationException; +import promql.PromQLLexer; +import promql.PromQLParser; + +class PromQueryLangGrammar { + + // returns error msg, or empty if query is valid + static Optional validateExpression(String query) { + try { + parseExpression(query); + return Optional.empty(); + } catch (ParseCancellationException e) { + //TODO: add more descriptive msg + return Optional.of("Syntax error"); + } + } + + static PromQLParser.ExpressionContext parseExpression(String query) { + return createParser(query).expression(); + } + + private static PromQLParser createParser(String str) { + var parser = new PromQLParser(new CommonTokenStream(new PromQLLexer(CharStreams.fromString(str)))); + parser.removeErrorListeners(); + parser.setErrorHandler(new BailErrorStrategy()); + return parser; + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java new file mode 100644 index 000000000..2aca2e7a3 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java @@ -0,0 +1,51 @@ +package io.kafbat.ui.service.graphs; + +import com.google.common.collect.Sets; +import io.kafbat.ui.exception.ValidationException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.text.StrSubstitutor; + +class PromQueryTemplate { + + private static final String CLUSTER_LABEL_NAME = "cluster"; + + private final String queryTemplate; + private final Set paramsNames; + + PromQueryTemplate(GraphDescription d) { + this(d.prometheusQuery(), d.params()); + } + + PromQueryTemplate(String templateQueryString, Set paramsNames) { + this.queryTemplate = templateQueryString; + this.paramsNames = paramsNames; + } + + String getQuery(String clusterName, Map paramValues) { + var missingParams = Sets.difference(paramsNames, paramValues.keySet()); + if (!missingParams.isEmpty()) { + throw new ValidationException("Not all params set for query, missing: " + missingParams); + } + Map replacements = new HashMap<>(paramValues); + replacements.put(CLUSTER_LABEL_NAME, clusterName); + return replaceParams(replacements); + } + + // returns error msg or empty if no errors found + Optional validateSyntax() { + Map fakeReplacements = new HashMap<>(); + fakeReplacements.put(CLUSTER_LABEL_NAME, "1"); + paramsNames.forEach(paramName -> fakeReplacements.put(paramName, "1")); + + String prepared = replaceParams(fakeReplacements); + return PromQueryLangGrammar.validateExpression(prepared); + } + + private String replaceParams(Map replacements) { + return new StrSubstitutor(replacements).replace(queryTemplate); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java b/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java index 641a4a32a..46649606f 100644 --- a/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java +++ b/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java @@ -2,9 +2,9 @@ import com.google.common.collect.ImmutableMap; import io.kafbat.ui.model.KafkaCluster; -import io.kafbat.ui.model.Statistics; import io.kafbat.ui.service.StatisticsCache; import io.kafbat.ui.service.integration.odd.schema.DataSetFieldsExtractors; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; import io.kafbat.ui.sr.model.SchemaSubject; import java.net.URI; import java.util.List; @@ -37,10 +37,10 @@ class TopicsExporter { Flux export(KafkaCluster cluster) { String clusterOddrn = Oddrn.clusterOddrn(cluster); - Statistics stats = statisticsCache.get(cluster); - return Flux.fromIterable(stats.getTopicDescriptions().keySet()) + var clusterState = statisticsCache.get(cluster).getClusterState(); + return Flux.fromIterable(clusterState.getTopicStates().keySet()) .filter(topicFilter) - .flatMap(topic -> createTopicDataEntity(cluster, topic, stats)) + .flatMap(topic -> createTopicDataEntity(cluster, topic, clusterState.getTopicStates().get(topic))) .onErrorContinue( (th, topic) -> log.warn("Error exporting data for topic {}, cluster {}", topic, cluster.getName(), th)) .buffer(100) @@ -50,7 +50,7 @@ Flux export(KafkaCluster cluster) { .items(topicsEntities)); } - private Mono createTopicDataEntity(KafkaCluster cluster, String topic, Statistics stats) { + private Mono createTopicDataEntity(KafkaCluster cluster, String topic, ScrapedClusterState.TopicState topicState) { KafkaPath topicOddrnPath = Oddrn.topicOddrnPath(cluster, topic); return Mono.zip( @@ -70,13 +70,13 @@ private Mono createTopicDataEntity(KafkaCluster cluster, String topi .addMetadataItem( new MetadataExtension() .schemaUrl(URI.create("wontbeused.oops")) - .metadata(getTopicMetadata(topic, stats))); + .metadata(getTopicMetadata(topicState))); } ); } - private Map getNonDefaultConfigs(String topic, Statistics stats) { - List config = stats.getTopicConfigs().get(topic); + private Map getNonDefaultConfigs(ScrapedClusterState.TopicState topicState) { + List config = topicState.configs(); if (config == null) { return Map.of(); } @@ -85,12 +85,12 @@ private Map getNonDefaultConfigs(String topic, Statistics stats) .collect(Collectors.toMap(ConfigEntry::name, ConfigEntry::value)); } - private Map getTopicMetadata(String topic, Statistics stats) { - TopicDescription topicDescription = stats.getTopicDescriptions().get(topic); + private Map getTopicMetadata(ScrapedClusterState.TopicState topicState) { + TopicDescription topicDescription = topicState.description(); return ImmutableMap.builder() .put("partitions", topicDescription.partitions().size()) .put("replication_factor", topicDescription.partitions().get(0).replicas().size()) - .putAll(getNonDefaultConfigs(topic, stats)) + .putAll(getNonDefaultConfigs(topicState)) .build(); } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/MetricsCollector.java b/api/src/main/java/io/kafbat/ui/service/metrics/MetricsCollector.java deleted file mode 100644 index e9a08e8cb..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/MetricsCollector.java +++ /dev/null @@ -1,68 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import io.kafbat.ui.model.KafkaCluster; -import io.kafbat.ui.model.Metrics; -import io.kafbat.ui.model.MetricsConfig; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.Node; -import org.springframework.stereotype.Component; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.util.function.Tuple2; -import reactor.util.function.Tuples; - -@Component -@Slf4j -@RequiredArgsConstructor -public class MetricsCollector { - - private final JmxMetricsRetriever jmxMetricsRetriever; - private final PrometheusMetricsRetriever prometheusMetricsRetriever; - - public Mono getBrokerMetrics(KafkaCluster cluster, Collection nodes) { - return Flux.fromIterable(nodes) - .flatMap(n -> getMetrics(cluster, n).map(lst -> Tuples.of(n, lst))) - .collectMap(Tuple2::getT1, Tuple2::getT2) - .map(this::collectMetrics) - .defaultIfEmpty(Metrics.empty()); - } - - private Mono> getMetrics(KafkaCluster kafkaCluster, Node node) { - Flux metricFlux = Flux.empty(); - if (kafkaCluster.getMetricsConfig() != null) { - String type = kafkaCluster.getMetricsConfig().getType(); - if (type == null || type.equalsIgnoreCase(MetricsConfig.JMX_METRICS_TYPE)) { - metricFlux = jmxMetricsRetriever.retrieve(kafkaCluster, node); - } else if (type.equalsIgnoreCase(MetricsConfig.PROMETHEUS_METRICS_TYPE)) { - metricFlux = prometheusMetricsRetriever.retrieve(kafkaCluster, node); - } - } - return metricFlux.collectList(); - } - - public Metrics collectMetrics(Map> perBrokerMetrics) { - Metrics.MetricsBuilder builder = Metrics.builder() - .perBrokerMetrics( - perBrokerMetrics.entrySet() - .stream() - .collect(Collectors.toMap(e -> e.getKey().id(), Map.Entry::getValue))); - - populateWellknowMetrics(perBrokerMetrics).apply(builder); - - return builder.build(); - } - - private WellKnownMetrics populateWellknowMetrics(Map> perBrokerMetrics) { - WellKnownMetrics wellKnownMetrics = new WellKnownMetrics(); - perBrokerMetrics.forEach((node, metrics) -> - metrics.forEach(metric -> - wellKnownMetrics.populate(node, metric))); - return wellKnownMetrics; - } - -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/MetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/MetricsRetriever.java deleted file mode 100644 index aa7d8cc2e..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/MetricsRetriever.java +++ /dev/null @@ -1,9 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import io.kafbat.ui.model.KafkaCluster; -import org.apache.kafka.common.Node; -import reactor.core.publisher.Flux; - -interface MetricsRetriever { - Flux retrieve(KafkaCluster c, Node node); -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParser.java deleted file mode 100644 index 5662706d4..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParser.java +++ /dev/null @@ -1,46 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.math.NumberUtils; - -@Slf4j -class PrometheusEndpointMetricsParser { - - /** - * Matches openmetrics format. For example, string: - * kafka_server_BrokerTopicMetrics_FiveMinuteRate{name="BytesInPerSec",topic="__consumer_offsets",} 16.94886650744339 - * will produce: - * name=kafka_server_BrokerTopicMetrics_FiveMinuteRate - * value=16.94886650744339 - * labels={name="BytesInPerSec", topic="__consumer_offsets"}", - */ - private static final Pattern PATTERN = Pattern.compile( - "(?^\\w+)([ \t]*\\{*(?.*)}*)[ \\t]+(?[\\d]+\\.?[\\d]+)?"); - - static Optional parse(String s) { - Matcher matcher = PATTERN.matcher(s); - if (matcher.matches()) { - String value = matcher.group("value"); - String metricName = matcher.group("metricName"); - if (metricName == null || !NumberUtils.isCreatable(value)) { - return Optional.empty(); - } - var labels = Arrays.stream(matcher.group("properties").split(",")) - .filter(str -> !"".equals(str)) - .map(str -> str.split("=")) - .filter(spit -> spit.length == 2) - .collect(Collectors.toUnmodifiableMap( - str -> str[0].trim(), - str -> str[1].trim().replace("\"", ""))); - - return Optional.of(RawMetric.create(metricName, labels, new BigDecimal(value))); - } - return Optional.empty(); - } -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetriever.java deleted file mode 100644 index fa9da4a95..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetriever.java +++ /dev/null @@ -1,70 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; -import io.kafbat.ui.config.ClustersProperties; -import io.kafbat.ui.model.KafkaCluster; -import io.kafbat.ui.model.MetricsConfig; -import io.kafbat.ui.util.WebClientConfigurator; -import java.util.Arrays; -import java.util.Optional; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.common.Node; -import org.springframework.stereotype.Service; -import org.springframework.util.unit.DataSize; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.util.UriComponentsBuilder; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -@Service -@Slf4j -class PrometheusMetricsRetriever implements MetricsRetriever { - - private static final String METRICS_ENDPOINT_PATH = "/metrics"; - private static final int DEFAULT_EXPORTER_PORT = 11001; - - @Override - public Flux retrieve(KafkaCluster c, Node node) { - log.debug("Retrieving metrics from prometheus exporter: {}:{}", node.host(), c.getMetricsConfig().getPort()); - - MetricsConfig metricsConfig = c.getMetricsConfig(); - var webClient = new WebClientConfigurator() - .configureBufferSize(DataSize.ofMegabytes(20)) - .configureBasicAuth(metricsConfig.getUsername(), metricsConfig.getPassword()) - .configureSsl( - c.getOriginalProperties().getSsl(), - new ClustersProperties.KeystoreConfig( - metricsConfig.getKeystoreLocation(), - metricsConfig.getKeystorePassword())) - .build(); - - return retrieve(webClient, node.host(), c.getMetricsConfig()); - } - - @VisibleForTesting - Flux retrieve(WebClient webClient, String host, MetricsConfig metricsConfig) { - int port = Optional.ofNullable(metricsConfig.getPort()).orElse(DEFAULT_EXPORTER_PORT); - boolean sslEnabled = metricsConfig.isSsl() || metricsConfig.getKeystoreLocation() != null; - var request = webClient.get() - .uri(UriComponentsBuilder.newInstance() - .scheme(sslEnabled ? "https" : "http") - .host(host) - .port(port) - .path(METRICS_ENDPOINT_PATH).build().toUri()); - - WebClient.ResponseSpec responseSpec = request.retrieve(); - return responseSpec.bodyToMono(String.class) - .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) - .onErrorResume(th -> Mono.empty()) - .flatMapMany(body -> - Flux.fromStream( - Arrays.stream(body.split("\\n")) - .filter(str -> !Strings.isNullOrEmpty(str) && !str.startsWith("#")) // skipping comments strings - .map(PrometheusEndpointMetricsParser::parse) - .filter(Optional::isPresent) - .map(Optional::get) - ) - ); - } -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java b/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java index 80cc6073b..28e3fb361 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java @@ -1,10 +1,15 @@ package io.kafbat.ui.service.metrics; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; + import java.math.BigDecimal; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import lombok.AllArgsConstructor; -import lombok.EqualsAndHashCode; -import lombok.ToString; +import java.util.stream.Stream; public interface RawMetric { @@ -14,47 +19,32 @@ public interface RawMetric { BigDecimal value(); - // Key, that can be used for metrics reductions - default Object identityKey() { - return name() + "_" + labels(); - } - - RawMetric copyWithValue(BigDecimal newValue); - //-------------------------------------------------- static RawMetric create(String name, Map labels, BigDecimal value) { return new SimpleMetric(name, labels, value); } - @AllArgsConstructor - @EqualsAndHashCode - @ToString - class SimpleMetric implements RawMetric { - - private final String name; - private final Map labels; - private final BigDecimal value; - - @Override - public String name() { - return name; - } - - @Override - public Map labels() { - return labels; - } - - @Override - public BigDecimal value() { - return value; - } - - @Override - public RawMetric copyWithValue(BigDecimal newValue) { - return new SimpleMetric(name, labels, newValue); + static Stream groupIntoMfs(Collection rawMetrics) { + Map map = new LinkedHashMap<>(); + for (RawMetric m : rawMetrics) { + var gauge = map.computeIfAbsent(m.name(), + (n) -> GaugeSnapshot.builder() + .name(m.name()) + .help(m.name()) + ); + + List lbls = m.labels().keySet().stream().toList(); + List lblVals = lbls.stream().map(l -> m.labels().get(l)).toList(); + + GaugeSnapshot.GaugeDataPointSnapshot point = GaugeSnapshot.GaugeDataPointSnapshot.builder() + .value(m.value().doubleValue()) + .labels(Labels.of(lbls, lblVals)).build(); + gauge.dataPoint(point); } + return map.values().stream().map(GaugeSnapshot.Builder::build); } + record SimpleMetric(String name, Map labels, BigDecimal value) implements RawMetric { } + } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java new file mode 100644 index 000000000..dd553cc88 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -0,0 +1,101 @@ +package io.kafbat.ui.service.metrics; + +import static io.prometheus.metrics.model.snapshots.CounterSnapshot.*; +import static io.prometheus.metrics.model.snapshots.GaugeSnapshot.*; +import static java.util.stream.Collectors.toMap; + +import com.google.common.collect.Streams; +import io.kafbat.ui.model.Metrics; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class SummarizedMetrics { + + private final Metrics metrics; + + public Stream asStream() { + return Streams.concat( + metrics.getInferredMetrics().asStream(), + metrics.getPerBrokerScrapedMetrics() + .values() + .stream() + .flatMap(Collection::stream) + .collect( + toMap( + mfs -> mfs.getMetadata().getName(), + Optional::of, SummarizedMetrics::summarizeMetricSnapshot, LinkedHashMap::new + ) + ).values() + .stream() + .filter(Optional::isPresent) + .map(Optional::get) + ); + } + + //returns Optional.empty if merging not supported for metric type + @SuppressWarnings("unchecked") + private static Optional summarizeMetricSnapshot(Optional mfs1opt, + Optional mfs2opt) { + + if ((mfs1opt.isEmpty() || mfs2opt.isEmpty()) || !(mfs1opt.get().getClass().equals(mfs2opt.get().getClass()))) { + return Optional.empty(); + } + + var mfs1 = mfs1opt.get(); + + if (mfs1 instanceof GaugeSnapshot || mfs1 instanceof CounterSnapshot) { + BiFunction pointFactory; + Function valueGetter; + Function, MetricSnapshot> builder; + + if (mfs1 instanceof CounterSnapshot) { + pointFactory = (l, v) -> CounterDataPointSnapshot.builder() + .labels(l) + .value(v) + .build(); + valueGetter = (dp) -> ((CounterDataPointSnapshot)dp).getValue(); + builder = (dps) -> + new CounterSnapshot(mfs1.getMetadata(), (Collection)dps); + } else { + pointFactory = (l,v) -> GaugeDataPointSnapshot.builder() + .labels(l) + .value(v) + .build(); + valueGetter = (dp) -> ((GaugeDataPointSnapshot)dp).getValue(); + builder = (dps) -> + new GaugeSnapshot(mfs1.getMetadata(), (Collection)dps); + } + + Collection points = + Stream.concat(mfs1.getDataPoints().stream(), mfs2opt.get().getDataPoints().stream()) + .collect( + toMap( + // merging samples with same labels + DataPointSnapshot::getLabels, + s -> s, + (s1, s2) -> pointFactory.apply( + s1.getLabels(), + valueGetter.apply(s1) + valueGetter.apply(s2) + ), + LinkedHashMap::new + ) + ).values(); + return Optional.of(builder.apply(points)); + } else { + return Optional.empty(); + } + } + + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/WellKnownMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/WellKnownMetrics.java deleted file mode 100644 index 80e5f023c..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/WellKnownMetrics.java +++ /dev/null @@ -1,70 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; -import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; - -import io.kafbat.ui.model.Metrics; -import java.math.BigDecimal; -import java.util.HashMap; -import java.util.Map; -import org.apache.kafka.common.Node; - -class WellKnownMetrics { - - private static final String BROKER_TOPIC_METRICS = "BrokerTopicMetrics"; - private static final String FIFTEEN_MINUTE_RATE = "FifteenMinuteRate"; - - // per broker - final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); - final Map brokerBytesOutFifteenMinuteRate = new HashMap<>(); - - // per topic - final Map bytesInFifteenMinuteRate = new HashMap<>(); - final Map bytesOutFifteenMinuteRate = new HashMap<>(); - - void populate(Node node, RawMetric rawMetric) { - updateBrokerIOrates(node, rawMetric); - updateTopicsIOrates(rawMetric); - } - - void apply(Metrics.MetricsBuilder metricsBuilder) { - metricsBuilder.topicBytesInPerSec(bytesInFifteenMinuteRate); - metricsBuilder.topicBytesOutPerSec(bytesOutFifteenMinuteRate); - metricsBuilder.brokerBytesInPerSec(brokerBytesInFifteenMinuteRate); - metricsBuilder.brokerBytesOutPerSec(brokerBytesOutFifteenMinuteRate); - } - - private void updateBrokerIOrates(Node node, RawMetric rawMetric) { - String name = rawMetric.name(); - if (!brokerBytesInFifteenMinuteRate.containsKey(node.id()) - && rawMetric.labels().size() == 1 - && "BytesInPerSec".equalsIgnoreCase(rawMetric.labels().get("name")) - && containsIgnoreCase(name, BROKER_TOPIC_METRICS) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { - brokerBytesInFifteenMinuteRate.put(node.id(), rawMetric.value()); - } - if (!brokerBytesOutFifteenMinuteRate.containsKey(node.id()) - && rawMetric.labels().size() == 1 - && "BytesOutPerSec".equalsIgnoreCase(rawMetric.labels().get("name")) - && containsIgnoreCase(name, BROKER_TOPIC_METRICS) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { - brokerBytesOutFifteenMinuteRate.put(node.id(), rawMetric.value()); - } - } - - private void updateTopicsIOrates(RawMetric rawMetric) { - String name = rawMetric.name(); - String topic = rawMetric.labels().get("topic"); - if (topic != null - && containsIgnoreCase(name, BROKER_TOPIC_METRICS) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { - String nameProperty = rawMetric.labels().get("name"); - if ("BytesInPerSec".equalsIgnoreCase(nameProperty)) { - bytesInFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); - } else if ("BytesOutPerSec".equalsIgnoreCase(nameProperty)) { - bytesOutFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); - } - } - } - -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java new file mode 100644 index 000000000..d752a0297 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java @@ -0,0 +1,95 @@ +package io.kafbat.ui.service.metrics.prometheus; + + +import static io.kafbat.ui.util.MetricsUtils.appendLabel; +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; + +import com.google.common.annotations.VisibleForTesting; +import io.kafbat.ui.model.Metrics; +import io.kafbat.ui.util.MetricsUtils; +import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; + +public final class PrometheusExpose { + + private static final String CLUSTER_EXPOSE_LBL_NAME = "cluster"; + private static final String BROKER_EXPOSE_LBL_NAME = "broker_id"; + + private static final HttpHeaders PROMETHEUS_EXPOSE_ENDPOINT_HEADERS; + + static { + PROMETHEUS_EXPOSE_ENDPOINT_HEADERS = new HttpHeaders(); + PROMETHEUS_EXPOSE_ENDPOINT_HEADERS.set(CONTENT_TYPE, OpenMetricsTextFormatWriter.CONTENT_TYPE); + } + + private PrometheusExpose() { + } + + public static ResponseEntity exposeAllMetrics(Map clustersMetrics) { + return constructHttpsResponse(getMetricsForGlobalExpose(clustersMetrics)); + } + + private static MetricSnapshots getMetricsForGlobalExpose(Map clustersMetrics) { + return new MetricSnapshots(clustersMetrics.entrySet() + .stream() + .flatMap(e -> prepareMetricsForGlobalExpose(e.getKey(), e.getValue())) + // merging MFS with same name with LinkedHashMap(for order keeping) + .collect(Collectors.toMap(mfs -> mfs.getMetadata().getName(), mfs -> mfs, + MetricsUtils::concatDataPoints, LinkedHashMap::new)) + .values()); + } + + public static Stream prepareMetricsForGlobalExpose(String clusterName, Metrics metrics) { + return Stream.concat( + metrics.getInferredMetrics().asStream(), + extractBrokerMetricsWithLabel(metrics) + ) + .map(mfs -> appendLabel(mfs, CLUSTER_EXPOSE_LBL_NAME, clusterName)); + } + + private static Stream extractBrokerMetricsWithLabel(Metrics metrics) { + return metrics.getPerBrokerScrapedMetrics().entrySet().stream() + .flatMap(e -> { + String brokerId = String.valueOf(e.getKey()); + return e.getValue().stream().map(mfs -> appendLabel(mfs, BROKER_EXPOSE_LBL_NAME, brokerId)); + }); + } + + @VisibleForTesting + @SneakyThrows + public static ResponseEntity constructHttpsResponse(MetricSnapshots metrics) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(false, false); + writer.write(buffer, metrics); + return ResponseEntity + .ok() + .headers(PROMETHEUS_EXPOSE_ENDPOINT_HEADERS) + .body(buffer.toString(StandardCharsets.UTF_8)); + } + + // copied from io.prometheus.client.exporter.common.TextFormat:writeEscapedLabelValue + public static String escapedLabelValue(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + switch (c) { + case '\\' -> sb.append("\\\\"); + case '\"' -> sb.append("\\\""); + case '\n' -> sb.append("\\n"); + default -> sb.append(c); + } + } + return sb.toString(); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java new file mode 100644 index 000000000..13ff4ee00 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java @@ -0,0 +1,89 @@ +package io.kafbat.ui.service.metrics.scrape; + +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; +import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; + +import io.kafbat.ui.model.Metrics; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// Scans external jmx/prometheus metric and tries to infer io rates +class IoRatesMetricsScanner { + + // per broker + final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); + final Map brokerBytesOutFifteenMinuteRate = new HashMap<>(); + + // per topic + final Map bytesInFifteenMinuteRate = new HashMap<>(); + final Map bytesOutFifteenMinuteRate = new HashMap<>(); + + IoRatesMetricsScanner(Map> perBrokerMetrics) { + for (Map.Entry> broker : perBrokerMetrics.entrySet()) { + Integer nodeId = broker.getKey(); + List metrics = broker.getValue(); + for (MetricSnapshot metric : metrics) { + String name = metric.getMetadata().getName(); + if (metric instanceof GaugeSnapshot gauge) { + for (GaugeSnapshot.GaugeDataPointSnapshot dataPoint : gauge.getDataPoints()) { + updateBrokerIOrates(nodeId, name, dataPoint); + updateTopicsIOrates(name, dataPoint); + } + } + } + } + } + + Metrics.IoRates get() { + return Metrics.IoRates.builder() + .topicBytesInPerSec(bytesInFifteenMinuteRate) + .topicBytesOutPerSec(bytesOutFifteenMinuteRate) + .brokerBytesInPerSec(brokerBytesInFifteenMinuteRate) + .brokerBytesOutPerSec(brokerBytesOutFifteenMinuteRate) + .build(); + } + + private void updateBrokerIOrates(int nodeId, String name, GaugeSnapshot.GaugeDataPointSnapshot point) { + Labels labels = point.getLabels(); + if (!brokerBytesInFifteenMinuteRate.containsKey(nodeId) + && labels.size() == 1 + && "BytesInPerSec".equalsIgnoreCase(labels.getValue(0)) + && containsIgnoreCase(name, "BrokerTopicMetrics") + && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + brokerBytesInFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(point.getValue())); + } + if (!brokerBytesOutFifteenMinuteRate.containsKey(nodeId) + && labels.size() == 1 + && "BytesOutPerSec".equalsIgnoreCase(labels.getValue(0)) + && containsIgnoreCase(name, "BrokerTopicMetrics") + && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + brokerBytesOutFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(point.getValue())); + } + } + + private void updateTopicsIOrates(String name, GaugeSnapshot.GaugeDataPointSnapshot point) { + Labels labels = point.getLabels(); + if (labels.contains("topic") + && containsIgnoreCase(name, "BrokerTopicMetrics") + && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + String topic = labels.get("topic"); + if (labels.contains("name")) { + var nameLblVal = labels.get("name"); + if ("BytesInPerSec".equalsIgnoreCase(nameLblVal)) { + BigDecimal val = BigDecimal.valueOf(point.getValue()); + bytesInFifteenMinuteRate.merge(topic, val, BigDecimal::add); + } else if ("BytesOutPerSec".equalsIgnoreCase(nameLblVal)) { + BigDecimal val = BigDecimal.valueOf(point.getValue()); + bytesOutFifteenMinuteRate.merge(topic, val, BigDecimal::add); + } + } + } + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java new file mode 100644 index 000000000..70da53307 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java @@ -0,0 +1,92 @@ +package io.kafbat.ui.service.metrics.scrape; + +import static io.kafbat.ui.config.ClustersProperties.Cluster; +import static io.kafbat.ui.model.MetricsScrapeProperties.JMX_METRICS_TYPE; +import static io.kafbat.ui.model.MetricsScrapeProperties.PROMETHEUS_METRICS_TYPE; + +import io.kafbat.ui.model.Metrics; +import io.kafbat.ui.model.MetricsScrapeProperties; +import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; +import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetricsScraper; +import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; +import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsScraper; +import io.kafbat.ui.service.metrics.scrape.prometheus.PrometheusScraper; +import io.kafbat.ui.service.metrics.sink.MetricsSink; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import jakarta.annotation.Nullable; +import java.util.Collection; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Mono; + +@Slf4j +@RequiredArgsConstructor +public class MetricsScrapping { + + private final String clusterName; + private final MetricsSink sink; + private final InferredMetricsScraper inferredMetricsScraper; + @Nullable + private final JmxMetricsScraper jmxMetricsScraper; + @Nullable + private final PrometheusScraper prometheusScraper; + + public static MetricsScrapping create(Cluster cluster, + JmxMetricsRetriever jmxMetricsRetriever) { + JmxMetricsScraper jmxMetricsScraper = null; + PrometheusScraper prometheusScraper = null; + var metrics = cluster.getMetrics(); + if (cluster.getMetrics() != null) { + var scrapeProperties = MetricsScrapeProperties.create(cluster); + if (metrics.getType().equalsIgnoreCase(JMX_METRICS_TYPE) && metrics.getPort() != null) { + jmxMetricsScraper = new JmxMetricsScraper(scrapeProperties, jmxMetricsRetriever); + } else if (metrics.getType().equalsIgnoreCase(PROMETHEUS_METRICS_TYPE)) { + prometheusScraper = new PrometheusScraper(scrapeProperties); + } + } + return new MetricsScrapping( + cluster.getName(), + MetricsSink.create(cluster), + new InferredMetricsScraper(), + jmxMetricsScraper, + prometheusScraper + ); + } + + public Mono scrape(ScrapedClusterState clusterState, Collection nodes) { + Mono inferred = inferredMetricsScraper.scrape(clusterState); + Mono external = scrapeExternal(nodes); + return inferred.zipWith( + external, + (inf, ext) -> Metrics.builder() + .inferredMetrics(inf) + .ioRates(ext.ioRates()) + .perBrokerScrapedMetrics(ext.perBrokerMetrics()) + .build() + ).doOnNext(this::sendMetricsToSink); + } + + private void sendMetricsToSink(Metrics metrics) { + sink.send(prepareMetricsForSending(metrics)) + .doOnError(th -> log.warn("Error sending metrics to metrics sink", th)) + .subscribe(); + } + + private Stream prepareMetricsForSending(Metrics metrics) { + return PrometheusExpose.prepareMetricsForGlobalExpose(clusterName, metrics); + } + + private Mono scrapeExternal(Collection nodes) { + if (jmxMetricsScraper != null) { + return jmxMetricsScraper.scrape(nodes); + } + if (prometheusScraper != null) { + return prometheusScraper.scrape(nodes); + } + return Mono.just(PerBrokerScrapedMetrics.empty()); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java new file mode 100644 index 000000000..7bd733a65 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/PerBrokerScrapedMetrics.java @@ -0,0 +1,18 @@ +package io.kafbat.ui.service.metrics.scrape; + +import io.kafbat.ui.model.Metrics; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.List; +import java.util.Map; + +public record PerBrokerScrapedMetrics(Map> perBrokerMetrics) { + + static PerBrokerScrapedMetrics empty() { + return new PerBrokerScrapedMetrics(Map.of()); + } + + Metrics.IoRates ioRates() { + return new IoRatesMetricsScanner(perBrokerMetrics).get(); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java new file mode 100644 index 000000000..5ae280c64 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java @@ -0,0 +1,198 @@ +package io.kafbat.ui.service.metrics.scrape; + +import static io.kafbat.ui.model.InternalLogDirStats.LogDirSpaceStats; +import static io.kafbat.ui.model.InternalLogDirStats.SegmentStats; +import static io.kafbat.ui.service.ReactiveAdminClient.ClusterDescription; + +import com.google.common.collect.Table; +import io.kafbat.ui.model.InternalLogDirStats; +import io.kafbat.ui.model.InternalPartitionsOffsets; +import io.kafbat.ui.service.ReactiveAdminClient; +import jakarta.annotation.Nullable; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.ConsumerGroupDescription; +import org.apache.kafka.clients.admin.ConsumerGroupListing; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import reactor.core.publisher.Mono; + +@Builder(toBuilder = true) +@RequiredArgsConstructor +@Value +public class ScrapedClusterState { + + Instant scrapeFinishedAt; + Map nodesStates; + Map topicStates; + Map consumerGroupsStates; + + public record NodeState(int id, + Node node, + @Nullable SegmentStats segmentStats, + @Nullable LogDirSpaceStats logDirSpaceStats) { + } + + public record TopicState( + String name, + TopicDescription description, + List configs, + Map startOffsets, + Map endOffsets, + @Nullable SegmentStats segmentStats, + @Nullable Map partitionsSegmentStats) { + } + + public record ConsumerGroupState( + String group, + ConsumerGroupDescription description, + Map committedOffsets) { + } + + public static ScrapedClusterState empty() { + return ScrapedClusterState.builder() + .scrapeFinishedAt(Instant.now()) + .nodesStates(Map.of()) + .topicStates(Map.of()) + .consumerGroupsStates(Map.of()) + .build(); + } + + public ScrapedClusterState updateTopics(Map descriptions, + Map> configs, + InternalPartitionsOffsets partitionsOffsets) { + var updatedTopicStates = new HashMap<>(topicStates); + descriptions.forEach((topic, description) -> { + SegmentStats segmentStats = null; + Map partitionsSegmentStats = null; + if (topicStates.containsKey(topic)) { + segmentStats = topicStates.get(topic).segmentStats(); + partitionsSegmentStats = topicStates.get(topic).partitionsSegmentStats(); + } + updatedTopicStates.put( + topic, + new TopicState( + topic, + description, + configs.getOrDefault(topic, List.of()), + partitionsOffsets.topicOffsets(topic, true), + partitionsOffsets.topicOffsets(topic, false), + segmentStats, + partitionsSegmentStats + ) + ); + }); + return toBuilder() + .topicStates(updatedTopicStates) + .build(); + } + + public ScrapedClusterState topicDeleted(String topic) { + var newTopicStates = new HashMap<>(topicStates); + newTopicStates.remove(topic); + return toBuilder() + .topicStates(newTopicStates) + .build(); + } + + public static Mono scrape(ClusterDescription clusterDescription, + ReactiveAdminClient ac) { + return Mono.zip( + ac.describeLogDirs(clusterDescription.getNodes().stream().map(Node::id).toList()) + .map(InternalLogDirStats::new), + ac.listConsumerGroups().map(l -> l.stream().map(ConsumerGroupListing::groupId).toList()), + ac.describeTopics(), + ac.getTopicsConfig() + ).flatMap(phase1 -> + Mono.zip( + ac.listOffsets(phase1.getT3().values(), OffsetSpec.latest()), + ac.listOffsets(phase1.getT3().values(), OffsetSpec.earliest()), + ac.describeConsumerGroups(phase1.getT2()), + ac.listConsumerGroupOffsets(phase1.getT2(), null) + ).map(phase2 -> + create( + clusterDescription, + phase1.getT1(), + phase1.getT3(), + phase1.getT4(), + phase2.getT1(), + phase2.getT2(), + phase2.getT3(), + phase2.getT4() + ))); + } + + private static ScrapedClusterState create(ClusterDescription clusterDescription, + InternalLogDirStats segmentStats, + Map topicDescriptions, + Map> topicConfigs, + Map latestOffsets, + Map earliestOffsets, + Map consumerDescriptions, + Table consumerOffsets) { + + + Map topicStates = new HashMap<>(); + topicDescriptions.forEach((name, desc) -> + topicStates.put( + name, + new TopicState( + name, + desc, + topicConfigs.getOrDefault(name, List.of()), + filterTopic(name, earliestOffsets), + filterTopic(name, latestOffsets), + segmentStats.getTopicStats().get(name), + Optional.ofNullable(segmentStats.getPartitionsStats()) + .map(topicForFilter -> filterTopic(name, topicForFilter)) + .orElse(null) + ))); + + Map consumerGroupsStates = new HashMap<>(); + consumerDescriptions.forEach((name, desc) -> + consumerGroupsStates.put( + name, + new ConsumerGroupState( + name, + desc, + consumerOffsets.row(name) + ))); + + Map nodesStates = new HashMap<>(); + clusterDescription.getNodes().forEach(node -> + nodesStates.put( + node.id(), + new NodeState( + node.id(), + node, + segmentStats.getBrokerStats().get(node.id()), + segmentStats.getBrokerDirsStats().get(node.id()) + ))); + + return new ScrapedClusterState( + Instant.now(), + nodesStates, + topicStates, + consumerGroupsStates + ); + } + + private static Map filterTopic(String topicForFilter, Map tpMap) { + return tpMap.entrySet() + .stream() + .filter(tp -> tp.getKey().topic().equals(topicForFilter)) + .collect(Collectors.toMap(e -> e.getKey().partition(), Map.Entry::getValue)); + } + + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java new file mode 100644 index 000000000..7486b5480 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java @@ -0,0 +1,23 @@ +package io.kafbat.ui.service.metrics.scrape.inferred; + +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.List; +import java.util.stream.Stream; + +public class InferredMetrics { + + private final List metrics; + + public static InferredMetrics empty() { + return new InferredMetrics(List.of()); + } + + public InferredMetrics(List metrics) { + this.metrics = metrics; + } + + public Stream asStream() { + return metrics.stream(); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java new file mode 100644 index 000000000..3ef7ca8fd --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -0,0 +1,231 @@ +package io.kafbat.ui.service.metrics.scrape.inferred; + +import com.google.common.annotations.VisibleForTesting; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; +import io.prometheus.metrics.core.metrics.Gauge; +import io.prometheus.metrics.core.metrics.Metric; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.PrometheusNaming; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.MemberDescription; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Mono; + +@Slf4j +@RequiredArgsConstructor +public class InferredMetricsScraper { + + private ScrapedClusterState prevState = null; + + public synchronized Mono scrape(ScrapedClusterState newState) { + var inferred = infer(prevState, newState); + this.prevState = newState; + return Mono.just(inferred); + } + + @VisibleForTesting + static InferredMetrics infer(@Nullable ScrapedClusterState prevState, ScrapedClusterState newState) { + var registry = new MetricsRegistry(); + fillNodesMetrics(registry, newState); + fillTopicMetrics(registry, newState); + fillConsumerGroupsMetrics(registry, newState); + List metrics = registry.metrics.values().stream().map(Metric::collect).toList(); + log.debug("{} metric families inferred from cluster state", metrics.size()); + return new InferredMetrics(metrics); + } + + private static class MetricsRegistry { + + final Map metrics = new LinkedHashMap<>(); + + void gauge(String name, + String help, + List lbls, + List lblVals, + Number value) { + + + Gauge gauge = (Gauge) metrics.computeIfAbsent(name, (n) -> + Gauge.builder() + .name(PrometheusNaming.sanitizeMetricName(name)) + .labelNames(lbls.toArray(new String[0])) + .help(help).build() + ); + + gauge.labelValues(lblVals.toArray(new String[0])) + .set(value.doubleValue()); + } + } + + private static void fillNodesMetrics(MetricsRegistry registry, ScrapedClusterState newState) { + registry.gauge( + "broker_count", + "Number of brokers in the Kafka cluster", + List.of(), + List.of(), + newState.getNodesStates().size() + ); + + newState.getNodesStates().forEach((nodeId, state) -> { + if (state.segmentStats() != null) { + registry.gauge( + "broker_bytes_disk", + "Written disk size in bytes of a broker", + List.of("node_id"), + List.of(nodeId.toString()), + state.segmentStats().getSegmentSize() + ); + } + if (state.logDirSpaceStats() != null) { + if (state.logDirSpaceStats().usableBytes() != null) { + registry.gauge( + "broker_bytes_usable", + "Usable disk size in bytes of a broker", + List.of("node_id"), + List.of(nodeId.toString()), + state.logDirSpaceStats().usableBytes() + ); + } + if (state.logDirSpaceStats().totalBytes() != null) { + registry.gauge( + "broker_bytes_total", + "Total disk size in bytes of a broker", + List.of("node_id"), + List.of(nodeId.toString()), + state.logDirSpaceStats().totalBytes() + ); + } + } + }); + } + + private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterState clusterState) { + registry.gauge( + "topic_count", + "Number of topics in the Kafka cluster", + List.of(), + List.of(), + clusterState.getTopicStates().size() + ); + + clusterState.getTopicStates().forEach((topicName, state) -> { + registry.gauge( + "kafka_topic_partitions", + "Number of partitions for this Topic", + List.of("topic"), + List.of(topicName), + state.description().partitions().size() + ); + state.endOffsets().forEach((partition, endOffset) -> registry.gauge( + "kafka_topic_partition_current_offset", + "Current Offset of a Broker at Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(partition)), + endOffset + )); + state.startOffsets().forEach((partition, startOffset) -> registry.gauge( + "kafka_topic_partition_oldest_offset", + "Oldest Offset of a Broker at Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(partition)), + startOffset + )); + state.description().partitions().forEach(p -> { + registry.gauge( + "kafka_topic_partition_in_sync_replica", + "Number of In-Sync Replicas for this Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(p.partition())), + p.isr().size() + ); + registry.gauge( + "kafka_topic_partition_replicas", + "Number of Replicas for this Topic/Partition", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(p.partition())), + p.replicas().size() + ); + registry.gauge( + "kafka_topic_partition_leader", + "Leader Broker ID of this Topic/Partition (-1, if no leader)", + List.of("topic", "partition"), + List.of(topicName, String.valueOf(p.partition())), + Optional.ofNullable(p.leader()).map(Node::id).orElse(-1) + ); + }); + if (state.segmentStats() != null) { + registry.gauge( + "topic_bytes_disk", + "Disk size in bytes of a topic", + List.of("topic"), + List.of(topicName), + state.segmentStats().getSegmentSize() + ); + } + }); + } + + private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedClusterState clusterState) { + registry.gauge( + "group_count", + "Number of consumer groups in the Kafka cluster", + List.of(), + List.of(), + clusterState.getConsumerGroupsStates().size() + ); + + clusterState.getConsumerGroupsStates().forEach((groupName, state) -> { + registry.gauge( + "group_state", + "State of the consumer group, value = ordinal of org.apache.kafka.common.ConsumerGroupState", + List.of("group"), + List.of(groupName), + state.description().state().ordinal() + ); + registry.gauge( + "group_member_count", + "Number of member assignments in the consumer group.", + List.of("group"), + List.of(groupName), + state.description().members().size() + ); + registry.gauge( + "group_host_count", + "Number of distinct hosts in the consumer group.", + List.of("group"), + List.of(groupName), + state.description().members().stream().map(MemberDescription::host).distinct().count() + ); + + state.committedOffsets().forEach((tp, committedOffset) -> { + registry.gauge( + "kafka_consumergroup_current_offset", + "Current Offset of a ConsumerGroup at Topic/Partition", + List.of("consumergroup", "topic", "partition"), + List.of(groupName, tp.topic(), String.valueOf(tp.partition())), + committedOffset + ); + + Optional.ofNullable(clusterState.getTopicStates().get(tp.topic())) + .flatMap(s -> Optional.ofNullable(s.endOffsets().get(tp.partition()))) + .ifPresent(endOffset -> + registry.gauge( + "kafka_consumergroup_lag", + "Current Approximate Lag of a ConsumerGroup at Topic/Partition", + List.of("consumergroup", "topic", "partition"), + List.of(groupName, tp.topic(), String.valueOf(tp.partition())), + endOffset - committedOffset //TODO: check +-1 + )); + + }); + }); + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java similarity index 93% rename from api/src/main/java/io/kafbat/ui/service/metrics/JmxMetricsFormatter.java rename to api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 37323c7dd..a9f50b610 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -1,5 +1,6 @@ -package io.kafbat.ui.service.metrics; +package io.kafbat.ui.service.metrics.scrape.jmx; +import io.kafbat.ui.service.metrics.RawMetric; import java.math.BigDecimal; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -13,13 +14,13 @@ /** * Converts JMX metrics into JmxExporter prometheus format: format. */ -class JmxMetricsFormatter { +public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 private static final Pattern PROPERTY_PATTERN = Pattern.compile( "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)"); - static List constructMetricsList(ObjectName jmxMetric, + public static List constructMetricsList(ObjectName jmxMetric, MBeanAttributeInfo[] attributes, Object[] attrValues) { String domain = fixIllegalChars(jmxMetric.getDomain()); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/JmxMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java similarity index 58% rename from api/src/main/java/io/kafbat/ui/service/metrics/JmxMetricsRetriever.java rename to api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java index bcb5665bc..62171ccd3 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/JmxMetricsRetriever.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java @@ -1,6 +1,7 @@ -package io.kafbat.ui.service.metrics; +package io.kafbat.ui.service.metrics.scrape.jmx; -import io.kafbat.ui.model.KafkaCluster; +import io.kafbat.ui.model.MetricsScrapeProperties; +import io.kafbat.ui.service.metrics.RawMetric; import java.io.Closeable; import java.util.ArrayList; import java.util.HashMap; @@ -17,15 +18,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.common.Node; -import org.springframework.stereotype.Service; -import reactor.core.publisher.Flux; +import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -@Service +@Component //need to be a component, since @Slf4j -class JmxMetricsRetriever implements MetricsRetriever, Closeable { +public class JmxMetricsRetriever implements Closeable { private static final boolean SSL_JMX_SUPPORTED; @@ -43,35 +43,34 @@ public void close() { JmxSslSocketFactory.clearFactoriesCache(); } - @Override - public Flux retrieve(KafkaCluster c, Node node) { - if (isSslJmxEndpoint(c) && !SSL_JMX_SUPPORTED) { - log.warn("Cluster {} has jmx ssl configured, but it is not supported", c.getName()); - return Flux.empty(); + public Mono> retrieveFromNode(MetricsScrapeProperties scrapeProperties, Node node) { + if (isSslJmxEndpoint(scrapeProperties) && !SSL_JMX_SUPPORTED) { + log.warn("Cluster has jmx ssl configured, but it is not supported by app"); + return Mono.just(List.of()); } - return Mono.fromSupplier(() -> retrieveSync(c, node)) - .subscribeOn(Schedulers.boundedElastic()) - .flatMapMany(Flux::fromIterable); + return Mono.fromSupplier(() -> retrieveSync(scrapeProperties, node)) + .subscribeOn(Schedulers.boundedElastic()); } - private boolean isSslJmxEndpoint(KafkaCluster cluster) { - return cluster.getMetricsConfig().getKeystoreLocation() != null; + private boolean isSslJmxEndpoint(MetricsScrapeProperties scrapeProperties) { + return scrapeProperties.getKeystoreConfig() != null + && scrapeProperties.getKeystoreConfig().getKeystoreLocation() != null; } @SneakyThrows - private List retrieveSync(KafkaCluster c, Node node) { - String jmxUrl = JMX_URL + node.host() + ":" + c.getMetricsConfig().getPort() + "/" + JMX_SERVICE_TYPE; + private List retrieveSync(MetricsScrapeProperties scrapeProperties, Node node) { + String jmxUrl = JMX_URL + node.host() + ":" + scrapeProperties.getPort() + "/" + JMX_SERVICE_TYPE; log.debug("Collection JMX metrics for {}", jmxUrl); List result = new ArrayList<>(); - withJmxConnector(jmxUrl, c, jmxConnector -> getMetricsFromJmx(jmxConnector, result)); + withJmxConnector(jmxUrl, scrapeProperties, jmxConnector -> getMetricsFromJmx(jmxConnector, result)); log.debug("{} metrics collected for {}", result.size(), jmxUrl); return result; } private void withJmxConnector(String jmxUrl, - KafkaCluster c, + MetricsScrapeProperties scrapeProperties, Consumer consumer) { - var env = prepareJmxEnvAndSetThreadLocal(c); + var env = prepareJmxEnvAndSetThreadLocal(scrapeProperties); try (JMXConnector connector = JMXConnectorFactory.newJMXConnector(new JMXServiceURL(jmxUrl), env)) { try { connector.connect(env); @@ -87,25 +86,25 @@ private void withJmxConnector(String jmxUrl, } } - private Map prepareJmxEnvAndSetThreadLocal(KafkaCluster cluster) { - var metricsConfig = cluster.getMetricsConfig(); + private Map prepareJmxEnvAndSetThreadLocal(MetricsScrapeProperties scrapeProperties) { Map env = new HashMap<>(); - if (isSslJmxEndpoint(cluster)) { - var clusterSsl = cluster.getOriginalProperties().getSsl(); + if (isSslJmxEndpoint(scrapeProperties)) { + var truststoreConfig = scrapeProperties.getTruststoreConfig(); + var keystoreConfig = scrapeProperties.getKeystoreConfig(); JmxSslSocketFactory.setSslContextThreadLocal( - clusterSsl != null ? clusterSsl.getTruststoreLocation() : null, - clusterSsl != null ? clusterSsl.getTruststorePassword() : null, - metricsConfig.getKeystoreLocation(), - metricsConfig.getKeystorePassword() + truststoreConfig != null ? truststoreConfig.getTruststoreLocation() : null, + truststoreConfig != null ? truststoreConfig.getTruststorePassword() : null, + keystoreConfig != null ? keystoreConfig.getKeystoreLocation() : null, + keystoreConfig != null ? keystoreConfig.getKeystorePassword() : null ); JmxSslSocketFactory.editJmxConnectorEnv(env); } - if (StringUtils.isNotEmpty(metricsConfig.getUsername()) - && StringUtils.isNotEmpty(metricsConfig.getPassword())) { + if (StringUtils.isNotEmpty(scrapeProperties.getUsername()) + && StringUtils.isNotEmpty(scrapeProperties.getPassword())) { env.put( JMXConnector.CREDENTIALS, - new String[] {metricsConfig.getUsername(), metricsConfig.getPassword()} + new String[] {scrapeProperties.getUsername(), scrapeProperties.getPassword()} ); } return env; diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java new file mode 100644 index 000000000..ab024582a --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java @@ -0,0 +1,35 @@ +package io.kafbat.ui.service.metrics.scrape.jmx; + +import io.kafbat.ui.model.MetricsScrapeProperties; +import io.kafbat.ui.service.metrics.RawMetric; +import io.kafbat.ui.service.metrics.scrape.PerBrokerScrapedMetrics; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuples; + +public class JmxMetricsScraper { + + private final JmxMetricsRetriever jmxMetricsRetriever; + private final MetricsScrapeProperties scrapeProperties; + + public JmxMetricsScraper(MetricsScrapeProperties scrapeProperties, + JmxMetricsRetriever jmxMetricsRetriever) { + this.scrapeProperties = scrapeProperties; + this.jmxMetricsRetriever = jmxMetricsRetriever; + } + + public Mono scrape(Collection nodes) { + Mono>> collected = Flux.fromIterable(nodes) + .flatMap(n -> jmxMetricsRetriever.retrieveFromNode(scrapeProperties, n).map(metrics -> Tuples.of(n, metrics))) + .collectMap( + t -> t.getT1().id(), + t -> RawMetric.groupIntoMfs(t.getT2()).toList() + ); + return collected.map(PerBrokerScrapedMetrics::new); + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/JmxSslSocketFactory.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java similarity index 97% rename from api/src/main/java/io/kafbat/ui/service/metrics/JmxSslSocketFactory.java rename to api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java index 27ecf505b..a477a2e0b 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/JmxSslSocketFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java @@ -1,4 +1,4 @@ -package io.kafbat.ui.service.metrics; +package io.kafbat.ui.service.metrics.scrape.jmx; import com.google.common.base.Preconditions; import java.io.FileInputStream; @@ -61,9 +61,8 @@ class JmxSslSocketFactory extends javax.net.ssl.SSLSocketFactory { } catch (Exception e) { log.error("----------------------------------"); log.error("SSL can't be enabled for JMX retrieval. " - + "Make sure your java app run with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg. Err: {}", + + "Make sure your java app is running with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg. Err: {}", e.getMessage()); - log.trace("SSL can't be enabled for JMX retrieval", e); log.error("----------------------------------"); } SSL_JMX_SUPPORTED = sslJmxSupported; diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java new file mode 100644 index 000000000..12ab39267 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java @@ -0,0 +1,375 @@ +package io.kafbat.ui.service.metrics.scrape.prometheus; + +import com.google.common.base.Enums; +import io.prometheus.metrics.model.registry.PrometheusRegistry; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.InfoSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricMetadata; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.PrometheusNaming; +import io.prometheus.metrics.model.snapshots.SummarySnapshot; +import io.prometheus.metrics.model.snapshots.Unit; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +public class PrometheusEndpointParser { + + enum Type { + GAUGE, + COUNTER, + HISTOGRAM, + SUMMARY, + INFO + } + + record Sample(String name, Double value, Labels labels) {} + + private static final Type DEFAULT_TYPE = Type.GAUGE; + + private PrometheusEndpointParser() { + } + + private static class ParserContext { + final List registered = new ArrayList<>(); + + String name; + String help; + Type type; + String unit; + Set allowedNames = new HashSet<>(); + List samples = new ArrayList<>(); + + void registerAndReset() { + if (!samples.isEmpty()) { + MetricMetadata metadata = new MetricMetadata( + PrometheusNaming.sanitizeMetricName(name), + Optional.ofNullable(help).orElse(name), + Optional.ofNullable(unit).map(Unit::new).orElse(null) + ); + + registered.add( switch (type) { + case GAUGE -> new GaugeSnapshot(metadata, convertGauges(samples)); + case COUNTER -> new CounterSnapshot(metadata, convertCounters(samples)); + case HISTOGRAM -> new HistogramSnapshot(metadata, convertHistograms(samples)); + case SUMMARY -> new SummarySnapshot(metadata, convertSummary(samples)); + case INFO -> new InfoSnapshot(metadata, convertInfo(samples)); + } ); + } + //resetting state: + name = null; + help = null; + type = null; + unit = null; + allowedNames.clear(); + samples.clear(); + } + + private Collection convertInfo(List samples) { + // TODO: Implement this + return List.of(); + } + + private Collection convertSummary(List samples) { + // TODO: Implement this + return List.of(); + } + + private Collection convertHistograms(List samples) { + // TODO: Implement this + return List.of(); + } + + private Collection convertCounters(List samples) { + return samples.stream().map(s -> new CounterSnapshot.CounterDataPointSnapshot( + s.value, s.labels, null, 0 + )).toList(); + } + + private Collection convertGauges(List samples) { + return samples.stream().map(s -> new GaugeSnapshot.GaugeDataPointSnapshot( + s.value, s.labels, null + )).toList(); + } + + List getRegistered() { + registerAndReset(); // last in progress metric should be registered + return registered; + } + } + + // general logic taken from https://github.com/prometheus/client_python/blob/master/prometheus_client/parser.py + public static List parse(Stream lines) { + PrometheusRegistry registry = new PrometheusRegistry(); + + ParserContext context = new ParserContext(); + lines.map(String::trim) + .filter(s -> !s.isBlank()) + .forEach(line -> { + if (line.charAt(0) == '#') { + String[] parts = line.split("[ \t]+", 4); + if (parts.length >= 3) { + switch (parts[1]) { + case "HELP" -> processHelp(context, parts); + case "TYPE" -> processType(context, parts); + case "UNIT" -> processUnit(context, parts); + default -> { /* probably a comment */ } + } + } + } else { + processSample(context, line); + } + }); + return context.getRegistered(); + } + + private static void processUnit(ParserContext context, String[] parts) { + if (!parts[2].equals(context.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = parts[2]; + context.type = DEFAULT_TYPE; + context.allowedNames.add(context.name); + } + if (parts.length == 4) { + context.unit = parts[3]; + } + } + + private static void processHelp(ParserContext context, String[] parts) { + if (!parts[2].equals(context.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = parts[2]; + context.type = DEFAULT_TYPE; + context.allowedNames.add(context.name); + } + if (parts.length == 4) { + context.help = unescapeHelp(parts[3]); + } + } + + private static void processType(ParserContext context, String[] parts) { + if (!parts[2].equals(context.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = parts[2]; + } + + context.type = Enums.getIfPresent(Type.class, parts[3].toUpperCase()).or(DEFAULT_TYPE); + switch (context.type) { + case SUMMARY -> { + context.allowedNames.add(context.name); + context.allowedNames.add(context.name + "_count"); + context.allowedNames.add(context.name + "_sum"); + context.allowedNames.add(context.name + "_created"); + } + case HISTOGRAM -> { + context.allowedNames.add(context.name + "_count"); + context.allowedNames.add(context.name + "_sum"); + context.allowedNames.add(context.name + "_bucket"); + context.allowedNames.add(context.name + "_created"); + } + case COUNTER -> { + context.allowedNames.add(context.name); + context.allowedNames.add(context.name + "_total"); + context.allowedNames.add(context.name + "_created"); + } + case INFO -> { + context.allowedNames.add(context.name); + context.allowedNames.add(context.name + "_info"); + } + default -> context.allowedNames.add(context.name); + } + } + + private static void processSample(ParserContext context, String line) { + parseSampleLine(line).ifPresent(sample -> { + if (!context.allowedNames.contains(sample.name)) { + // starting new metric family - need to register (if possible) prev one + context.registerAndReset(); + context.name = sample.name; + context.type = DEFAULT_TYPE; + context.allowedNames.add(sample.name); + } + context.samples.add(sample); + }); + } + + private static String unescapeHelp(String text) { + // algorithm from https://github.com/prometheus/client_python/blob/a2dae6caeaf3c300db416ba10a2a3271693addd4/prometheus_client/parser.py + if (text == null || !text.contains("\\")) { + return text; + } + StringBuilder result = new StringBuilder(); + boolean slash = false; + for (int c = 0; c < text.length(); c++) { + char charAt = text.charAt(c); + if (slash) { + if (charAt == '\\') { + result.append('\\'); + } else if (charAt == 'n') { + result.append('\n'); + } else { + result.append('\\').append(charAt); + } + slash = false; + } else { + if (charAt == '\\') { + slash = true; + } else { + result.append(charAt); + } + } + } + if (slash) { + result.append("\\"); + } + return result.toString(); + } + + //returns empty if line is not valid sample string + private static Optional parseSampleLine(String line) { + + // algorithm copied from https://github.com/prometheus/client_python/blob/a2dae6caeaf3c300db416ba10a2a3271693addd4/prometheus_client/parser.py + StringBuilder name = new StringBuilder(); + StringBuilder labelname = new StringBuilder(); + StringBuilder labelvalue = new StringBuilder(); + StringBuilder value = new StringBuilder(); + List lblNames = new ArrayList<>(); + List lblVals = new ArrayList<>(); + + String state = "name"; + + for (int c = 0; c < line.length(); c++) { + char charAt = line.charAt(c); + if (state.equals("name")) { + if (charAt == '{') { + state = "startoflabelname"; + } else if (charAt == ' ' || charAt == '\t') { + state = "endofname"; + } else { + name.append(charAt); + } + } else if (state.equals("endofname")) { + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '{') { + state = "startoflabelname"; + } else { + value.append(charAt); + state = "value"; + } + } else if (state.equals("startoflabelname")) { + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '}') { + state = "endoflabels"; + } else { + labelname.append(charAt); + state = "labelname"; + } + } else if (state.equals("labelname")) { + if (charAt == '=') { + state = "labelvaluequote"; + } else if (charAt == '}') { + state = "endoflabels"; + } else if (charAt == ' ' || charAt == '\t') { + state = "labelvalueequals"; + } else { + labelname.append(charAt); + } + } else if (state.equals("labelvalueequals")) { + if (charAt == '=') { + state = "labelvaluequote"; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + return Optional.empty(); + } + } else if (state.equals("labelvaluequote")) { + if (charAt == '"') { + state = "labelvalue"; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + return Optional.empty(); + } + } else if (state.equals("labelvalue")) { + if (charAt == '\\') { + state = "labelvalueslash"; + } else if (charAt == '"') { + lblNames.add(labelname.toString()); + lblVals.add(labelvalue.toString()); + labelname.setLength(0); + labelvalue.setLength(0); + state = "nextlabel"; + } else { + labelvalue.append(charAt); + } + } else if (state.equals("labelvalueslash")) { + state = "labelvalue"; + if (charAt == '\\') { + labelvalue.append('\\'); + } else if (charAt == 'n') { + labelvalue.append('\n'); + } else if (charAt == '"') { + labelvalue.append('"'); + } else { + labelvalue.append('\\').append(charAt); + } + } else if (state.equals("nextlabel")) { + if (charAt == ',') { + state = "labelname"; + } else if (charAt == '}') { + state = "endoflabels"; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + return Optional.empty(); + } + } else if (state.equals("endoflabels")) { + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + value.append(charAt); + state = "value"; + } + } else if (state.equals("value")) { + if (charAt == ' ' || charAt == '\t') { + break; // timestamps are NOT supported - ignoring + } else { + value.append(charAt); + } + } + } + + return Optional.of( + new Sample( + name.toString(), + parseDouble(value.toString()), Labels.of(lblNames, lblVals) + ) + ); + } + + private static double parseDouble(String valueString) { + if (valueString.equalsIgnoreCase("NaN")) { + return Double.NaN; + } else if (valueString.equalsIgnoreCase("+Inf")) { + return Double.POSITIVE_INFINITY; + } else if (valueString.equalsIgnoreCase("-Inf")) { + return Double.NEGATIVE_INFINITY; + } + return Double.parseDouble(valueString); + } + + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java new file mode 100644 index 000000000..4a40be6a3 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java @@ -0,0 +1,53 @@ +package io.kafbat.ui.service.metrics.scrape.prometheus; + +import io.kafbat.ui.model.MetricsScrapeProperties; +import io.kafbat.ui.util.WebClientConfigurator; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.List; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.unit.DataSize; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Mono; + +@Slf4j +class PrometheusMetricsRetriever { + + private static final String METRICS_ENDPOINT_PATH = "/metrics"; + private static final int DEFAULT_EXPORTER_PORT = 11001; + + private final int port; + private final boolean sslEnabled; + private final WebClient webClient; + + PrometheusMetricsRetriever(MetricsScrapeProperties scrapeProperties) { + this.port = Optional.ofNullable(scrapeProperties.getPort()).orElse(DEFAULT_EXPORTER_PORT); + this.sslEnabled = scrapeProperties.isSsl() || scrapeProperties.getKeystoreConfig() != null; + this.webClient = new WebClientConfigurator() + .configureBufferSize(DataSize.ofMegabytes(20)) + .configureBasicAuth(scrapeProperties.getUsername(), scrapeProperties.getPassword()) + .configureSsl(scrapeProperties.getTruststoreConfig(), scrapeProperties.getKeystoreConfig()) + .build(); + } + + Mono> retrieve(String host) { + log.debug("Retrieving metrics from prometheus endpoint: {}:{}", host, port); + + var uri = UriComponentsBuilder.newInstance() + .scheme(sslEnabled ? "https" : "http") + .host(host) + .port(port) + .path(METRICS_ENDPOINT_PATH) + .build() + .toUri(); + + return webClient.get() + .uri(uri) + .retrieve() + .bodyToMono(String.class) + .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) + .map(body -> PrometheusEndpointParser.parse(body.lines())) + .onErrorResume(th -> Mono.just(List.of())); + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java new file mode 100644 index 000000000..38bcc44e5 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java @@ -0,0 +1,29 @@ +package io.kafbat.ui.service.metrics.scrape.prometheus; + +import io.kafbat.ui.model.MetricsScrapeProperties; +import io.kafbat.ui.service.metrics.scrape.PerBrokerScrapedMetrics; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; + +public class PrometheusScraper { + + private final PrometheusMetricsRetriever retriever; + + public PrometheusScraper(MetricsScrapeProperties scrapeProperties) { + this.retriever = new PrometheusMetricsRetriever(scrapeProperties); + } + + public Mono scrape(Collection clusterNodes) { + Mono>> collected = Flux.fromIterable(clusterNodes) + .flatMap(n -> retriever.retrieve(n.host()).map(metrics -> Tuples.of(n, metrics))) + .collectMap(t -> t.getT1().id(), Tuple2::getT2); + return collected.map(PerBrokerScrapedMetrics::new); + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java new file mode 100644 index 000000000..0c7b9c1ac --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java @@ -0,0 +1,78 @@ +package io.kafbat.ui.service.metrics.sink; + +import static io.kafbat.ui.service.MessagesService.createProducer; +import static io.kafbat.ui.service.metrics.prometheus.PrometheusExpose.escapedLabelValue; +import static io.kafbat.ui.util.MetricsUtils.readPointValue; +import static io.kafbat.ui.util.MetricsUtils.toGoString; +import static org.apache.kafka.clients.producer.ProducerConfig.COMPRESSION_TYPE_CONFIG; + +import com.fasterxml.jackson.databind.json.JsonMapper; +import io.kafbat.ui.config.ClustersProperties; +import io.prometheus.metrics.model.snapshots.Label; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import reactor.core.publisher.Mono; + +/* + * Format of records copied from https://github.com/Telefonica/prometheus-kafka-adapter + */ +@RequiredArgsConstructor +class KafkaSink implements MetricsSink { + + record KafkaMetric(String timestamp, String value, String name, Map labels) { } + + private static final JsonMapper JSON_MAPPER = new JsonMapper(); + + private static final Map PRODUCER_ADDITIONAL_CONFIGS = Map.of(COMPRESSION_TYPE_CONFIG, "gzip"); + + private final String topic; + private final Producer producer; + + static KafkaSink create(ClustersProperties.Cluster cluster, String targetTopic) { + return new KafkaSink(targetTopic, createProducer(cluster, PRODUCER_ADDITIONAL_CONFIGS)); + } + + @Override + public Mono send(Stream metrics) { + return Mono.fromRunnable(() -> { + String ts = Instant.now() + .truncatedTo(ChronoUnit.SECONDS) + .atZone(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_DATE_TIME); + + metrics.flatMap(m -> createRecord(ts, m)).forEach(producer::send); + }); + } + + private Stream> createRecord(String ts, MetricSnapshot metric) { + String name = metric.getMetadata().getName(); + return metric.getDataPoints().stream() + .map(sample -> { + var lbls = new LinkedHashMap(); + lbls.put("__name__", name); + + for (Label label : sample.getLabels()) { + lbls.put(label.getName(), escapedLabelValue(label.getValue())); + } + + var km = new KafkaMetric(ts, toGoString(readPointValue(sample)), name, lbls); + return new ProducerRecord<>(topic, toJsonBytes(km)); + }); + } + + @SneakyThrows + private static byte[] toJsonBytes(KafkaMetric m) { + return JSON_MAPPER.writeValueAsBytes(m); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java new file mode 100644 index 000000000..4136f54bc --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java @@ -0,0 +1,55 @@ +package io.kafbat.ui.service.metrics.sink; + +import static org.springframework.util.StringUtils.hasText; + +import io.kafbat.ui.config.ClustersProperties; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public interface MetricsSink { + + static MetricsSink create(ClustersProperties.Cluster cluster) { + List sinks = new ArrayList<>(); + Optional.ofNullable(cluster.getMetrics()) + .flatMap(metrics -> Optional.ofNullable(metrics.getStore())) + .flatMap(store -> Optional.ofNullable(store.getPrometheus())) + .ifPresent(prometheusConf -> { + if (hasText(prometheusConf.getUrl()) && Boolean.TRUE.equals(prometheusConf.getRemoteWrite())) { + sinks.add(new PrometheusRemoteWriteSink(prometheusConf.getUrl(), cluster.getSsl())); + } + if (hasText(prometheusConf.getPushGatewayUrl())) { + sinks.add( + PrometheusPushGatewaySink.create( + prometheusConf.getPushGatewayUrl(), + prometheusConf.getPushGatewayUsername(), + prometheusConf.getPushGatewayPassword() + )); + } + }); + + Optional.ofNullable(cluster.getMetrics()) + .flatMap(metrics -> Optional.ofNullable(metrics.getStore())) + .flatMap(store -> Optional.ofNullable(store.getKafka())) + .flatMap(kafka -> Optional.ofNullable(kafka.getTopic())) + .ifPresent(topic -> sinks.add(KafkaSink.create(cluster, topic))); + + return compoundSink(sinks); + } + + private static MetricsSink compoundSink(List sinks) { + return metricsStream -> { + var materialized = metricsStream.toList(); + return Flux.fromIterable(sinks) + .flatMap(sink -> sink.send(materialized.stream())) + .then(); + }; + } + + Mono send(Stream metrics); + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java new file mode 100644 index 000000000..622858656 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java @@ -0,0 +1,50 @@ +package io.kafbat.ui.service.metrics.sink; + +import static org.springframework.util.StringUtils.hasText; + +import io.prometheus.metrics.exporter.pushgateway.PushGateway; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +@RequiredArgsConstructor +class PrometheusPushGatewaySink implements MetricsSink { + + private final PushGateway pushGateway; + + @SneakyThrows + static PrometheusPushGatewaySink create(String url, + @Nullable String username, + @Nullable String passw) { + PushGateway.Builder builder = PushGateway.builder() + .address(url); + + + if (hasText(username) && hasText(passw)) { + builder.basicAuth(username, passw); + } + return new PrometheusPushGatewaySink(builder.build()); + } + + @Override + public Mono send(Stream metrics) { + List metricsToPush = metrics.toList(); + if (metricsToPush.isEmpty()) { + return Mono.empty(); + } + return Mono.fromRunnable(() -> pushSync(metricsToPush)) + .subscribeOn(Schedulers.boundedElastic()); + } + + @SneakyThrows + private void pushSync(List metricsToPush) { + pushGateway.push(() -> new MetricSnapshots(metricsToPush)); + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java new file mode 100644 index 000000000..b72690799 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java @@ -0,0 +1,80 @@ +package io.kafbat.ui.service.metrics.sink; + + +import io.kafbat.ui.config.ClustersProperties.TruststoreConfig; +import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import io.kafbat.ui.util.MetricsUtils; +import io.kafbat.ui.util.WebClientConfigurator; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.Label; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import jakarta.annotation.Nullable; +import java.net.URI; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import org.springframework.util.unit.DataSize; +import org.springframework.web.reactive.function.client.WebClient; +import org.xerial.snappy.Snappy; +import prometheus.Remote; +import prometheus.Types; +import reactor.core.publisher.Mono; + +class PrometheusRemoteWriteSink implements MetricsSink { + + private final URI writeEndpoint; + private final WebClient webClient; + + PrometheusRemoteWriteSink(String prometheusUrl, @Nullable TruststoreConfig truststoreConfig) { + this.writeEndpoint = URI.create(prometheusUrl).resolve("/api/v1/write"); + this.webClient = new WebClientConfigurator() + .configureSsl(truststoreConfig, null) + .configureBufferSize(DataSize.ofMegabytes(20)) + .build(); + } + + @SneakyThrows + @Override + public Mono send(Stream metrics) { + byte[] bytesToWrite = Snappy.compress(createWriteRequest(metrics).toByteArray()); + return webClient.post() + .uri(writeEndpoint) + .header("Content-Type", "application/x-protobuf") + .header("User-Agent", "promremote-kafbat-ui/0.1.0") + .header("Content-Encoding", "snappy") + .header("X-Prometheus-Remote-Write-Version", "0.1.0") + .bodyValue(bytesToWrite) + .retrieve() + .toBodilessEntity() + .then(); + } + + private static Remote.WriteRequest createWriteRequest(Stream metrics) { + long currentTs = System.currentTimeMillis(); + Remote.WriteRequest.Builder request = Remote.WriteRequest.newBuilder(); + metrics.forEach(mfs -> { + for (DataPointSnapshot dataPoint : mfs.getDataPoints()) { + Types.TimeSeries.Builder timeSeriesBuilder = Types.TimeSeries.newBuilder(); + timeSeriesBuilder.addLabels( + Types.Label.newBuilder().setName("__name__").setValue(mfs.getMetadata().getName()) + ); + for (Label label : dataPoint.getLabels()) { + timeSeriesBuilder.addLabels( + Types.Label.newBuilder() + .setName(label.getName()) + .setValue(PrometheusExpose.escapedLabelValue(label.getValue())) + ); + } + timeSeriesBuilder.addSamples( + Types.Sample.newBuilder() + .setValue(MetricsUtils.readPointValue(dataPoint)) + .setTimestamp(currentTs) + ); + request.addTimeseries(timeSeriesBuilder); + } + }); + //TODO: pass Metadata + return request.build(); + } + + +} diff --git a/api/src/main/java/io/kafbat/ui/util/KafkaServicesValidation.java b/api/src/main/java/io/kafbat/ui/util/KafkaServicesValidation.java index 019a33543..1871dbcc1 100644 --- a/api/src/main/java/io/kafbat/ui/util/KafkaServicesValidation.java +++ b/api/src/main/java/io/kafbat/ui/util/KafkaServicesValidation.java @@ -4,6 +4,7 @@ import io.kafbat.ui.connect.api.KafkaConnectClientApi; import io.kafbat.ui.model.ApplicationPropertyValidationDTO; +import io.kafbat.ui.prometheus.api.PrometheusClientApi; import io.kafbat.ui.service.ReactiveAdminClient; import io.kafbat.ui.service.ksql.KsqlApiClient; import io.kafbat.ui.sr.api.KafkaSrClientApi; @@ -140,5 +141,18 @@ public static Mono validateKsql( .onErrorResume(KafkaServicesValidation::invalid); } + public static Mono validatePrometheusStore( + Supplier> clientSupplier) { + ReactiveFailover client; + try { + client = clientSupplier.get(); + } catch (Exception e) { + log.error("Error creating Prometheus client", e); + return invalid("Error creating Prometheus client: " + e.getMessage()); + } + return client.mono(c -> c.query("1", null, null)) + .then(valid()) + .onErrorResume(KafkaServicesValidation::invalid); + } } diff --git a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java new file mode 100644 index 000000000..a7753301c --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java @@ -0,0 +1,107 @@ +package io.kafbat.ui.util; + +import static io.prometheus.metrics.model.snapshots.CounterSnapshot.*; +import static io.prometheus.metrics.model.snapshots.HistogramSnapshot.*; +import static io.prometheus.metrics.model.snapshots.SummarySnapshot.*; + +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.SummarySnapshot; +import java.util.Collection; +import java.util.List; +import java.util.stream.Stream; +import org.apache.kafka.clients.MetadataSnapshot; + +public final class MetricsUtils { + public static double readPointValue(DataPointSnapshot dps) { + return switch (dps) { + case GaugeDataPointSnapshot guage -> guage.getValue(); + case CounterDataPointSnapshot counter -> counter.getValue(); + default -> 0; + }; + } + + public static String toGoString(double d) { + if (d == Double.POSITIVE_INFINITY) { + return "+Inf"; + } else if (d == Double.NEGATIVE_INFINITY) { + return "-Inf"; + } else { + return Double.toString(d); + } + } + + public static MetricSnapshot appendLabel(MetricSnapshot md, String name, String value) { + return switch (md) { + case GaugeSnapshot gauge -> new GaugeSnapshot(gauge.getMetadata(), gauge.getDataPoints() + .stream().map(dp -> + new GaugeDataPointSnapshot( + dp.getValue(), + extendLabels(dp.getLabels(), name, value), + dp.getExemplar() + ) + ).toList()); + case CounterSnapshot counter -> new CounterSnapshot(counter.getMetadata(), counter.getDataPoints() + .stream().map(dp -> + new CounterDataPointSnapshot( + dp.getValue(), + extendLabels(dp.getLabels(), name, value), + dp.getExemplar(), + dp.getCreatedTimestampMillis(), + dp.getScrapeTimestampMillis() + ) + ).toList()); + case HistogramSnapshot histogram -> new HistogramSnapshot(histogram.getMetadata(), histogram.getDataPoints() + .stream().map(dp -> + new HistogramDataPointSnapshot( + dp.getClassicBuckets(), + dp.getSum(), + extendLabels(dp.getLabels(), name, value), + dp.getExemplars(), + dp.getCreatedTimestampMillis() + ) + ).toList()); + case SummarySnapshot summary -> new SummarySnapshot(summary.getMetadata(), summary.getDataPoints() + .stream().map(dp -> + new SummaryDataPointSnapshot( + dp.getCount(), + dp.getSum(), + dp.getQuantiles(), + extendLabels(dp.getLabels(), name, value), + dp.getExemplars(), + dp.getCreatedTimestampMillis() + ) + ).toList()); + default -> md; + }; + } + + @SuppressWarnings("unchecked") + public static MetricSnapshot concatDataPoints(MetricSnapshot d1, MetricSnapshot d2) { + List dataPoints = Stream.concat( + d1.getDataPoints().stream(), d2.getDataPoints().stream() + ).toList(); + + return switch (d1) { + case GaugeSnapshot g -> new GaugeSnapshot(g.getMetadata(), + (Collection) dataPoints); + case CounterSnapshot c -> new CounterSnapshot(c.getMetadata(), + (Collection) dataPoints); + case HistogramSnapshot h -> new HistogramSnapshot(h.getMetadata(), + (Collection) dataPoints); + case SummarySnapshot s -> new SummarySnapshot(s.getMetadata(), + (Collection) dataPoints); + default -> d1; + }; + } + + + public static Labels extendLabels(Labels labels, String name, String value) { + return labels.add(name, value); + } +} diff --git a/api/src/main/java/io/kafbat/ui/util/ReactiveFailover.java b/api/src/main/java/io/kafbat/ui/util/ReactiveFailover.java index 872e9ddf9..81efd63fd 100644 --- a/api/src/main/java/io/kafbat/ui/util/ReactiveFailover.java +++ b/api/src/main/java/io/kafbat/ui/util/ReactiveFailover.java @@ -81,9 +81,6 @@ private Mono mono(Function> f, List> candid .flatMap(f) .onErrorResume(failoverExceptionsPredicate, th -> { publisher.markFailed(); - if (candidates.size() == 1) { - return Mono.error(th); - } var newCandidates = candidates.stream().skip(1).filter(PublisherHolder::isActive).toList(); if (newCandidates.isEmpty()) { return Mono.error(th); @@ -106,9 +103,6 @@ private Flux flux(Function> f, List> candid .flatMapMany(f) .onErrorResume(failoverExceptionsPredicate, th -> { publisher.markFailed(); - if (candidates.size() == 1) { - return Flux.error(th); - } var newCandidates = candidates.stream().skip(1).filter(PublisherHolder::isActive).toList(); if (newCandidates.isEmpty()) { return Flux.error(th); diff --git a/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java b/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java new file mode 100644 index 000000000..fda959a2b --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java @@ -0,0 +1,23 @@ +package io.kafbat.ui.util; + +import io.kafbat.ui.config.ClustersProperties; +import java.util.Properties; +import javax.annotation.Nullable; +import org.apache.kafka.common.config.SslConfigs; + +public final class SslPropertiesUtil { + + private SslPropertiesUtil() { + } + + public static void addKafkaSslProperties(@Nullable ClustersProperties.TruststoreConfig truststoreConfig, + Properties sink) { + if (truststoreConfig != null && truststoreConfig.getTruststoreLocation() != null) { + sink.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststoreConfig.getTruststoreLocation()); + if (truststoreConfig.getTruststorePassword() != null) { + sink.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, truststoreConfig.getTruststorePassword()); + } + } + } + +} diff --git a/api/src/test/java/io/kafbat/ui/container/PrometheusContainer.java b/api/src/test/java/io/kafbat/ui/container/PrometheusContainer.java new file mode 100644 index 000000000..6df330789 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/container/PrometheusContainer.java @@ -0,0 +1,19 @@ +package io.kafbat.ui.container; + +import org.testcontainers.containers.GenericContainer; + +public class PrometheusContainer extends GenericContainer { + + public PrometheusContainer() { + super("prom/prometheus:latest"); + setCommandParts(new String[] { + "--web.enable-remote-write-receiver", + "--config.file=/etc/prometheus/prometheus.yml" + }); + addExposedPort(9090); + } + + public String url() { + return "http://" + getHost() + ":" + getMappedPort(9090); + } +} diff --git a/api/src/test/java/io/kafbat/ui/model/PartitionDistributionStatsTest.java b/api/src/test/java/io/kafbat/ui/model/PartitionDistributionStatsTest.java index 6c5086061..f168a7c12 100644 --- a/api/src/test/java/io/kafbat/ui/model/PartitionDistributionStatsTest.java +++ b/api/src/test/java/io/kafbat/ui/model/PartitionDistributionStatsTest.java @@ -23,28 +23,23 @@ void skewCalculatedBasedOnPartitionsCounts() { Node n4 = new Node(4, "n4", 9092); var stats = PartitionDistributionStats.create( - Statistics.builder() - .clusterDescription( - new ReactiveAdminClient.ClusterDescription(null, "test", Set.of(n1, n2, n3), null)) - .topicDescriptions( - Map.of( - "t1", new TopicDescription( - "t1", false, - List.of( - new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), - new TopicPartitionInfo(1, n2, List.of(n2, n3), List.of(n2, n3)) - ) - ), - "t2", new TopicDescription( - "t2", false, - List.of( - new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), - new TopicPartitionInfo(1, null, List.of(n2, n1), List.of(n1)) - ) - ) + List.of( + new TopicDescription( + "t1", false, + List.of( + new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), + new TopicPartitionInfo(1, n2, List.of(n2, n3), List.of(n2, n3)) + ) + ), + new TopicDescription( + "t2", false, + List.of( + new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), + new TopicPartitionInfo(1, null, List.of(n2, n1), List.of(n1)) ) ) - .build(), 4 + ), + 4 ); assertThat(stats.getPartitionLeaders()) diff --git a/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java b/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java index 08b211b40..3fb1a804f 100644 --- a/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java +++ b/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java @@ -72,7 +72,7 @@ void shouldListFirst25Topics() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -98,7 +98,7 @@ void shouldListFirst25TopicsSortedByNameDescendingOrder() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())); init(internalTopics); @@ -125,7 +125,7 @@ void shouldCalculateCorrectPageCountForNonDivisiblePageSize() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -144,7 +144,7 @@ void shouldCorrectlyHandleNonPositivePageNumberAndPageSize() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -163,7 +163,7 @@ void shouldListBotInternalAndNonInternalTopics() { .map(Objects::toString) .map(name -> new TopicDescription(name, Integer.parseInt(name) % 10 == 0, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -184,7 +184,7 @@ void shouldListOnlyNonInternalTopics() { .map(Objects::toString) .map(name -> new TopicDescription(name, Integer.parseInt(name) % 5 == 0, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -205,7 +205,7 @@ void shouldListOnlyTopicsContainingOne() { .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); @@ -227,7 +227,7 @@ void shouldListTopicsOrderedByPartitionsCount() { new TopicPartitionInfo(p, null, List.of(), List.of())) .collect(Collectors.toList()))) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), InternalPartitionsOffsets.empty(), - Metrics.empty(), InternalLogDirStats.empty(), "_")) + Metrics.empty(), null, null, "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())); init(internalTopics); diff --git a/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java b/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java index 6bf6887a7..2c51703dc 100644 --- a/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java @@ -1,5 +1,8 @@ package io.kafbat.ui.service.integration.odd; +import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.TopicState; +import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.empty; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; @@ -9,6 +12,8 @@ import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.model.Statistics; import io.kafbat.ui.service.StatisticsCache; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.TopicState; import io.kafbat.ui.sr.api.KafkaSrClientApi; import io.kafbat.ui.sr.model.SchemaSubject; import io.kafbat.ui.sr.model.SchemaType; @@ -59,15 +64,22 @@ void doesNotExportTopicsWhichDontFitFiltrationRule() { .thenReturn(Mono.error(WebClientResponseException.create(404, "NF", new HttpHeaders(), null, null, null))); stats = Statistics.empty() .toBuilder() - .topicDescriptions( - Map.of( - "_hidden", new TopicDescription("_hidden", false, List.of( - new TopicPartitionInfo(0, null, List.of(), List.of()) - )), - "visible", new TopicDescription("visible", false, List.of( - new TopicPartitionInfo(0, null, List.of(), List.of()) - )) - ) + .clusterState( + empty().toBuilder().topicStates( + Map.of( + "_hidden", + new TopicState( + "_hidden", + new TopicDescription("_hidden", false, List.of( + new TopicPartitionInfo(0, null, List.of(), List.of()) + )), null, null, null, null, null), + "visible", + new TopicState("visible", + new TopicDescription("visible", false, List.of( + new TopicPartitionInfo(0, null, List.of(), List.of()) + )), null, null, null, null, null) + ) + ).build() ) .build(); @@ -101,41 +113,44 @@ void doesExportTopicData() { stats = Statistics.empty() .toBuilder() - .topicDescriptions( - Map.of( - "testTopic", - new TopicDescription( - "testTopic", - false, - List.of( - new TopicPartitionInfo( - 0, - null, - List.of( - new Node(1, "host1", 9092), - new Node(2, "host2", 9092) + .clusterState( + ScrapedClusterState.empty().toBuilder() + .topicStates( + Map.of( + "testTopic", + new TopicState( + "testTopic", + new TopicDescription( + "testTopic", + false, + List.of( + new TopicPartitionInfo( + 0, + null, + List.of( + new Node(1, "host1", 9092), + new Node(2, "host2", 9092) + ), + List.of() + ) + ) ), - List.of()) - )) - ) - ) - .topicConfigs( - Map.of( - "testTopic", List.of( - new ConfigEntry( - "custom.config", - "100500", - ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, - false, - false, - List.of(), - ConfigEntry.ConfigType.INT, - null + List.of( + new ConfigEntry( + "custom.config", + "100500", + ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, + false, + false, + List.of(), + ConfigEntry.ConfigType.INT, + null + ) + ), null, null, null, null + ) ) - ) - ) - ) - .build(); + ).build() + ).build(); StepVerifier.create(topicsExporter.export(cluster)) .assertNext(entityList -> { diff --git a/api/src/test/java/io/kafbat/ui/service/ksql/KsqlApiClientTest.java b/api/src/test/java/io/kafbat/ui/service/ksql/KsqlApiClientTest.java index 90e549662..f50a7b6dd 100644 --- a/api/src/test/java/io/kafbat/ui/service/ksql/KsqlApiClientTest.java +++ b/api/src/test/java/io/kafbat/ui/service/ksql/KsqlApiClientTest.java @@ -10,12 +10,14 @@ import io.kafbat.ui.AbstractIntegrationTest; import java.time.Duration; import java.util.Map; +import org.junit.Ignore; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.shaded.org.awaitility.Awaitility; import reactor.test.StepVerifier; +@Ignore class KsqlApiClientTest extends AbstractIntegrationTest { @BeforeAll diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/JmxMetricsFormatterTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/JmxMetricsFormatterTest.java index 577d8aa6b..71a04ac68 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/JmxMetricsFormatterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/JmxMetricsFormatterTest.java @@ -2,6 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsFormatter; import java.math.BigDecimal; import java.util.List; import java.util.Map; diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java new file mode 100644 index 000000000..fd235b44c --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java @@ -0,0 +1,57 @@ +package io.kafbat.ui.service.metrics; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.DataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.MetricMetadata; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.Optional; + +public class MetricsUtils { + public static boolean isTheSameMetric(MetricSnapshot m1, MetricSnapshot m2) { + if (m1.getClass().equals(m2.getClass())) { + MetricMetadata metadata1 = m1.getMetadata(); + MetricMetadata metadata2 = m2.getMetadata(); + if ( + + metadata1.getName().equals(metadata2.getName()) && + metadata1.getHelp().equals(metadata2.getHelp()) && + Optional.ofNullable( + metadata1.getUnit()).map(u -> u.equals(metadata2.getUnit()) + ).orElse(metadata2.getUnit() == null) + ) { + if (m1.getDataPoints().size() == m2.getDataPoints().size()) { + for (int i = 0; i < m1.getDataPoints().size(); i++) { + var m1dp = m1.getDataPoints().get(i); + var m2dp = m2.getDataPoints().get(i); + boolean same = isTheSameDataPoint(m1dp, m2dp); + if (!same) return false; + } + return true; + } + } + } + return false; + } + + public static boolean isTheSameDataPoint(DataPointSnapshot dp1, DataPointSnapshot dp2) { + if (dp1.getClass().equals(dp2.getClass())) { + if (Optional.ofNullable(dp1.getLabels()).map(l -> l.equals(dp2.getLabels())) + .orElse(dp2.getLabels() == null)) { + if (dp1 instanceof GaugeSnapshot.GaugeDataPointSnapshot g1) { + GaugeSnapshot.GaugeDataPointSnapshot g2 = (GaugeSnapshot.GaugeDataPointSnapshot)dp2; + return Double.compare(g1.getValue(), g2.getValue()) == 0; + } + if (dp1 instanceof CounterSnapshot.CounterDataPointSnapshot c1) { + CounterSnapshot.CounterDataPointSnapshot c2 = (CounterSnapshot.CounterDataPointSnapshot)dp2; + return Double.compare(c1.getValue(), c2.getValue()) == 0; + } + return true; + } + } + return false; + } + +} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParserTest.java deleted file mode 100644 index 88636d019..000000000 --- a/api/src/test/java/io/kafbat/ui/service/metrics/PrometheusEndpointMetricsParserTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Map; -import java.util.Optional; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -class PrometheusEndpointMetricsParserTest { - - @Test - void test() { - String metricsString = - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate" - + "{name=\"BytesOutPerSec\",topic=\"__confluent.support.metrics\",} 123.1234"; - - Optional parsedOpt = PrometheusEndpointMetricsParser.parse(metricsString); - - Assertions.assertThat(parsedOpt).hasValueSatisfying(metric -> { - assertThat(metric.name()).isEqualTo("kafka_server_BrokerTopicMetrics_FifteenMinuteRate"); - assertThat(metric.value()).isEqualTo("123.1234"); - assertThat(metric.labels()).containsExactlyEntriesOf( - Map.of( - "name", "BytesOutPerSec", - "topic", "__confluent.support.metrics" - )); - }); - } - -} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetrieverTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetrieverTest.java deleted file mode 100644 index a4c63e391..000000000 --- a/api/src/test/java/io/kafbat/ui/service/metrics/PrometheusMetricsRetrieverTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import io.kafbat.ui.model.MetricsConfig; -import java.io.IOException; -import java.math.BigDecimal; -import java.util.List; -import java.util.Map; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.test.StepVerifier; - -class PrometheusMetricsRetrieverTest { - - private final PrometheusMetricsRetriever retriever = new PrometheusMetricsRetriever(); - - private final MockWebServer mockWebServer = new MockWebServer(); - - @BeforeEach - void startMockServer() throws IOException { - mockWebServer.start(); - } - - @AfterEach - void stopMockServer() throws IOException { - mockWebServer.close(); - } - - @Test - void callsMetricsEndpointAndConvertsResponceToRawMetric() { - var url = mockWebServer.url("/metrics"); - mockWebServer.enqueue(prepareResponse()); - - MetricsConfig metricsConfig = prepareMetricsConfig(url.port(), null, null); - - StepVerifier.create(retriever.retrieve(WebClient.create(), url.host(), metricsConfig)) - .expectNextSequence(expectedRawMetrics()) - // third metric should not be present, since it has "NaN" value - .verifyComplete(); - } - - @Test - void callsSecureMetricsEndpointAndConvertsResponceToRawMetric() { - var url = mockWebServer.url("/metrics"); - mockWebServer.enqueue(prepareResponse()); - - - MetricsConfig metricsConfig = prepareMetricsConfig(url.port(), "username", "password"); - - StepVerifier.create(retriever.retrieve(WebClient.create(), url.host(), metricsConfig)) - .expectNextSequence(expectedRawMetrics()) - // third metric should not be present, since it has "NaN" value - .verifyComplete(); - } - - MockResponse prepareResponse() { - // body copied from real jmx exporter - return new MockResponse().setBody( - "# HELP kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate Attribute exposed for management \n" - + "# TYPE kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate untyped\n" - + "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate{name=\"RequestHandlerAvgIdlePercent\",} 0.898\n" - + "# HELP kafka_server_socket_server_metrics_request_size_avg The average size of requests sent. \n" - + "# TYPE kafka_server_socket_server_metrics_request_size_avg untyped\n" - + "kafka_server_socket_server_metrics_request_size_avg{listener=\"PLAIN\",networkProcessor=\"1\",} 101.1\n" - + "kafka_server_socket_server_metrics_request_size_avg{listener=\"PLAIN2\",networkProcessor=\"5\",} NaN" - ); - } - - MetricsConfig prepareMetricsConfig(Integer port, String username, String password) { - return MetricsConfig.builder() - .ssl(false) - .port(port) - .type(MetricsConfig.PROMETHEUS_METRICS_TYPE) - .username(username) - .password(password) - .build(); - } - - List expectedRawMetrics() { - - var firstMetric = RawMetric.create( - "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate", - Map.of("name", "RequestHandlerAvgIdlePercent"), - new BigDecimal("0.898") - ); - - var secondMetric = RawMetric.create( - "kafka_server_socket_server_metrics_request_size_avg", - Map.of("listener", "PLAIN", "networkProcessor", "1"), - new BigDecimal("101.1") - ); - return List.of(firstMetric, secondMetric); - } -} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/WellKnownMetricsTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/WellKnownMetricsTest.java deleted file mode 100644 index 777db86cb..000000000 --- a/api/src/test/java/io/kafbat/ui/service/metrics/WellKnownMetricsTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package io.kafbat.ui.service.metrics; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.kafbat.ui.model.Metrics; -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; -import org.apache.kafka.common.Node; -import org.junit.jupiter.api.Test; - -class WellKnownMetricsTest { - - private final WellKnownMetrics wellKnownMetrics = new WellKnownMetrics(); - - @Test - void bytesIoTopicMetricsPopulated() { - populateWith( - new Node(0, "host", 123), - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",topic=\"test-topic\",} 1.0", - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",topic=\"test-topic\",} 2.0", - "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", - "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0", - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0" - ); - assertThat(wellKnownMetrics.bytesInFifteenMinuteRate) - .containsEntry("test-topic", new BigDecimal("3.0")); - assertThat(wellKnownMetrics.bytesOutFifteenMinuteRate) - .containsEntry("test-topic", new BigDecimal("6.0")); - } - - @Test - void bytesIoBrokerMetricsPopulated() { - populateWith( - new Node(1, "host1", 123), - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",} 1.0", - "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",} 2.0" - ); - populateWith( - new Node(2, "host2", 345), - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",} 10.0", - "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",} 20.0" - ); - - assertThat(wellKnownMetrics.brokerBytesInFifteenMinuteRate) - .hasSize(2) - .containsEntry(1, new BigDecimal("1.0")) - .containsEntry(2, new BigDecimal("10.0")); - - assertThat(wellKnownMetrics.brokerBytesOutFifteenMinuteRate) - .hasSize(2) - .containsEntry(1, new BigDecimal("2.0")) - .containsEntry(2, new BigDecimal("20.0")); - } - - @Test - void appliesInnerStateToMetricsBuilder() { - //filling per topic io rates - wellKnownMetrics.bytesInFifteenMinuteRate.put("topic", new BigDecimal(1)); - wellKnownMetrics.bytesOutFifteenMinuteRate.put("topic", new BigDecimal(2)); - - //filling per broker io rates - wellKnownMetrics.brokerBytesInFifteenMinuteRate.put(1, new BigDecimal(1)); - wellKnownMetrics.brokerBytesOutFifteenMinuteRate.put(1, new BigDecimal(2)); - wellKnownMetrics.brokerBytesInFifteenMinuteRate.put(2, new BigDecimal(10)); - wellKnownMetrics.brokerBytesOutFifteenMinuteRate.put(2, new BigDecimal(20)); - - Metrics.MetricsBuilder builder = Metrics.builder(); - wellKnownMetrics.apply(builder); - var metrics = builder.build(); - - // checking per topic io rates - assertThat(metrics.getTopicBytesInPerSec()).containsExactlyEntriesOf(wellKnownMetrics.bytesInFifteenMinuteRate); - assertThat(metrics.getTopicBytesOutPerSec()).containsExactlyEntriesOf(wellKnownMetrics.bytesOutFifteenMinuteRate); - - // checking per broker io rates - assertThat(metrics.getBrokerBytesInPerSec()).containsExactlyInAnyOrderEntriesOf( - Map.of(1, new BigDecimal(1), 2, new BigDecimal(10))); - assertThat(metrics.getBrokerBytesOutPerSec()).containsExactlyInAnyOrderEntriesOf( - Map.of(1, new BigDecimal(2), 2, new BigDecimal(20))); - } - - private void populateWith(Node n, String... prometheusMetric) { - Arrays.stream(prometheusMetric) - .map(PrometheusEndpointMetricsParser::parse) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(m -> wellKnownMetrics.populate(n, m)); - } - -} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java new file mode 100644 index 000000000..0cc945d66 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java @@ -0,0 +1,67 @@ +package io.kafbat.ui.service.metrics.prometheus; + +import static io.kafbat.ui.service.metrics.MetricsUtils.isTheSameMetric; +import static io.kafbat.ui.service.metrics.prometheus.PrometheusExpose.prepareMetricsForGlobalExpose; +import static org.assertj.core.api.Assertions.assertThat; + +import io.kafbat.ui.model.Metrics; +import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricMetadata; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class PrometheusExposeTest { + + @Test + void prepareMetricsForGlobalExposeAppendsClusterAndBrokerIdLabelsToMetrics() { + + var inferredMfs = new GaugeSnapshot(new MetricMetadata("infer", "help"), List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(100, Labels.of("lbl1","lblVal1"), null))); + + var broker1Mfs = new GaugeSnapshot(new MetricMetadata("brok", "help"), List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(101, Labels.of("broklbl1","broklblVal1"), null))); + + var broker2Mfs = new GaugeSnapshot(new MetricMetadata("brok", "help"), List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(102, Labels.of("broklbl1","broklblVal1"), null))); + + List prepared = prepareMetricsForGlobalExpose( + "testCluster", + Metrics.builder() + .inferredMetrics(new InferredMetrics(List.of(inferredMfs))) + .perBrokerScrapedMetrics(Map.of(1, List.of(broker1Mfs), 2, List.of(broker2Mfs))) + .build() + ).toList(); + + assertThat(prepared) + .hasSize(3) + .anyMatch(p -> isTheSameMetric(p, new GaugeSnapshot(new MetricMetadata("infer", "help"), List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(100, + Labels.of("cluster", "testCluster", "lbl1", "lblVal1"), null + )) + ))) + .anyMatch(p -> isTheSameMetric(p, new GaugeSnapshot(new MetricMetadata("brok", "help"), List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(101, + Labels.of( + "cluster", "testCluster", + "broker_id", "1", + "broklbl1","broklblVal1" + ), null + )) + ))) + .anyMatch(p -> isTheSameMetric(p, new GaugeSnapshot(new MetricMetadata("brok", "help"), List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(102, + Labels.of( + "cluster", "testCluster", + "broker_id", "2", + "broklbl1","broklblVal1" + ), null + )) + ) + )); + } + +} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java new file mode 100644 index 000000000..19dc2997f --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java @@ -0,0 +1,75 @@ +package io.kafbat.ui.service.metrics.scrape; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; + +import io.kafbat.ui.service.metrics.scrape.prometheus.PrometheusEndpointParser; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import org.apache.kafka.common.Node; +import org.junit.jupiter.api.Test; + +class IoRatesMetricsScannerTest { + + private IoRatesMetricsScanner ioRatesMetricsScanner; + + @Test + void bytesIoTopicMetricsPopulated() { + populateWith( + nodeMetrics( + new Node(0, "host", 123), + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",topic=\"test\",} 1.0", + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",topic=\"test\",} 2.0", + "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test\",} 1.0", + "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test\",} 2.0", + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test\",} 1.0", + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test\",} 2.0" + ) + ); + assertThat(ioRatesMetricsScanner.bytesInFifteenMinuteRate) + .containsEntry("test", new BigDecimal("3.0")); + assertThat(ioRatesMetricsScanner.bytesOutFifteenMinuteRate) + .containsEntry("test", new BigDecimal("6.0")); + } + + @Test + void bytesIoBrokerMetricsPopulated() { + populateWith( + nodeMetrics( + new Node(1, "host1", 123), + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",} 1.0", + "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",} 2.0" + ), + nodeMetrics( + new Node(2, "host2", 345), + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",} 10.0", + "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",} 20.0" + ) + ); + + assertThat(ioRatesMetricsScanner.brokerBytesInFifteenMinuteRate) + .hasSize(2) + .containsEntry(1, new BigDecimal("1.0")) + .containsEntry(2, new BigDecimal("10.0")); + + assertThat(ioRatesMetricsScanner.brokerBytesOutFifteenMinuteRate) + .hasSize(2) + .containsEntry(1, new BigDecimal("2.0")) + .containsEntry(2, new BigDecimal("20.0")); + } + + @SafeVarargs + private void populateWith(Map.Entry>... entries) { + ioRatesMetricsScanner = new IoRatesMetricsScanner( + stream(entries).collect(toMap(Map.Entry::getKey, Map.Entry::getValue)) + ); + } + + private Map.Entry> nodeMetrics(Node n, String... prometheusMetrics) { + return Map.entry(n.id(), PrometheusEndpointParser.parse(stream(prometheusMetrics))); + } + +} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java new file mode 100644 index 000000000..5353089cf --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java @@ -0,0 +1,198 @@ +package io.kafbat.ui.service.metrics.scrape; + +import static io.kafbat.ui.service.metrics.MetricsUtils.isTheSameMetric; +import static io.kafbat.ui.service.metrics.scrape.prometheus.PrometheusEndpointParser.parse; +import static java.lang.Double.POSITIVE_INFINITY; +import static org.assertj.core.api.Assertions.assertThat; + +import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.core.metrics.Gauge; +import io.prometheus.metrics.core.metrics.Histogram; +import io.prometheus.metrics.core.metrics.Info; +import io.prometheus.metrics.core.metrics.Summary; +import io.prometheus.metrics.model.registry.PrometheusRegistry; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricMetadata; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import org.junit.jupiter.api.Test; + +class PrometheusEndpointParserTest { + + @Test + void parsesAllGeneratedMetricTypes() { + MetricSnapshots original = generateMfs(); + String exposed = PrometheusExpose.constructHttpsResponse(original).getBody(); + List parsed = parse(exposed.lines()); + assertThat(parsed).containsExactlyElementsOf(original); + } + + @Test + void parsesMetricsFromPrometheusEndpointOutput() { + String expose = """ + # HELP http_requests_total The total number of HTTP requests. + # TYPE http_requests_total counter + http_requests_total{method="post",code="200",} 1027 1395066363000 + http_requests_total{method="post",code="400",} 3 1395066363000 + # Minimalistic line: + metric_without_timestamp_and_labels 12.47 + # A weird metric from before the epoch: + something_weird{problem="division by zero"} +Inf -3982045 + # TYPE something_untyped untyped + something_untyped{} -123123 + # TYPE unit_test_seconds counter + # UNIT unit_test_seconds seconds + # HELP unit_test_seconds Testing that unit parsed properly + unit_test_seconds_total 4.20072246e+06 + # HELP http_request_duration_seconds A histogram of the request duration. + # TYPE http_request_duration_seconds histogram + http_request_duration_seconds_bucket{le="0.05"} 24054 + http_request_duration_seconds_bucket{le="0.1"} 33444 + http_request_duration_seconds_bucket{le="0.2"} 100392 + http_request_duration_seconds_bucket{le="0.5"} 129389 + http_request_duration_seconds_bucket{le="1"} 133988 + http_request_duration_seconds_bucket{le="+Inf"} 144320 + http_request_duration_seconds_sum 53423 + http_request_duration_seconds_count 144320 + """; + List parsed = parse(expose.lines()); + + assertThat(parsed).anyMatch(p -> isTheSameMetric(p, + new CounterSnapshot( + new MetricMetadata("http_requests", "The total number of HTTP requests."), + List.of( + new CounterSnapshot.CounterDataPointSnapshot( + 1027, + Labels.of("method", "post", "code", "200"), + null, + 0 + ), + new CounterSnapshot.CounterDataPointSnapshot( + 3, + Labels.of("method", "post", "code", "400"), + null, + 0 + ) + ) + )) + ).anyMatch(p -> isTheSameMetric(p, + new GaugeSnapshot( + new MetricMetadata("metric_without_timestamp_and_labels", "metric_without_timestamp_and_labels"), + List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(12.47, Labels.EMPTY, null) + ) + )) + ).anyMatch(p -> isTheSameMetric(p, + new GaugeSnapshot( + new MetricMetadata("something_weird", "something_weird"), + List.of( + new GaugeSnapshot.GaugeDataPointSnapshot(POSITIVE_INFINITY, + Labels.of("problem", "division by zero"), null) + ) + )) + +// new MetricFamilySamples( +// "something_untyped", +// Type.GAUGE, +// "something_untyped", +// List.of(new Sample("something_untyped", List.of(), List.of(), -123123)) +// ), +// new MetricFamilySamples( +// "unit_test_seconds", +// "seconds", +// Type.COUNTER, +// "Testing that unit parsed properly", +// List.of(new Sample("unit_test_seconds_total", List.of(), List.of(), 4.20072246e+06)) +// ), +// new MetricFamilySamples( +// "http_request_duration_seconds", +// Type.HISTOGRAM, +// "A histogram of the request duration.", +// List.of( +// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.05"), 24054), +// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.1"), 33444), +// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.2"), 100392), +// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.5"), 129389), +// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("1"), 133988), +// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("+Inf"), 144320), +// new Sample("http_request_duration_seconds_sum", List.of(), List.of(), 53423), +// new Sample("http_request_duration_seconds_count", List.of(), List.of(), 144320) +// ) +// ) + ); + } + + private MetricSnapshots generateMfs() { + PrometheusRegistry collectorRegistry = new PrometheusRegistry(); + + Gauge.builder() + .name("test_gauge") + .help("help for gauge") + .register(collectorRegistry) + .set(42); + + Info.builder() + .name("test_info") + .help("help for info") + .labelNames("branch", "version", "revision") + .register(collectorRegistry) + .addLabelValues("HEAD", "1.2.3", "e0704b"); + + Counter.builder() + .name("counter_no_labels") + .help("help for counter no lbls") + .register(collectorRegistry) + .inc(111); + + var counterWithLbls = Counter.builder() + .name("counter_with_labels") + .help("help for counter with lbls") + .labelNames("lbl1", "lbl2") + .register(collectorRegistry); + + counterWithLbls.labelValues("v1", "v2").inc(234); + counterWithLbls.labelValues("v11", "v22").inc(345); + + var histogram = Histogram.builder() + .name("test_hist") + .help("help for hist") +// .linearBuckets(0.0, 1.0, 10) + .labelNames("lbl1", "lbl2") + .register(collectorRegistry); + + var summary = Summary.builder() + .name("test_summary") + .help("help for hist") + .labelNames("lbl1", "lbl2") + .register(collectorRegistry); + + for (int i = 0; i < 30; i++) { + var val = ThreadLocalRandom.current().nextDouble(10.0); + histogram.labelValues("v1", "v2").observe(val); + summary.labelValues("v1", "v2").observe(val); + } + +// //emulating unknown type +// collectorRegistry.register(new Collector() { +// @Override +// public List collect() { +// return List.of( +// new MetricFamilySamples( +// "test_unknown", +// Type.UNKNOWN, +// "help for unknown", +// List.of(new Sample("test_unknown", List.of("l1"), List.of("v1"), 23432.0)) +// ) +// ); +// } +// }); +// return Lists.newArrayList(Iterators.forEnumeration(collectorRegistry.scrape())); + return collectorRegistry.scrape(); + } + +} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java new file mode 100644 index 000000000..191de675c --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java @@ -0,0 +1,120 @@ +package io.kafbat.ui.service.metrics.scrape.inferred; + +import static io.kafbat.ui.model.InternalLogDirStats.LogDirSpaceStats; +import static io.kafbat.ui.model.InternalLogDirStats.SegmentStats; +import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.ConsumerGroupState; +import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.NodeState; +import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.TopicState; +import static org.assertj.core.api.Assertions.assertThat; + +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.kafka.clients.admin.ConsumerGroupDescription; +import org.apache.kafka.clients.admin.MemberAssignment; +import org.apache.kafka.clients.admin.MemberDescription; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionInfo; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +class InferredMetricsScraperTest { + + final InferredMetricsScraper scraper = new InferredMetricsScraper(); + + @Test + void allExpectedMetricsScraped() { + var segmentStats = new SegmentStats(1234L, 3); + var logDirStats = new LogDirSpaceStats(234L, 345L, Map.of(), Map.of()); + + Node node1 = new Node(1, "node1", 9092); + Node node2 = new Node(2, "node2", 9092); + + Mono scraped = scraper.scrape( + ScrapedClusterState.builder() + .scrapeFinishedAt(Instant.now()) + .nodesStates( + Map.of( + 1, new NodeState(1, node1, segmentStats, logDirStats), + 2, new NodeState(2, node2, segmentStats, logDirStats) + ) + ) + .topicStates( + Map.of( + "t1", + new TopicState( + "t1", + new TopicDescription( + "t1", + false, + List.of( + new TopicPartitionInfo(0, node1, List.of(node1, node2), List.of(node1, node2)), + new TopicPartitionInfo(1, node1, List.of(node1, node2), List.of(node1)) + ) + ), + List.of(), + Map.of(0, 100L, 1, 101L), + Map.of(0, 200L, 1, 201L), + segmentStats, + Map.of(0, segmentStats, 1, segmentStats) + ) + ) + ) + .consumerGroupsStates( + Map.of( + "cg1", + new ConsumerGroupState( + "cg1", + new ConsumerGroupDescription( + "cg1", + true, + List.of( + new MemberDescription( + "memb1", Optional.empty(), "client1", "hst1", + new MemberAssignment(Set.of(new TopicPartition("t1", 0))) + ) + ), + null, + org.apache.kafka.common.ConsumerGroupState.STABLE, + node1 + ), + Map.of(new TopicPartition("t1", 0), 150L) + ) + ) + ) + .build() + ); + + StepVerifier.create(scraped) + .assertNext(inferredMetrics -> + assertThat(inferredMetrics.asStream().map(m -> m.getMetadata().getName())).containsExactlyInAnyOrder( + "broker_count", + "broker_bytes_disk", + "broker_bytes_usable", + "broker_bytes", + "topic_count", + "kafka_topic_partitions", + "kafka_topic_partition_current_offset", + "kafka_topic_partition_oldest_offset", + "kafka_topic_partition_in_sync_replica", + "kafka_topic_partition_replicas", + "kafka_topic_partition_leader", + "topic_bytes_disk", + "group_count", + "group_state", + "group_member_count", + "group_host_count", + "kafka_consumergroup_current_offset", + "kafka_consumergroup_lag" + ) + ) + .verifyComplete(); + } + +} diff --git a/contract/build.gradle b/contract/build.gradle index 6aba087a9..98666b1b3 100644 --- a/contract/build.gradle +++ b/contract/build.gradle @@ -3,15 +3,16 @@ import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { id "java-library" alias(libs.plugins.openapi.generator) + alias(libs.plugins.protobuf) } - def specDir = project.layout.projectDirectory.dir("src/main/resources/swagger/") def targetDir = project.layout.buildDirectory.dir("generated").get() dependencies { implementation libs.spring.starter.webflux implementation libs.spring.starter.validation + implementation libs.protobuf api libs.swagger.integration.jakarta api libs.jackson.databind.nullable api libs.jakarta.annotation.api @@ -95,6 +96,23 @@ tasks.register('generateSchemaRegistryClient', GenerateTask) { dateLibrary : "java8",] } +tasks.register('generatePrometheusClient', GenerateTask) { + generatorName = "java" + inputSpec = specDir.file("prometheus-query-api.yaml").asFile.absolutePath + outputDir = targetDir.dir("prometheus-api").asFile.absolutePath + generateApiTests = false + generateModelTests = false + apiPackage = "io.kafbat.ui.prometheus.api" + invokerPackage = "io.kafbat.ui.prometheus" + modelPackage = "io.kafbat.ui.prometheus.model" + + configOptions = [asyncNative : "true", + library : "webclient", + useJakartaEe : "true", + useBeanValidation: "true", + dateLibrary : "java8",] +} + sourceSets { main { java { @@ -102,6 +120,7 @@ sourceSets { srcDir targetDir.dir("kafka-connect-client/src/main/java") srcDir targetDir.dir("kafbat-ui-client/src/main/java") srcDir targetDir.dir("kafka-sr-client/src/main/java") + srcDir targetDir.dir("prometheus-api/src/main/java") } resources { @@ -110,5 +129,10 @@ sourceSets { } } -compileJava.dependsOn generateUiClient, generateBackendApi, generateConnectClient, generateSchemaRegistryClient -processResources.dependsOn generateUiClient, generateBackendApi, generateConnectClient, generateSchemaRegistryClient +protobuf { + protoc { artifact = "com.google.protobuf:protoc:${libs.versions.protobuf.get()}" } +} + + +compileJava.dependsOn generateUiClient, generateBackendApi, generateConnectClient, generateSchemaRegistryClient, generatePrometheusClient +processResources.dependsOn generateUiClient, generateBackendApi, generateConnectClient, generateSchemaRegistryClient, generatePrometheusClient diff --git a/contract/src/main/proto/prometheus/gogoproto/gogo.proto b/contract/src/main/proto/prometheus/gogoproto/gogo.proto new file mode 100644 index 000000000..2f0a3c76b --- /dev/null +++ b/contract/src/main/proto/prometheus/gogoproto/gogo.proto @@ -0,0 +1,133 @@ +// Protocol Buffers for Go with Gadgets +// +// Copyright (c) 2013, The GoGo Authors. All rights reserved. +// http://github.com/gogo/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; +package gogoproto; + +import "google/protobuf/descriptor.proto"; + +option java_package = "com.google.protobuf"; +option java_outer_classname = "GoGoProtos"; +option go_package = "github.com/gogo/protobuf/gogoproto"; + +extend google.protobuf.EnumOptions { + optional bool goproto_enum_prefix = 62001; + optional bool goproto_enum_stringer = 62021; + optional bool enum_stringer = 62022; + optional string enum_customname = 62023; + optional bool enumdecl = 62024; +} + +extend google.protobuf.EnumValueOptions { + optional string enumvalue_customname = 66001; +} + +extend google.protobuf.FileOptions { + optional bool goproto_getters_all = 63001; + optional bool goproto_enum_prefix_all = 63002; + optional bool goproto_stringer_all = 63003; + optional bool verbose_equal_all = 63004; + optional bool face_all = 63005; + optional bool gostring_all = 63006; + optional bool populate_all = 63007; + optional bool stringer_all = 63008; + optional bool onlyone_all = 63009; + + optional bool equal_all = 63013; + optional bool description_all = 63014; + optional bool testgen_all = 63015; + optional bool benchgen_all = 63016; + optional bool marshaler_all = 63017; + optional bool unmarshaler_all = 63018; + optional bool stable_marshaler_all = 63019; + + optional bool sizer_all = 63020; + + optional bool goproto_enum_stringer_all = 63021; + optional bool enum_stringer_all = 63022; + + optional bool unsafe_marshaler_all = 63023; + optional bool unsafe_unmarshaler_all = 63024; + + optional bool goproto_extensions_map_all = 63025; + optional bool goproto_unrecognized_all = 63026; + optional bool gogoproto_import = 63027; + optional bool protosizer_all = 63028; + optional bool compare_all = 63029; + optional bool typedecl_all = 63030; + optional bool enumdecl_all = 63031; + + optional bool goproto_registration = 63032; +} + +extend google.protobuf.MessageOptions { + optional bool goproto_getters = 64001; + optional bool goproto_stringer = 64003; + optional bool verbose_equal = 64004; + optional bool face = 64005; + optional bool gostring = 64006; + optional bool populate = 64007; + optional bool stringer = 67008; + optional bool onlyone = 64009; + + optional bool equal = 64013; + optional bool description = 64014; + optional bool testgen = 64015; + optional bool benchgen = 64016; + optional bool marshaler = 64017; + optional bool unmarshaler = 64018; + optional bool stable_marshaler = 64019; + + optional bool sizer = 64020; + + optional bool unsafe_marshaler = 64023; + optional bool unsafe_unmarshaler = 64024; + + optional bool goproto_extensions_map = 64025; + optional bool goproto_unrecognized = 64026; + + optional bool protosizer = 64028; + optional bool compare = 64029; + + optional bool typedecl = 64030; +} + +extend google.protobuf.FieldOptions { + optional bool nullable = 65001; + optional bool embed = 65002; + optional string customtype = 65003; + optional string customname = 65004; + optional string jsontag = 65005; + optional string moretags = 65006; + optional string casttype = 65007; + optional string castkey = 65008; + optional string castvalue = 65009; + + optional bool stdtime = 65010; + optional bool stdduration = 65011; +} diff --git a/contract/src/main/proto/prometheus/remote.proto b/contract/src/main/proto/prometheus/remote.proto new file mode 100644 index 000000000..809d33f70 --- /dev/null +++ b/contract/src/main/proto/prometheus/remote.proto @@ -0,0 +1,88 @@ +// Copyright 2016 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package prometheus; + +option go_package = "prompb"; + +import "prometheus/types.proto"; +import "prometheus/gogoproto/gogo.proto"; + +message WriteRequest { + repeated prometheus.TimeSeries timeseries = 1 [(gogoproto.nullable) = false]; + // Cortex uses this field to determine the source of the write request. + // We reserve it to avoid any compatibility issues. + reserved 2; + repeated prometheus.MetricMetadata metadata = 3 [(gogoproto.nullable) = false]; +} + +// ReadRequest represents a remote read request. +message ReadRequest { + repeated Query queries = 1; + + enum ResponseType { + // Server will return a single ReadResponse message with matched series that includes list of raw samples. + // It's recommended to use streamed response types instead. + // + // Response headers: + // Content-Type: "application/x-protobuf" + // Content-Encoding: "snappy" + SAMPLES = 0; + // Server will stream a delimited ChunkedReadResponse message that + // contains XOR or HISTOGRAM(!) encoded chunks for a single series. + // Each message is following varint size and fixed size bigendian + // uint32 for CRC32 Castagnoli checksum. + // + // Response headers: + // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" + // Content-Encoding: "" + STREAMED_XOR_CHUNKS = 1; + } + + // accepted_response_types allows negotiating the content type of the response. + // + // Response types are taken from the list in the FIFO order. If no response type in `accepted_response_types` is + // implemented by server, error is returned. + // For request that do not contain `accepted_response_types` field the SAMPLES response type will be used. + repeated ResponseType accepted_response_types = 2; +} + +// ReadResponse is a response when response_type equals SAMPLES. +message ReadResponse { + // In same order as the request's queries. + repeated QueryResult results = 1; +} + +message Query { + int64 start_timestamp_ms = 1; + int64 end_timestamp_ms = 2; + repeated prometheus.LabelMatcher matchers = 3; + prometheus.ReadHints hints = 4; +} + +message QueryResult { + // Samples within a time series must be ordered by time. + repeated prometheus.TimeSeries timeseries = 1; +} + +// ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS. +// We strictly stream full series after series, optionally split by time. This means that a single frame can contain +// partition of the single series, but once a new series is started to be streamed it means that no more chunks will +// be sent for previous one. Series are returned sorted in the same way TSDB block are internally. +message ChunkedReadResponse { + repeated prometheus.ChunkedSeries chunked_series = 1; + + // query_index represents an index of the query from ReadRequest.queries these chunks relates to. + int64 query_index = 2; +} diff --git a/contract/src/main/proto/prometheus/types.proto b/contract/src/main/proto/prometheus/types.proto new file mode 100644 index 000000000..4a077bdb7 --- /dev/null +++ b/contract/src/main/proto/prometheus/types.proto @@ -0,0 +1,187 @@ +// Copyright 2017 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package prometheus; + +option go_package = "prompb"; + +import "prometheus/gogoproto/gogo.proto"; + +message MetricMetadata { + enum MetricType { + UNKNOWN = 0; + COUNTER = 1; + GAUGE = 2; + HISTOGRAM = 3; + GAUGEHISTOGRAM = 4; + SUMMARY = 5; + INFO = 6; + STATESET = 7; + } + + // Represents the metric type, these match the set from Prometheus. + // Refer to model/textparse/interface.go for details. + MetricType type = 1; + string metric_family_name = 2; + string help = 4; + string unit = 5; +} + +message Sample { + double value = 1; + // timestamp is in ms format, see model/timestamp/timestamp.go for + // conversion from time.Time to Prometheus timestamp. + int64 timestamp = 2; +} + +message Exemplar { + // Optional, can be empty. + repeated Label labels = 1 [(gogoproto.nullable) = false]; + double value = 2; + // timestamp is in ms format, see model/timestamp/timestamp.go for + // conversion from time.Time to Prometheus timestamp. + int64 timestamp = 3; +} + +// A native histogram, also known as a sparse histogram. +// Original design doc: +// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit +// The appendix of this design doc also explains the concept of float +// histograms. This Histogram message can represent both, the usual +// integer histogram as well as a float histogram. +message Histogram { + enum ResetHint { + UNKNOWN = 0; // Need to test for a counter reset explicitly. + YES = 1; // This is the 1st histogram after a counter reset. + NO = 2; // There was no counter reset between this and the previous Histogram. + GAUGE = 3; // This is a gauge histogram where counter resets don't happen. + } + + oneof count { // Count of observations in the histogram. + uint64 count_int = 1; + double count_float = 2; + } + double sum = 3; // Sum of observations in the histogram. + // The schema defines the bucket schema. Currently, valid numbers + // are -4 <= n <= 8. They are all for base-2 bucket schemas, where 1 + // is a bucket boundary in each case, and then each power of two is + // divided into 2^n logarithmic buckets. Or in other words, each + // bucket boundary is the previous boundary times 2^(2^-n). In the + // future, more bucket schemas may be added using numbers < -4 or > + // 8. + sint32 schema = 4; + double zero_threshold = 5; // Breadth of the zero bucket. + oneof zero_count { // Count in zero bucket. + uint64 zero_count_int = 6; + double zero_count_float = 7; + } + + // Negative Buckets. + repeated BucketSpan negative_spans = 8 [(gogoproto.nullable) = false]; + // Use either "negative_deltas" or "negative_counts", the former for + // regular histograms with integer counts, the latter for float + // histograms. + repeated sint64 negative_deltas = 9; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + repeated double negative_counts = 10; // Absolute count of each bucket. + + // Positive Buckets. + repeated BucketSpan positive_spans = 11 [(gogoproto.nullable) = false]; + // Use either "positive_deltas" or "positive_counts", the former for + // regular histograms with integer counts, the latter for float + // histograms. + repeated sint64 positive_deltas = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + repeated double positive_counts = 13; // Absolute count of each bucket. + + ResetHint reset_hint = 14; + // timestamp is in ms format, see model/timestamp/timestamp.go for + // conversion from time.Time to Prometheus timestamp. + int64 timestamp = 15; +} + +// A BucketSpan defines a number of consecutive buckets with their +// offset. Logically, it would be more straightforward to include the +// bucket counts in the Span. However, the protobuf representation is +// more compact in the way the data is structured here (with all the +// buckets in a single array separate from the Spans). +message BucketSpan { + sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative). + uint32 length = 2; // Length of consecutive buckets. +} + +// TimeSeries represents samples and labels for a single time series. +message TimeSeries { + // For a timeseries to be valid, and for the samples and exemplars + // to be ingested by the remote system properly, the labels field is required. + repeated Label labels = 1 [(gogoproto.nullable) = false]; + repeated Sample samples = 2 [(gogoproto.nullable) = false]; + repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false]; + repeated Histogram histograms = 4 [(gogoproto.nullable) = false]; +} + +message Label { + string name = 1; + string value = 2; +} + +message Labels { + repeated Label labels = 1 [(gogoproto.nullable) = false]; +} + +// Matcher specifies a rule, which can match or set of labels or not. +message LabelMatcher { + enum Type { + EQ = 0; + NEQ = 1; + RE = 2; + NRE = 3; + } + Type type = 1; + string name = 2; + string value = 3; +} + +message ReadHints { + int64 step_ms = 1; // Query step size in milliseconds. + string func = 2; // String representation of surrounding function or aggregation. + int64 start_ms = 3; // Start time in milliseconds. + int64 end_ms = 4; // End time in milliseconds. + repeated string grouping = 5; // List of label names used in aggregation. + bool by = 6; // Indicate whether it is without or by. + int64 range_ms = 7; // Range vector selector range in milliseconds. +} + +// Chunk represents a TSDB chunk. +// Time range [min, max] is inclusive. +message Chunk { + int64 min_time_ms = 1; + int64 max_time_ms = 2; + + // We require this to match chunkenc.Encoding. + enum Encoding { + UNKNOWN = 0; + XOR = 1; + HISTOGRAM = 2; + FLOAT_HISTOGRAM = 3; + } + Encoding type = 3; + bytes data = 4; +} + +// ChunkedSeries represents single, encoded time series. +message ChunkedSeries { + // Labels should be sorted. + repeated Label labels = 1 [(gogoproto.nullable) = false]; + // Chunks will be in start time order and may overlap. + repeated Chunk chunks = 2 [(gogoproto.nullable) = false]; +} diff --git a/contract/src/main/resources/swagger/kafbat-ui-api.yaml b/contract/src/main/resources/swagger/kafbat-ui-api.yaml index 454d78c7c..e418bcb34 100644 --- a/contract/src/main/resources/swagger/kafbat-ui-api.yaml +++ b/contract/src/main/resources/swagger/kafbat-ui-api.yaml @@ -31,6 +31,52 @@ paths: items: $ref: '#/components/schemas/Cluster' + /api/clusters/{clusterName}/graphs/descriptions: + get: + tags: + - Graphs + summary: getGraphsList + operationId: getGraphsList + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/GraphDescriptions' + + /api/clusters/{clusterName}/graphs/prometheus: + post: + tags: + - Graphs + summary: getGraphData + operationId: getGraphData + parameters: + - name: clusterName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GraphDataRequest' + responses: + 200: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PrometheusApiQueryResponse' + /api/clusters/{clusterName}/cache: post: tags: @@ -155,6 +201,20 @@ paths: schema: $ref: '#/components/schemas/ClusterMetrics' + /metrics: + get: + tags: + - PrometheusExpose + summary: getAllMetrics + operationId: getAllMetrics + responses: + 200: + description: OK + content: + application/text: + schema: + type: string + /api/clusters/{clusterName}/stats: get: tags: @@ -4117,6 +4177,112 @@ components: location: type: string + GraphDataRequest: + type: object + properties: + id: + type: string + parameters: + type: object + additionalProperties: + type: string + from: + type: string + format: date-time + to: + type: string + format: date-time + + GraphDescriptions: + type: object + properties: + graphs: + type: array + items: + $ref: '#/components/schemas/GraphDescription' + + GraphDescription: + type: object + required: ["id"] + properties: + id: + type: string + description: Id that should be used to query data on API level + type: + type: string + enum: ["range", "instant"] + defaultPeriod: + type: string + description: ISO_8601 duration string (for "range" graphs only) + parameters: + type: array + items: + $ref: '#/components/schemas/GraphParameter' + + GraphParameter: + type: object + required: ["name"] + properties: + name: + type: string + + PrometheusApiBaseResponse: + type: object + required: [ status ] + properties: + status: + type: string + enum: [ "success", "error" ] + error: + type: string + errorType: + type: string + warnings: + type: array + items: + type: string + + PrometheusApiQueryResponse: + type: object + allOf: + - $ref: "#/components/schemas/PrometheusApiBaseResponse" + properties: + data: + $ref: '#/components/schemas/PrometheusApiQueryResponseData' + + PrometheusApiQueryResponseData: + type: object + required: [ "resultType" ] + properties: + resultType: + type: string + enum: [ "matrix", "vector", "scalar", "string"] + result: + type: array + items: { } + description: | + Depending on resultType format can vary: + "vector": + [ + { + "metric": { "": "", ... }, + "value": [ , "" ], + "histogram": [ , ] + }, ... + ] + "matrix": + [ + { + "metric": { "": "", ... }, + "values": [ [ , "" ], ... ], + "histograms": [ [ , ], ... ] + }, ... + ] + "scalar": + [ , "" ] + "string": + [ , "" ] + ApplicationConfigValidation: type: object properties: @@ -4149,6 +4315,8 @@ components: $ref: '#/components/schemas/ApplicationPropertyValidation' ksqldb: $ref: '#/components/schemas/ApplicationPropertyValidation' + prometheusStorage: + $ref: '#/components/schemas/ApplicationPropertyValidation' ApplicationConfig: type: object @@ -4352,6 +4520,31 @@ components: type: string keystorePassword: type: string + prometheusExpose: + type: boolean + store: + type: object + properties: + prometheus: + type: object + properties: + url: + type: string + remoteWrite: + type: boolean + pushGatewayUrl: + type: string + pushGatewayUsername: + type: string + pushGatewayPassword: + type: string + pushGatewayJobName: + type: string + kafka: + type: object + properties: + topic: + type: string properties: type: object additionalProperties: true diff --git a/contract/src/main/resources/swagger/prometheus-query-api.yaml b/contract/src/main/resources/swagger/prometheus-query-api.yaml new file mode 100644 index 000000000..9ba26a460 --- /dev/null +++ b/contract/src/main/resources/swagger/prometheus-query-api.yaml @@ -0,0 +1,363 @@ +openapi: 3.0.1 +info: + title: | + Prometheus query HTTP API + version: 0.1.0 + contact: { } + +tags: + - name: /promclient +servers: + - url: /localhost + + +paths: + /api/v1/label/{label_name}/values: + get: + tags: + - PrometheusClient + summary: Returns label values + description: "returns a list of label values for a provided label name" + operationId: getLabelValues + parameters: + - name: label_name + in: path + required: true + schema: + type: string + - name: start + in: query + description: Start timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: End timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: match[] + in: query + description: Repeated series selector argument that selects the series from which to read the label values. + schema: + type: string + format: series_selector + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/LabelValuesResponse' + + /api/v1/labels: + get: + tags: + - PrometheusClient + summary: Returns label names + description: returns a list of label names + operationId: getLabelNames + parameters: + - name: start + in: query + description: | + Start timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: | + End timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: match[] + in: query + description: Repeated series selector argument that selects the series from which to read the label values. Optional. + schema: + type: string + format: series_selector + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/LabelNamesResponse' + + /api/v1/metadata: + get: + tags: + - PrometheusClient + summary: Returns metric metadata + description: returns a list of label names + operationId: getMetricMetadata + parameters: + - name: limit + in: query + description: Maximum number of metrics to return. + required: true + schema: + type: integer + - name: metric + in: query + description: A metric name to filter metadata for. All metric metadata is retrieved if left empty. + schema: + type: string + responses: + 200: + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/MetadataResponse' + 201: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/MetadataResponse' + + /api/v1/query: + get: + tags: + - PrometheusClient + summary: Evaluates instant query + description: | + Evaluates an instant query at a single point in time + operationId: query + parameters: + - name: query + in: query + description: | + Prometheus expression query string. + required: true + schema: + type: string + - name: time + in: query + description: | + Evaluation timestamp. Optional. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: timeout + in: query + description: | + Evaluation timeout. Optional. + schema: + type: string + format: duration + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/QueryResponse' + + + /api/v1/query_range: + get: + tags: + - PrometheusClient + summary: Evaluates query over range of time. + description: Evaluates an expression query over a range of time + operationId: queryRange + parameters: + - name: query + in: query + description: Prometheus expression query string. + required: true + schema: + type: string + - name: start + in: query + description: Start timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: End timestamp. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: step + in: query + description: | + Query resolution step width in ```duration``` format or float number of seconds. + schema: + type: string + format: duration | float + - name: timeout + in: query + description: | + Evaluation timeout. Optional. Defaults to and is capped by the value of the ```-query.timeout``` flag. + schema: + type: string + format: duration + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: "#/components/schemas/QueryResponse" + + + /api/v1/series: + get: + tags: + - PrometheusClient + summary: Returns time series + operationId: getSeries + parameters: + - name: start + in: query + description: | + Start timestamp. Optional. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: end + in: query + description: | + End timestamp. Optional. + schema: + type: string + format: rfc3339 | unix_timestamp + - name: match[] + in: query + description: | + Repeated series selector argument that selects the series to return. At least one ```match[]``` argument must be provided. + required: true + schema: + type: string + format: series_selector + responses: + 200: + description: | + Success + content: + application/json: + schema: + $ref: '#/components/schemas/SeriesResponse' + +components: + schemas: + BaseResponse: + type: object + required: [ status ] + properties: + status: + type: string + enum: [ "success", "error" ] + error: + type: string + errorType: + type: string + warnings: + type: array + items: + type: string + + QueryResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + $ref: '#/components/schemas/QueryResponseData' + + QueryResponseData: + type: object + required: [ "resultType" ] + properties: + resultType: + type: string + enum: [ "matrix", "vector", "scalar", "string"] + result: + type: array + items: { } + description: | + Depending on resultType format can vary: + "vector": + [ + { + "metric": { "": "", ... }, + "value": [ , "" ], + "histogram": [ , ] + }, ... + ] + "matrix": + [ + { + "metric": { "": "", ... }, + "values": [ [ , "" ], ... ], + "histograms": [ [ , ], ... ] + }, ... + ] + "scalar": + [ , "" ] + "string": + [ , "" ] + + SeriesResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: array + description: a list of objects that contain the label name/value pairs which + identify each series + items: + type: object + properties: + __name__: + type: string + job: + type: string + instance: + type: string + + MetadataResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: object + additionalProperties: + type: array + items: + type: object + additionalProperties: true + + LabelValuesResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: array + description: a list of string label values + items: + type: string + + LabelNamesResponse: + type: object + allOf: + - $ref: "#/components/schemas/BaseResponse" + properties: + data: + type: array + description: a list of string label names + items: + type: string diff --git a/documentation/compose/kafbat-ui.yaml b/documentation/compose/kafbat-ui.yaml index afbe0d26c..919f36547 100644 --- a/documentation/compose/kafbat-ui.yaml +++ b/documentation/compose/kafbat-ui.yaml @@ -24,6 +24,9 @@ services: KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS: kafka1:29092 KAFKA_CLUSTERS_1_METRICS_PORT: 9998 KAFKA_CLUSTERS_1_SCHEMAREGISTRY: http://schemaregistry1:8085 + KAFKA_CLUSTERS_0_METRICS_STORE_PROMETHEUS_URL: "http://prometheus:9090" + KAFKA_CLUSTERS_0_METRICS_STORE_PROMETHEUS_REMOTEWRITE: 'true' + KAFKA_CLUSTERS_0_METRICS_STORE_KAFKA_TOPIC: "kafka_metrics" DYNAMIC_CONFIG_ENABLED: 'true' kafka0: @@ -135,6 +138,16 @@ services: CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0 CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + prometheus: + image: prom/prometheus:latest + hostname: prometheus + container_name: prometheus + ports: + - 9090:9090 + volumes: + - ./scripts:/etc/prometheus + command: --web.enable-remote-write-receiver --config.file=/etc/prometheus/prometheus.yaml + kafka-init-topics: image: confluentinc/cp-kafka:7.8.0 volumes: diff --git a/documentation/compose/scripts/prometheus.yaml b/documentation/compose/scripts/prometheus.yaml new file mode 100644 index 000000000..457de126c --- /dev/null +++ b/documentation/compose/scripts/prometheus.yaml @@ -0,0 +1,14 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + +rule_files: + - alert.yml + +scrape_configs: + - job_name: services + metrics_path: /metrics + static_configs: + - targets: + - 'prometheus:9090' +# - 'kafka-ui:8080' diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ccda81869..6b0accaa6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,7 +28,7 @@ testcontainers = '1.20.6' swagger-integration-jakarta = '2.2.28' jakarta-annotation-api = '2.1.1' jackson-databind-nullable = '0.2.6' -antlr = '4.12.0' +antlr = '4.13.2' json-schema-validator = '2.2.14' checkstyle = '10.3.1' @@ -38,6 +38,9 @@ testng = '7.10.0' bonigarcia-webdrivermanager = '5.9.3' aspectj = '1.9.21' +prometheus = '1.3.6' +protobuf = '4.30.2' + # CVE fixes netty = '4.1.119.Final' @@ -52,6 +55,8 @@ node-gradle = { id = 'com.github.node-gradle.node', version = '7.1.0' } #jib = { id = 'com.google.cloud.tools.jib', version = '3.4.4' } docker-remote-api = { id = 'com.bmuschko.docker-remote-api', version = '9.4.0' } sonarqube = { id = "org.sonarqube", version = "6.0.1.5171" } +protobuf = {id = "com.google.protobuf", version = "0.9.5"} + [libraries] spring-starter-actuator = { module = 'org.springframework.boot:spring-boot-starter-actuator', version.ref = 'spring-boot' } @@ -137,3 +142,11 @@ bouncycastle-bcpkix = { module = 'org.bouncycastle:bcpkix-jdk18on', version = '1 # Google Managed Service for Apache Kafka support google-managed-kafka-login-handler = {module = 'com.google.cloud.hosted.kafka:managed-kafka-auth-login-handler', version = '1.0.5'} google-oauth-client = { module = 'com.google.oauth-client:google-oauth-client', version = '1.39.0' } + +prometheus-metrics-core = {module = 'io.prometheus:prometheus-metrics-core', version.ref = 'prometheus'} +prometheus-metrics-textformats = { module = 'io.prometheus:prometheus-metrics-exposition-textformats', version.ref = 'prometheus'} +prometheus-metrics-exporter-pushgateway = { module = 'io.prometheus:prometheus-metrics-exporter-pushgateway', version.ref = 'prometheus'} + +snappy = {module = 'org.xerial.snappy:snappy-java', version = '1.1.10.7'} + +protobuf = {module = "com.google.protobuf:protobuf-java", version.ref = "protobuf"} From f8026c43b1057d1de0d09cd62c41fa5bef501568 Mon Sep 17 00:00:00 2001 From: German Osin Date: Wed, 14 May 2025 10:44:36 +0200 Subject: [PATCH 02/29] second phase --- .../ui/service/metrics/scrape/PrometheusEndpointParserTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java index 5353089cf..7c063a94b 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java @@ -62,6 +62,8 @@ void parsesMetricsFromPrometheusEndpointOutput() { """; List parsed = parse(expose.lines()); + + assertThat(parsed).anyMatch(p -> isTheSameMetric(p, new CounterSnapshot( new MetricMetadata("http_requests", "The total number of HTTP requests."), From 08420177f92cadfb250e15d1c7273939d3283e52 Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 09:39:41 +0500 Subject: [PATCH 03/29] Refactoring + Prometheus metrics parser --- .../kafbat/ui/config/ClustersProperties.java | 6 - .../java/io/kafbat/ui/model/KafkaCluster.java | 2 +- .../ui/model/MetricsScrapeProperties.java | 3 +- .../java/io/kafbat/ui/model/Statistics.java | 17 +- .../ui/serdes/builtin/ProtobufFileSerde.java | 4 + .../ui/service/KafkaClusterFactory.java | 5 +- .../ui/service/ReactiveAdminClient.java | 4 + .../io/kafbat/ui/service/StatisticsCache.java | 13 +- .../kafbat/ui/service/StatisticsService.java | 58 +-- .../kafbat/ui/service/metrics/RawMetric.java | 32 +- .../ui/service/metrics/SummarizedMetrics.java | 19 +- .../metrics/prometheus/PrometheusExpose.java | 4 +- .../metrics/scrape/BrokerMetricsScraper.java | 11 + .../metrics/scrape/IoRatesMetricsScanner.java | 30 +- .../metrics/scrape/MetricsScrapping.java | 56 +-- .../scrape/inferred/InferredMetrics.java | 1 + .../scrape/jmx/JmxMetricsRetriever.java | 2 +- .../metrics/scrape/jmx/JmxMetricsScraper.java | 6 +- .../prometheus/PrometheusEndpointParser.java | 375 ----------------- .../PrometheusMetricsRetriever.java | 2 +- .../scrape/prometheus/PrometheusScraper.java | 4 +- .../PrometheusTextFormatParser.java | 394 ++++++++++++++++++ .../ui/service/metrics/sink/KafkaSink.java | 78 ---- .../ui/service/metrics/sink/MetricsSink.java | 22 +- .../sink/PrometheusPushGatewaySink.java | 16 +- .../sink/PrometheusRemoteWriteSink.java | 80 ---- .../java/io/kafbat/ui/util/MetricsUtils.java | 22 +- .../scrape/IoRatesMetricsScannerTest.java | 4 +- .../scrape/PrometheusEndpointParserTest.java | 200 --------- .../PrometheusTextFormatParserTest.java | 142 +++++++ 30 files changed, 729 insertions(+), 883 deletions(-) create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/BrokerMetricsScraper.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java create mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java delete mode 100644 api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java delete mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java diff --git a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java index 1a5e2c6e9..7aff0af7b 100644 --- a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java +++ b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java @@ -99,12 +99,6 @@ public static class MetricsConfig { @Data public static class MetricsStorage { PrometheusStorage prometheus; - KafkaMetricsStorage kafka; - } - - @Data - public static class KafkaMetricsStorage { - String topic; } @Data diff --git a/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java b/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java index 5281d07e8..227364eb2 100644 --- a/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java +++ b/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java @@ -32,9 +32,9 @@ public class KafkaCluster { private final boolean exposeMetricsViaPrometheusEndpoint; private final DataMasking masking; private final PollingSettings pollingSettings; + private final MetricsScrapping metricsScrapping; private final ReactiveFailover schemaRegistryClient; private final Map> connectsClients; private final ReactiveFailover ksqlClient; - private final MetricsScrapping metricsScrapping; private final ReactiveFailover prometheusStorageClient; } diff --git a/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java b/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java index 41e0fc6fd..806a2e192 100644 --- a/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java +++ b/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java @@ -7,11 +7,12 @@ import jakarta.annotation.Nullable; import java.util.Objects; import java.util.Optional; +import lombok.AccessLevel; import lombok.Builder; import lombok.Value; @Value -@Builder +@Builder(access = AccessLevel.PRIVATE) public class MetricsScrapeProperties { public static final String JMX_METRICS_TYPE = "JMX"; public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; diff --git a/api/src/main/java/io/kafbat/ui/model/Statistics.java b/api/src/main/java/io/kafbat/ui/model/Statistics.java index 5b6cd8f9f..6caba634a 100644 --- a/api/src/main/java/io/kafbat/ui/model/Statistics.java +++ b/api/src/main/java/io/kafbat/ui/model/Statistics.java @@ -3,7 +3,7 @@ import io.kafbat.ui.service.ReactiveAdminClient; import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; import java.util.List; -import java.util.Set; +import java.util.function.UnaryOperator; import java.util.stream.Stream; import lombok.Builder; import lombok.Value; @@ -25,14 +25,25 @@ public static Statistics empty() { .status(ServerStatusDTO.OFFLINE) .version("Unknown") .features(List.of()) - .clusterDescription( - new ReactiveAdminClient.ClusterDescription(null, null, List.of(), Set.of())) + .clusterDescription(ReactiveAdminClient.ClusterDescription.empty()) .metrics(Metrics.empty()) .clusterState(ScrapedClusterState.empty()) .build(); } + public static Statistics statsUpdateError(Throwable th) { + return empty().toBuilder().status(ServerStatusDTO.OFFLINE).lastKafkaException(th).build(); + } + + public static Statistics initializing() { + return empty().toBuilder().status(ServerStatusDTO.INITIALIZING).build(); + } + public Stream topicDescriptions() { return clusterState.getTopicStates().values().stream().map(ScrapedClusterState.TopicState::description); } + + public Statistics withClusterState(UnaryOperator stateUpdate) { + return toBuilder().clusterState(stateUpdate.apply(clusterState)).build(); + } } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java index 618c711b1..56655eda0 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java @@ -279,6 +279,10 @@ private static Map populateDescriptors(Map loadSchemas(Optional> protobufFiles, Optional protobufFilesDir) { + if (true) { + return Map.of(); + } + if (protobufFilesDir.isPresent()) { if (protobufFiles.isPresent()) { log.warn("protobufFiles properties will be ignored, since protobufFilesDir provided"); diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index 489f316f0..1d0a1f5f9 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -17,6 +17,7 @@ import io.kafbat.ui.prometheus.api.PrometheusClientApi; import io.kafbat.ui.service.ksql.KsqlApiClient; import io.kafbat.ui.service.masking.DataMasking; +import io.kafbat.ui.service.metrics.scrape.MetricsScrapping; import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; import io.kafbat.ui.sr.ApiClient; import io.kafbat.ui.sr.api.KafkaSrClientApi; @@ -51,7 +52,8 @@ public class KafkaClusterFactory { private final Duration responseTimeout; private final JmxMetricsRetriever jmxMetricsRetriever; - public KafkaClusterFactory(WebclientProperties webclientProperties, JmxMetricsRetriever jmxMetricsRetriever) { + public KafkaClusterFactory(WebclientProperties webclientProperties, + JmxMetricsRetriever jmxMetricsRetriever) { this.webClientMaxBuffSize = Optional.ofNullable(webclientProperties.getMaxInMemoryBufferSize()) .map(DataSize::parse) .orElse(DEFAULT_WEBCLIENT_BUFFER); @@ -74,6 +76,7 @@ public KafkaCluster create(ClustersProperties properties, builder.exposeMetricsViaPrometheusEndpoint(exposeMetricsViaPrometheusEndpoint(clusterProperties)); builder.masking(DataMasking.create(clusterProperties.getMasking())); builder.pollingSettings(PollingSettings.create(clusterProperties, properties)); + builder.metricsScrapping(MetricsScrapping.create(clusterProperties, jmxMetricsRetriever)); if (schemaRegistryConfigured(clusterProperties)) { builder.schemaRegistryClient(schemaRegistryClient(clusterProperties)); diff --git a/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java b/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java index 82f8658b2..c9d08708d 100644 --- a/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java +++ b/api/src/main/java/io/kafbat/ui/service/ReactiveAdminClient.java @@ -133,6 +133,10 @@ public static class ClusterDescription { Collection nodes; @Nullable // null, if ACL is disabled Set authorizedOperations; + + public static ClusterDescription empty() { + return new ReactiveAdminClient.ClusterDescription(null, null, List.of(), Set.of()); + } } @Builder diff --git a/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java b/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java index 7b37e8400..04306e9e8 100644 --- a/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java +++ b/api/src/main/java/io/kafbat/ui/service/StatisticsCache.java @@ -2,14 +2,11 @@ import io.kafbat.ui.model.InternalPartitionsOffsets; import io.kafbat.ui.model.KafkaCluster; -import io.kafbat.ui.model.ServerStatusDTO; import io.kafbat.ui.model.Statistics; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.springframework.stereotype.Component; @@ -20,7 +17,7 @@ public class StatisticsCache { private final Map cache = new ConcurrentHashMap<>(); public StatisticsCache(ClustersStorage clustersStorage) { - var initializing = Statistics.empty().toBuilder().status(ServerStatusDTO.INITIALIZING).build(); + Statistics initializing = Statistics.initializing(); clustersStorage.getKafkaClusters().forEach(c -> cache.put(c.getName(), initializing)); } @@ -35,9 +32,7 @@ public synchronized void update(KafkaCluster c, var stats = get(c); replace( c, - stats.toBuilder() - .clusterState(stats.getClusterState().updateTopics(descriptions, configs, partitionsOffsets)) - .build() + stats.withClusterState(s -> s.updateTopics(descriptions, configs, partitionsOffsets)) ); } @@ -45,9 +40,7 @@ public synchronized void onTopicDelete(KafkaCluster c, String topic) { var stats = get(c); replace( c, - stats.toBuilder() - .clusterState(stats.getClusterState().topicDeleted(topic)) - .build() + stats.withClusterState(s -> s.topicDeleted(topic)) ); } diff --git a/api/src/main/java/io/kafbat/ui/service/StatisticsService.java b/api/src/main/java/io/kafbat/ui/service/StatisticsService.java index 5946792f0..e874a5f9a 100644 --- a/api/src/main/java/io/kafbat/ui/service/StatisticsService.java +++ b/api/src/main/java/io/kafbat/ui/service/StatisticsService.java @@ -2,11 +2,13 @@ import static io.kafbat.ui.service.ReactiveAdminClient.ClusterDescription; +import io.kafbat.ui.model.ClusterFeature; import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.model.Metrics; import io.kafbat.ui.model.ServerStatusDTO; import io.kafbat.ui.model.Statistics; import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; +import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -27,38 +29,46 @@ public Mono updateCache(KafkaCluster c) { private Mono getStatistics(KafkaCluster cluster) { return adminClientService.get(cluster).flatMap(ac -> - ac.describeCluster() - .flatMap(description -> - ac.updateInternalStats(description.getController()) - .then( - Mono.zip( - featureService.getAvailableFeatures(ac, cluster, description), - loadClusterState(description, ac) - ).flatMap(featuresAndState -> - scrapeMetrics(cluster, featuresAndState.getT2(), description) - .map(metrics -> - Statistics.builder() - .status(ServerStatusDTO.ONLINE) - .clusterDescription(description) - .version(ac.getVersion()) - .metrics(metrics) - .features(featuresAndState.getT1()) - .clusterState(featuresAndState.getT2()) - .build()))))) - .doOnError(e -> - log.error("Failed to collect cluster {} info", cluster.getName(), e)) - .onErrorResume( - e -> Mono.just(Statistics.empty().toBuilder().lastKafkaException(e).build())); + ac.describeCluster() + .flatMap(description -> + ac.updateInternalStats(description.getController()) + .then( + Mono.zip( + featureService.getAvailableFeatures(ac, cluster, description), + loadClusterState(description, ac) + ).flatMap(t -> + scrapeMetrics(cluster, t.getT2(), description) + .map(metrics -> createStats(description, t.getT1(), t.getT2(), metrics, ac))))) + .doOnError(e -> + log.error("Failed to collect cluster {} info", cluster.getName(), e)) + .onErrorResume(t -> Mono.just(Statistics.statsUpdateError(t)))); } - private Mono loadClusterState(ClusterDescription clusterDescription, ReactiveAdminClient ac) { + private Statistics createStats(ClusterDescription description, + List features, + ScrapedClusterState scrapedClusterState, + Metrics metrics, + ReactiveAdminClient ac) { + return Statistics.builder() + .status(ServerStatusDTO.ONLINE) + .clusterDescription(description) + .version(ac.getVersion()) + .metrics(metrics) + .features(features) + .clusterState(scrapedClusterState) + .build(); + } + + private Mono loadClusterState(ClusterDescription clusterDescription, + ReactiveAdminClient ac) { return ScrapedClusterState.scrape(clusterDescription, ac); } private Mono scrapeMetrics(KafkaCluster cluster, ScrapedClusterState clusterState, ClusterDescription clusterDescription) { - return cluster.getMetricsScrapping().scrape(clusterState, clusterDescription.getNodes()); + return cluster.getMetricsScrapping() + .scrape(clusterState, clusterDescription.getNodes()); } } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java b/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java index 28e3fb361..b9e721fad 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java @@ -1,13 +1,12 @@ package io.kafbat.ui.service.metrics; -import io.prometheus.metrics.model.snapshots.GaugeSnapshot; -import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.core.metrics.Gauge; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.math.BigDecimal; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -25,26 +24,25 @@ static RawMetric create(String name, Map labels, BigDecimal valu return new SimpleMetric(name, labels, value); } - static Stream groupIntoMfs(Collection rawMetrics) { - Map map = new LinkedHashMap<>(); + static Stream groupIntoSnapshot(Collection rawMetrics) { + Map map = new LinkedHashMap<>(); for (RawMetric m : rawMetrics) { - var gauge = map.computeIfAbsent(m.name(), - (n) -> GaugeSnapshot.builder() + var lbls = m.labels().keySet().toArray(String[]::new); + var lblVals = Arrays.stream(lbls).map(l -> m.labels().get(l)).toArray(String[]::new); + var gauge = map.computeIfAbsent( + m.name(), + n -> Gauge.builder() .name(m.name()) .help(m.name()) + .labelNames(lbls) + .build() ); - - List lbls = m.labels().keySet().stream().toList(); - List lblVals = lbls.stream().map(l -> m.labels().get(l)).toList(); - - GaugeSnapshot.GaugeDataPointSnapshot point = GaugeSnapshot.GaugeDataPointSnapshot.builder() - .value(m.value().doubleValue()) - .labels(Labels.of(lbls, lblVals)).build(); - gauge.dataPoint(point); + gauge.labelValues(lblVals).set(m.value().doubleValue()); } - return map.values().stream().map(GaugeSnapshot.Builder::build); + return map.values().stream().map(Gauge::collect); } - record SimpleMetric(String name, Map labels, BigDecimal value) implements RawMetric { } + record SimpleMetric(String name, Map labels, BigDecimal value) implements RawMetric { + } } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java index dd553cc88..265ff0e7a 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -45,28 +45,29 @@ public Stream asStream() { //returns Optional.empty if merging not supported for metric type @SuppressWarnings("unchecked") - private static Optional summarizeMetricSnapshot(Optional mfs1opt, - Optional mfs2opt) { + private static Optional summarizeMetricSnapshot(Optional snap1Opt, + Optional snap2Opt) { - if ((mfs1opt.isEmpty() || mfs2opt.isEmpty()) || !(mfs1opt.get().getClass().equals(mfs2opt.get().getClass()))) { + if ((snap1Opt.isEmpty() || snap2Opt.isEmpty()) || !(snap1Opt.get().getClass().equals(snap2Opt.get().getClass()))) { return Optional.empty(); } - var mfs1 = mfs1opt.get(); + var snap1 = snap1Opt.get(); - if (mfs1 instanceof GaugeSnapshot || mfs1 instanceof CounterSnapshot) { + //TODO: add unknown + if (snap1 instanceof GaugeSnapshot || snap1 instanceof CounterSnapshot) { BiFunction pointFactory; Function valueGetter; Function, MetricSnapshot> builder; - if (mfs1 instanceof CounterSnapshot) { + if (snap1 instanceof CounterSnapshot) { pointFactory = (l, v) -> CounterDataPointSnapshot.builder() .labels(l) .value(v) .build(); valueGetter = (dp) -> ((CounterDataPointSnapshot)dp).getValue(); builder = (dps) -> - new CounterSnapshot(mfs1.getMetadata(), (Collection)dps); + new CounterSnapshot(snap1.getMetadata(), (Collection)dps); } else { pointFactory = (l,v) -> GaugeDataPointSnapshot.builder() .labels(l) @@ -74,11 +75,11 @@ private static Optional summarizeMetricSnapshot(Optional ((GaugeDataPointSnapshot)dp).getValue(); builder = (dps) -> - new GaugeSnapshot(mfs1.getMetadata(), (Collection)dps); + new GaugeSnapshot(snap1.getMetadata(), (Collection)dps); } Collection points = - Stream.concat(mfs1.getDataPoints().stream(), mfs2opt.get().getDataPoints().stream()) + Stream.concat(snap1.getDataPoints().stream(), snap2Opt.get().getDataPoints().stream()) .collect( toMap( // merging samples with same labels diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java index d752a0297..f8d6f8820 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java @@ -8,6 +8,7 @@ import io.kafbat.ui.model.Metrics; import io.kafbat.ui.util.MetricsUtils; import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter; +import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.io.ByteArrayOutputStream; @@ -69,7 +70,7 @@ private static Stream extractBrokerMetricsWithLabel(Metrics metr @SneakyThrows public static ResponseEntity constructHttpsResponse(MetricSnapshots metrics) { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(false, false); + PrometheusTextFormatWriter writer = new PrometheusTextFormatWriter(false); writer.write(buffer, metrics); return ResponseEntity .ok() @@ -78,6 +79,7 @@ public static ResponseEntity constructHttpsResponse(MetricSnapshots metr } // copied from io.prometheus.client.exporter.common.TextFormat:writeEscapedLabelValue + //TODO: RM public static String escapedLabelValue(String s) { StringBuilder sb = new StringBuilder(s.length()); for (int i = 0; i < s.length(); i++) { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/BrokerMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/BrokerMetricsScraper.java new file mode 100644 index 000000000..aa28e22a2 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/BrokerMetricsScraper.java @@ -0,0 +1,11 @@ +package io.kafbat.ui.service.metrics.scrape; + +import java.util.Collection; +import org.apache.kafka.common.Node; +import reactor.core.publisher.Mono; + +public interface BrokerMetricsScraper { + + Mono scrape(Collection clusterNodes); + +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java index 13ff4ee00..ed1968104 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java @@ -4,14 +4,15 @@ import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; import io.kafbat.ui.model.Metrics; -import io.prometheus.metrics.model.snapshots.DataPointSnapshot; import io.prometheus.metrics.model.snapshots.GaugeSnapshot; import io.prometheus.metrics.model.snapshots.Labels; import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot; import java.math.BigDecimal; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; // Scans external jmx/prometheus metric and tries to infer io rates class IoRatesMetricsScanner { @@ -31,10 +32,15 @@ class IoRatesMetricsScanner { for (MetricSnapshot metric : metrics) { String name = metric.getMetadata().getName(); if (metric instanceof GaugeSnapshot gauge) { - for (GaugeSnapshot.GaugeDataPointSnapshot dataPoint : gauge.getDataPoints()) { - updateBrokerIOrates(nodeId, name, dataPoint); - updateTopicsIOrates(name, dataPoint); - } + gauge.getDataPoints().forEach(dp -> { + updateBrokerIOrates(nodeId, name, dp.getLabels(), dp.getValue()); + updateTopicsIOrates(name, dp.getLabels(), dp.getValue()); + }); + } else if (metric instanceof UnknownSnapshot unknown) { + unknown.getDataPoints().forEach(dp -> { + updateBrokerIOrates(nodeId, name, dp.getLabels(), dp.getValue()); + updateTopicsIOrates(name, dp.getLabels(), dp.getValue()); + }); } } } @@ -49,26 +55,24 @@ Metrics.IoRates get() { .build(); } - private void updateBrokerIOrates(int nodeId, String name, GaugeSnapshot.GaugeDataPointSnapshot point) { - Labels labels = point.getLabels(); + private void updateBrokerIOrates(int nodeId, String name, Labels labels, double value) { if (!brokerBytesInFifteenMinuteRate.containsKey(nodeId) && labels.size() == 1 && "BytesInPerSec".equalsIgnoreCase(labels.getValue(0)) && containsIgnoreCase(name, "BrokerTopicMetrics") && endsWithIgnoreCase(name, "FifteenMinuteRate")) { - brokerBytesInFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(point.getValue())); + brokerBytesInFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(value)); } if (!brokerBytesOutFifteenMinuteRate.containsKey(nodeId) && labels.size() == 1 && "BytesOutPerSec".equalsIgnoreCase(labels.getValue(0)) && containsIgnoreCase(name, "BrokerTopicMetrics") && endsWithIgnoreCase(name, "FifteenMinuteRate")) { - brokerBytesOutFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(point.getValue())); + brokerBytesOutFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(value)); } } - private void updateTopicsIOrates(String name, GaugeSnapshot.GaugeDataPointSnapshot point) { - Labels labels = point.getLabels(); + private void updateTopicsIOrates(String name, Labels labels, double value) { if (labels.contains("topic") && containsIgnoreCase(name, "BrokerTopicMetrics") && endsWithIgnoreCase(name, "FifteenMinuteRate")) { @@ -76,10 +80,10 @@ && endsWithIgnoreCase(name, "FifteenMinuteRate")) { if (labels.contains("name")) { var nameLblVal = labels.get("name"); if ("BytesInPerSec".equalsIgnoreCase(nameLblVal)) { - BigDecimal val = BigDecimal.valueOf(point.getValue()); + BigDecimal val = BigDecimal.valueOf(value); bytesInFifteenMinuteRate.merge(topic, val, BigDecimal::add); } else if ("BytesOutPerSec".equalsIgnoreCase(nameLblVal)) { - BigDecimal val = BigDecimal.valueOf(point.getValue()); + BigDecimal val = BigDecimal.valueOf(value); bytesOutFifteenMinuteRate.merge(topic, val, BigDecimal::add); } } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java index 70da53307..0271a75db 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java @@ -4,6 +4,7 @@ import static io.kafbat.ui.model.MetricsScrapeProperties.JMX_METRICS_TYPE; import static io.kafbat.ui.model.MetricsScrapeProperties.PROMETHEUS_METRICS_TYPE; +import io.kafbat.ui.config.ClustersProperties.MetricsConfig; import io.kafbat.ui.model.Metrics; import io.kafbat.ui.model.MetricsScrapeProperties; import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; @@ -17,55 +18,54 @@ import jakarta.annotation.Nullable; import java.util.Collection; import java.util.stream.Stream; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.Node; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Slf4j -@RequiredArgsConstructor +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class MetricsScrapping { private final String clusterName; private final MetricsSink sink; private final InferredMetricsScraper inferredMetricsScraper; @Nullable - private final JmxMetricsScraper jmxMetricsScraper; - @Nullable - private final PrometheusScraper prometheusScraper; + private final BrokerMetricsScraper brokerMetricsScraper; public static MetricsScrapping create(Cluster cluster, JmxMetricsRetriever jmxMetricsRetriever) { - JmxMetricsScraper jmxMetricsScraper = null; - PrometheusScraper prometheusScraper = null; - var metrics = cluster.getMetrics(); + BrokerMetricsScraper scraper = null; + MetricsConfig metricsConfig = cluster.getMetrics(); if (cluster.getMetrics() != null) { var scrapeProperties = MetricsScrapeProperties.create(cluster); - if (metrics.getType().equalsIgnoreCase(JMX_METRICS_TYPE) && metrics.getPort() != null) { - jmxMetricsScraper = new JmxMetricsScraper(scrapeProperties, jmxMetricsRetriever); - } else if (metrics.getType().equalsIgnoreCase(PROMETHEUS_METRICS_TYPE)) { - prometheusScraper = new PrometheusScraper(scrapeProperties); + if (metricsConfig.getType().equalsIgnoreCase(JMX_METRICS_TYPE) && metricsConfig.getPort() != null) { + scraper = new JmxMetricsScraper(scrapeProperties, jmxMetricsRetriever); + } else if (metricsConfig.getType().equalsIgnoreCase(PROMETHEUS_METRICS_TYPE)) { + scraper = new PrometheusScraper(scrapeProperties); } } return new MetricsScrapping( cluster.getName(), MetricsSink.create(cluster), new InferredMetricsScraper(), - jmxMetricsScraper, - prometheusScraper + scraper ); } public Mono scrape(ScrapedClusterState clusterState, Collection nodes) { Mono inferred = inferredMetricsScraper.scrape(clusterState); - Mono external = scrapeExternal(nodes); + Mono brokerMetrics = scrapeBrokers(nodes); return inferred.zipWith( - external, - (inf, ext) -> Metrics.builder() - .inferredMetrics(inf) - .ioRates(ext.ioRates()) - .perBrokerScrapedMetrics(ext.perBrokerMetrics()) - .build() + brokerMetrics, + (inf, ext) -> + Metrics.builder() + .inferredMetrics(inf) + .ioRates(ext.ioRates()) + .perBrokerScrapedMetrics(ext.perBrokerMetrics()) + .build() ).doOnNext(this::sendMetricsToSink); } @@ -75,16 +75,16 @@ private void sendMetricsToSink(Metrics metrics) { .subscribe(); } - private Stream prepareMetricsForSending(Metrics metrics) { - return PrometheusExpose.prepareMetricsForGlobalExpose(clusterName, metrics); + private Flux prepareMetricsForSending(Metrics metrics) { + //need to be "cold" because sinks can resubscribe multiple times + return Flux.defer(() -> + Flux.fromStream( + PrometheusExpose.prepareMetricsForGlobalExpose(clusterName, metrics))); } - private Mono scrapeExternal(Collection nodes) { - if (jmxMetricsScraper != null) { - return jmxMetricsScraper.scrape(nodes); - } - if (prometheusScraper != null) { - return prometheusScraper.scrape(nodes); + private Mono scrapeBrokers(Collection nodes) { + if (brokerMetricsScraper != null) { + return brokerMetricsScraper.scrape(nodes); } return Mono.just(PerBrokerScrapedMetrics.empty()); } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java index 7486b5480..6d68ff1b4 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetrics.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Stream; +//metrics that we inferred from cluster state (always present for any setup) public class InferredMetrics { private final List metrics; diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java index 62171ccd3..ef8614be9 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java @@ -23,7 +23,7 @@ import reactor.core.scheduler.Schedulers; -@Component //need to be a component, since +@Component //need to be a component, since it is closeable @Slf4j public class JmxMetricsRetriever implements Closeable { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java index ab024582a..e334325e7 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsScraper.java @@ -2,6 +2,7 @@ import io.kafbat.ui.model.MetricsScrapeProperties; import io.kafbat.ui.service.metrics.RawMetric; +import io.kafbat.ui.service.metrics.scrape.BrokerMetricsScraper; import io.kafbat.ui.service.metrics.scrape.PerBrokerScrapedMetrics; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.util.Collection; @@ -12,7 +13,7 @@ import reactor.core.publisher.Mono; import reactor.util.function.Tuples; -public class JmxMetricsScraper { +public class JmxMetricsScraper implements BrokerMetricsScraper { private final JmxMetricsRetriever jmxMetricsRetriever; private final MetricsScrapeProperties scrapeProperties; @@ -23,12 +24,13 @@ public JmxMetricsScraper(MetricsScrapeProperties scrapeProperties, this.jmxMetricsRetriever = jmxMetricsRetriever; } + @Override public Mono scrape(Collection nodes) { Mono>> collected = Flux.fromIterable(nodes) .flatMap(n -> jmxMetricsRetriever.retrieveFromNode(scrapeProperties, n).map(metrics -> Tuples.of(n, metrics))) .collectMap( t -> t.getT1().id(), - t -> RawMetric.groupIntoMfs(t.getT2()).toList() + t -> RawMetric.groupIntoSnapshot(t.getT2()).toList() ); return collected.map(PerBrokerScrapedMetrics::new); } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java deleted file mode 100644 index 12ab39267..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusEndpointParser.java +++ /dev/null @@ -1,375 +0,0 @@ -package io.kafbat.ui.service.metrics.scrape.prometheus; - -import com.google.common.base.Enums; -import io.prometheus.metrics.model.registry.PrometheusRegistry; -import io.prometheus.metrics.model.snapshots.CounterSnapshot; -import io.prometheus.metrics.model.snapshots.GaugeSnapshot; -import io.prometheus.metrics.model.snapshots.HistogramSnapshot; -import io.prometheus.metrics.model.snapshots.InfoSnapshot; -import io.prometheus.metrics.model.snapshots.Labels; -import io.prometheus.metrics.model.snapshots.MetricMetadata; -import io.prometheus.metrics.model.snapshots.MetricSnapshot; -import io.prometheus.metrics.model.snapshots.PrometheusNaming; -import io.prometheus.metrics.model.snapshots.SummarySnapshot; -import io.prometheus.metrics.model.snapshots.Unit; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Stream; - -public class PrometheusEndpointParser { - - enum Type { - GAUGE, - COUNTER, - HISTOGRAM, - SUMMARY, - INFO - } - - record Sample(String name, Double value, Labels labels) {} - - private static final Type DEFAULT_TYPE = Type.GAUGE; - - private PrometheusEndpointParser() { - } - - private static class ParserContext { - final List registered = new ArrayList<>(); - - String name; - String help; - Type type; - String unit; - Set allowedNames = new HashSet<>(); - List samples = new ArrayList<>(); - - void registerAndReset() { - if (!samples.isEmpty()) { - MetricMetadata metadata = new MetricMetadata( - PrometheusNaming.sanitizeMetricName(name), - Optional.ofNullable(help).orElse(name), - Optional.ofNullable(unit).map(Unit::new).orElse(null) - ); - - registered.add( switch (type) { - case GAUGE -> new GaugeSnapshot(metadata, convertGauges(samples)); - case COUNTER -> new CounterSnapshot(metadata, convertCounters(samples)); - case HISTOGRAM -> new HistogramSnapshot(metadata, convertHistograms(samples)); - case SUMMARY -> new SummarySnapshot(metadata, convertSummary(samples)); - case INFO -> new InfoSnapshot(metadata, convertInfo(samples)); - } ); - } - //resetting state: - name = null; - help = null; - type = null; - unit = null; - allowedNames.clear(); - samples.clear(); - } - - private Collection convertInfo(List samples) { - // TODO: Implement this - return List.of(); - } - - private Collection convertSummary(List samples) { - // TODO: Implement this - return List.of(); - } - - private Collection convertHistograms(List samples) { - // TODO: Implement this - return List.of(); - } - - private Collection convertCounters(List samples) { - return samples.stream().map(s -> new CounterSnapshot.CounterDataPointSnapshot( - s.value, s.labels, null, 0 - )).toList(); - } - - private Collection convertGauges(List samples) { - return samples.stream().map(s -> new GaugeSnapshot.GaugeDataPointSnapshot( - s.value, s.labels, null - )).toList(); - } - - List getRegistered() { - registerAndReset(); // last in progress metric should be registered - return registered; - } - } - - // general logic taken from https://github.com/prometheus/client_python/blob/master/prometheus_client/parser.py - public static List parse(Stream lines) { - PrometheusRegistry registry = new PrometheusRegistry(); - - ParserContext context = new ParserContext(); - lines.map(String::trim) - .filter(s -> !s.isBlank()) - .forEach(line -> { - if (line.charAt(0) == '#') { - String[] parts = line.split("[ \t]+", 4); - if (parts.length >= 3) { - switch (parts[1]) { - case "HELP" -> processHelp(context, parts); - case "TYPE" -> processType(context, parts); - case "UNIT" -> processUnit(context, parts); - default -> { /* probably a comment */ } - } - } - } else { - processSample(context, line); - } - }); - return context.getRegistered(); - } - - private static void processUnit(ParserContext context, String[] parts) { - if (!parts[2].equals(context.name)) { - // starting new metric family - need to register (if possible) prev one - context.registerAndReset(); - context.name = parts[2]; - context.type = DEFAULT_TYPE; - context.allowedNames.add(context.name); - } - if (parts.length == 4) { - context.unit = parts[3]; - } - } - - private static void processHelp(ParserContext context, String[] parts) { - if (!parts[2].equals(context.name)) { - // starting new metric family - need to register (if possible) prev one - context.registerAndReset(); - context.name = parts[2]; - context.type = DEFAULT_TYPE; - context.allowedNames.add(context.name); - } - if (parts.length == 4) { - context.help = unescapeHelp(parts[3]); - } - } - - private static void processType(ParserContext context, String[] parts) { - if (!parts[2].equals(context.name)) { - // starting new metric family - need to register (if possible) prev one - context.registerAndReset(); - context.name = parts[2]; - } - - context.type = Enums.getIfPresent(Type.class, parts[3].toUpperCase()).or(DEFAULT_TYPE); - switch (context.type) { - case SUMMARY -> { - context.allowedNames.add(context.name); - context.allowedNames.add(context.name + "_count"); - context.allowedNames.add(context.name + "_sum"); - context.allowedNames.add(context.name + "_created"); - } - case HISTOGRAM -> { - context.allowedNames.add(context.name + "_count"); - context.allowedNames.add(context.name + "_sum"); - context.allowedNames.add(context.name + "_bucket"); - context.allowedNames.add(context.name + "_created"); - } - case COUNTER -> { - context.allowedNames.add(context.name); - context.allowedNames.add(context.name + "_total"); - context.allowedNames.add(context.name + "_created"); - } - case INFO -> { - context.allowedNames.add(context.name); - context.allowedNames.add(context.name + "_info"); - } - default -> context.allowedNames.add(context.name); - } - } - - private static void processSample(ParserContext context, String line) { - parseSampleLine(line).ifPresent(sample -> { - if (!context.allowedNames.contains(sample.name)) { - // starting new metric family - need to register (if possible) prev one - context.registerAndReset(); - context.name = sample.name; - context.type = DEFAULT_TYPE; - context.allowedNames.add(sample.name); - } - context.samples.add(sample); - }); - } - - private static String unescapeHelp(String text) { - // algorithm from https://github.com/prometheus/client_python/blob/a2dae6caeaf3c300db416ba10a2a3271693addd4/prometheus_client/parser.py - if (text == null || !text.contains("\\")) { - return text; - } - StringBuilder result = new StringBuilder(); - boolean slash = false; - for (int c = 0; c < text.length(); c++) { - char charAt = text.charAt(c); - if (slash) { - if (charAt == '\\') { - result.append('\\'); - } else if (charAt == 'n') { - result.append('\n'); - } else { - result.append('\\').append(charAt); - } - slash = false; - } else { - if (charAt == '\\') { - slash = true; - } else { - result.append(charAt); - } - } - } - if (slash) { - result.append("\\"); - } - return result.toString(); - } - - //returns empty if line is not valid sample string - private static Optional parseSampleLine(String line) { - - // algorithm copied from https://github.com/prometheus/client_python/blob/a2dae6caeaf3c300db416ba10a2a3271693addd4/prometheus_client/parser.py - StringBuilder name = new StringBuilder(); - StringBuilder labelname = new StringBuilder(); - StringBuilder labelvalue = new StringBuilder(); - StringBuilder value = new StringBuilder(); - List lblNames = new ArrayList<>(); - List lblVals = new ArrayList<>(); - - String state = "name"; - - for (int c = 0; c < line.length(); c++) { - char charAt = line.charAt(c); - if (state.equals("name")) { - if (charAt == '{') { - state = "startoflabelname"; - } else if (charAt == ' ' || charAt == '\t') { - state = "endofname"; - } else { - name.append(charAt); - } - } else if (state.equals("endofname")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else if (charAt == '{') { - state = "startoflabelname"; - } else { - value.append(charAt); - state = "value"; - } - } else if (state.equals("startoflabelname")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else if (charAt == '}') { - state = "endoflabels"; - } else { - labelname.append(charAt); - state = "labelname"; - } - } else if (state.equals("labelname")) { - if (charAt == '=') { - state = "labelvaluequote"; - } else if (charAt == '}') { - state = "endoflabels"; - } else if (charAt == ' ' || charAt == '\t') { - state = "labelvalueequals"; - } else { - labelname.append(charAt); - } - } else if (state.equals("labelvalueequals")) { - if (charAt == '=') { - state = "labelvaluequote"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - return Optional.empty(); - } - } else if (state.equals("labelvaluequote")) { - if (charAt == '"') { - state = "labelvalue"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - return Optional.empty(); - } - } else if (state.equals("labelvalue")) { - if (charAt == '\\') { - state = "labelvalueslash"; - } else if (charAt == '"') { - lblNames.add(labelname.toString()); - lblVals.add(labelvalue.toString()); - labelname.setLength(0); - labelvalue.setLength(0); - state = "nextlabel"; - } else { - labelvalue.append(charAt); - } - } else if (state.equals("labelvalueslash")) { - state = "labelvalue"; - if (charAt == '\\') { - labelvalue.append('\\'); - } else if (charAt == 'n') { - labelvalue.append('\n'); - } else if (charAt == '"') { - labelvalue.append('"'); - } else { - labelvalue.append('\\').append(charAt); - } - } else if (state.equals("nextlabel")) { - if (charAt == ',') { - state = "labelname"; - } else if (charAt == '}') { - state = "endoflabels"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - return Optional.empty(); - } - } else if (state.equals("endoflabels")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - value.append(charAt); - state = "value"; - } - } else if (state.equals("value")) { - if (charAt == ' ' || charAt == '\t') { - break; // timestamps are NOT supported - ignoring - } else { - value.append(charAt); - } - } - } - - return Optional.of( - new Sample( - name.toString(), - parseDouble(value.toString()), Labels.of(lblNames, lblVals) - ) - ); - } - - private static double parseDouble(String valueString) { - if (valueString.equalsIgnoreCase("NaN")) { - return Double.NaN; - } else if (valueString.equalsIgnoreCase("+Inf")) { - return Double.POSITIVE_INFINITY; - } else if (valueString.equalsIgnoreCase("-Inf")) { - return Double.NEGATIVE_INFINITY; - } - return Double.parseDouble(valueString); - } - - -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java index 4a40be6a3..13fe5f419 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java @@ -47,7 +47,7 @@ Mono> retrieve(String host) { .retrieve() .bodyToMono(String.class) .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) - .map(body -> PrometheusEndpointParser.parse(body.lines())) + .map(body -> new PrometheusTextFormatParser().parse(body)) .onErrorResume(th -> Mono.just(List.of())); } } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java index 38bcc44e5..d2eae43f7 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusScraper.java @@ -1,6 +1,7 @@ package io.kafbat.ui.service.metrics.scrape.prometheus; import io.kafbat.ui.model.MetricsScrapeProperties; +import io.kafbat.ui.service.metrics.scrape.BrokerMetricsScraper; import io.kafbat.ui.service.metrics.scrape.PerBrokerScrapedMetrics; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.util.Collection; @@ -12,7 +13,7 @@ import reactor.util.function.Tuple2; import reactor.util.function.Tuples; -public class PrometheusScraper { +public class PrometheusScraper implements BrokerMetricsScraper { private final PrometheusMetricsRetriever retriever; @@ -20,6 +21,7 @@ public PrometheusScraper(MetricsScrapeProperties scrapeProperties) { this.retriever = new PrometheusMetricsRetriever(scrapeProperties); } + @Override public Mono scrape(Collection clusterNodes) { Mono>> collected = Flux.fromIterable(clusterNodes) .flatMap(n -> retriever.retrieve(n.host()).map(metrics -> Tuples.of(n, metrics))) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java new file mode 100644 index 000000000..8076a73a1 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -0,0 +1,394 @@ +package io.kafbat.ui.service.metrics.scrape.prometheus; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import io.prometheus.metrics.model.snapshots.*; +import io.prometheus.metrics.model.snapshots.CounterSnapshot.CounterDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; + +import io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot.UnknownDataPointSnapshot; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; +import org.jspecify.annotations.Nullable; + +/** + * Parses the Prometheus text format into a {@link MetricSnapshots} object. + * This class is designed to be the functional inverse of + * {@code io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter}. + */ +class PrometheusTextFormatParser { + + // Regex to capture metric name, optional labels, value, and optional timestamp. + // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp + private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( + "^([a-zA-Z_:][a-zA-Z0-9_:]*)" + // Metric name + "(?:\\{([^}]*)\\})?" + // Optional labels (content in group 2) + "\\s+" + + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" + // Value (group 3) + "(?:\\s+([0-9]+))?$"); // Optional timestamp (group 4) + + private static final Pattern HELP_PATTERN = + Pattern.compile("^# HELP ([a-zA-Z_:][a-zA-Z0-9_:]*) (.*)"); + private static final Pattern TYPE_PATTERN = + Pattern.compile("^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (counter|gauge|histogram|summary|untyped)"); + private static final Pattern LABEL_PATTERN = + Pattern.compile("([a-zA-Z_:][a-zA-Z0-9_:]*)=\"((?:\\\\\"|\\\\\\\\|\\\\n|[^\"])*)\""); + + private record ParsedDataPoint(String name, Labels labels, double value, Long scrapedAt) { + } + + List parse(String textFormat) { + List snapshots = new ArrayList<>(); + var cxt = new ParsingContext(snapshots); + textFormat.lines() + .map(String::trim) + .filter(s -> !s.isEmpty()) + .forEach(line -> { + if (line.startsWith("#")) { + parseComment(line, cxt); + } else { + parseMetricLine(line, cxt); + } + }); + cxt.flushAndReset(); + return snapshots; + } + + private void parseComment(String line, ParsingContext cxt) { + if (line.startsWith("# HELP")) { + Matcher m = HELP_PATTERN.matcher(line); + if (m.matches()) { + cxt.metricNameAndHelp( + PrometheusNaming.sanitizeMetricName(m.group(1)), + m.group(2) + ); + } + } else if (line.startsWith("# TYPE")) { + Matcher m = TYPE_PATTERN.matcher(line); + if (m.matches()) { + cxt.metricNameAndType( + PrometheusNaming.sanitizeMetricName(m.group(1)), + MetricType.valueOf(m.group(2).toUpperCase()) + ); + } + } + } + + private void parseMetricLine(String line, ParsingContext cxt) { + Matcher m = METRIC_LINE_PATTERN.matcher(line); + if (m.matches()) { + String metricName = m.group(1); + String labelsString = m.group(2); + String valueString = m.group(3); + String timestampString = m.group(4); + cxt.dataPoint( + new ParsedDataPoint( + metricName, + Optional.ofNullable(labelsString).map(this::parseLabels).orElse(Labels.EMPTY), + parseDouble(valueString), + Optional.ofNullable(timestampString).map(Long::parseLong).orElse(0L))); + } + } + + private Labels parseLabels(String labelsString) { + Labels.Builder builder = Labels.builder(); + Matcher m = LABEL_PATTERN.matcher(labelsString); + while (m.find()) { + builder.label(m.group(1), unescapeLabelValue(m.group(2))); + } + return builder.build(); + } + + private String unescapeLabelValue(String value) { + return value.replace("\\\\", "\\").replace("\\\"", "\"").replace("\\n", "\n"); + } + + private static double parseDouble(String value) { + return switch (value) { + case "+Inf" -> Double.POSITIVE_INFINITY; + case "-Inf" -> Double.NEGATIVE_INFINITY; + case "NaN" -> Double.NaN; + default -> Double.parseDouble(value); + }; + } + + private enum MetricType { + COUNTER, + GAUGE, + UNTYPED, + HISTOGRAM, + SUMMARY + } + + private static class ParsingContext { + + private final List sink; + + private String currentMetricName; + private String currentHelp; + private MetricDataPointsAccumulator dataPoints; + + ParsingContext(List sink) { + this.sink = sink; + } + + private void reset() { + currentMetricName = null; + currentHelp = null; + dataPoints = null; + } + + void metricNameAndType(String metricName, MetricType metricType) { + if (!metricName.equals(currentMetricName)) { + flushAndReset(); + } + currentMetricName = metricName; + dataPoints = switch (metricType) { + case UNTYPED -> new UntypedDataPointsAccumulator(); + case GAUGE -> new GaugeDataPointsAccumulator(); + case COUNTER -> new CounterDataPointsAccumulator(metricName); + case HISTOGRAM -> new HistogramDataPointsAccumulator(metricName); + case SUMMARY -> new SummaryDataPointsAccumulator(metricName); + }; + } + + void metricNameAndHelp(String metricName, String help) { + if (!metricName.equals(currentMetricName)) { + flushAndReset(); + } + currentMetricName = metricName; + currentHelp = help; + } + + void dataPoint(ParsedDataPoint parsedDataPoint) { + if (currentMetricName == null) { + currentMetricName = parsedDataPoint.name; + } + if (dataPoints == null) { + dataPoints = new UntypedDataPointsAccumulator(); + } + if (!dataPoints.add(parsedDataPoint)) { + flushAndReset(); + dataPoint(parsedDataPoint); + } + } + + void flushAndReset() { + if (dataPoints != null) { + dataPoints.buildSnapshot(currentMetricName, currentHelp) + .ifPresent(sink::add); + } + reset(); + } + } + + interface MetricDataPointsAccumulator { + boolean add(ParsedDataPoint parsedDataPoint); + Optional buildSnapshot(String name, @Nullable String help); + } + + static class UntypedDataPointsAccumulator implements MetricDataPointsAccumulator { + + final List dataPoints = new ArrayList<>(); + String name; + + @Override + public boolean add(ParsedDataPoint dp) { + if (name == null) { + name = dp.name; + } else if (!name.equals(dp.name)) { + return false; + } + dataPoints.add( + UnknownDataPointSnapshot.builder() + .labels(dp.labels).value(dp.value).scrapeTimestampMillis(dp.scrapedAt).build()); + return true; + } + + @Override + public Optional buildSnapshot(String name, @Nullable String help) { + if (dataPoints.isEmpty()) { + return Optional.empty(); + } + var builder = UnknownSnapshot.builder().name(name).help(help); + dataPoints.forEach(builder::dataPoint); + return Optional.of(builder.build()); + } + } + + static class GaugeDataPointsAccumulator implements MetricDataPointsAccumulator { + + final List dataPoints = new ArrayList<>(); + + @Override + public boolean add(ParsedDataPoint dp) { + dataPoints.add( + GaugeDataPointSnapshot.builder() + .labels(dp.labels).value(dp.value).scrapeTimestampMillis(dp.scrapedAt).build()); + return true; + } + + @Override + public Optional buildSnapshot(String name, @Nullable String help) { + if (dataPoints.isEmpty()) { + return Optional.empty(); + } + var builder = GaugeSnapshot.builder().name(name).help(help); + dataPoints.forEach(builder::dataPoint); + return Optional.of(builder.build()); + } + } + + @RequiredArgsConstructor + static class CounterDataPointsAccumulator extends UntypedDataPointsAccumulator { + + final String name; + final List dataPoints = new ArrayList<>(); + + @Override + public boolean add(ParsedDataPoint dp) { + if (!dp.name.equals(name + "_total")) { + return false; + } + dataPoints.add( + CounterDataPointSnapshot.builder() + .labels(dp.labels).value(dp.value).scrapeTimestampMillis(dp.scrapedAt).build()); + return true; + } + + @Override + public Optional buildSnapshot(String name, @Nullable String help) { + if (dataPoints.isEmpty()) { + return Optional.empty(); + } + var builder = CounterSnapshot.builder().name(name).help(help); + dataPoints.forEach(builder::dataPoint); + return Optional.of(builder.build()); + } + } + + @RequiredArgsConstructor + static class HistogramDataPointsAccumulator implements MetricDataPointsAccumulator { + + //contains cumulative(!) counts + record Bucket(double le, long count) implements Comparable { + @Override + public int compareTo(@NotNull Bucket o) { + return Double.compare(le, o.le); + } + } + + final String name; + final Map sums = new HashMap<>(); + final Multimap buckets = HashMultimap.create(); + + @Override + public boolean add(ParsedDataPoint dp) { + if (dp.name.equals(name + "_bucket") && dp.labels.contains("le")) { + var histLbls = rmLabel(dp.labels, "le"); + buckets.put(histLbls, new Bucket(parseDouble(dp.labels.get("le")), (long) dp.value)); + return true; + } + if (dp.name.equals(name + "_count")) { + return true; //skipping counts + } + if (dp.name.equals(name + "_sum")) { + sums.put(dp.labels, dp.value); + return true; + } + return false; + } + + @Override + public Optional buildSnapshot(String name, @Nullable String help) { + if (buckets.isEmpty()) { + return Optional.empty(); + } + var builder = HistogramSnapshot.builder().name(name).help(help); + buckets.asMap().forEach((labels, buckets) -> { + buckets = buckets.stream().sorted().toList(); + long prevCount = 0; + var nonCumulativeBuckets = new ArrayList(); + for (Bucket b : buckets) { + nonCumulativeBuckets.add(new Bucket(b.le, b.count - prevCount)); + prevCount = b.count; + } + builder.dataPoint( + HistogramDataPointSnapshot.builder() + .labels(labels) + .classicHistogramBuckets( + ClassicHistogramBuckets.of( + nonCumulativeBuckets.stream().map(b -> b.le).toList(), + nonCumulativeBuckets.stream().map(b -> b.count).toList() + ) + ) + .sum(sums.getOrDefault(labels, Double.NaN)) + .build() + ); + }); + return Optional.of(builder.build()); + } + } + + @RequiredArgsConstructor + static class SummaryDataPointsAccumulator implements MetricDataPointsAccumulator { + + final String name; + final Map sums = new HashMap<>(); + final Map counts = new HashMap<>(); + final Multimap quantiles = HashMultimap.create(); + + @Override + public boolean add(ParsedDataPoint dp) { + if (dp.name.equals(name) && dp.labels.contains("quantile")) { + var histLbls = rmLabel(dp.labels, "quantile"); + quantiles.put(histLbls, new Quantile(parseDouble(dp.labels.get("quantile")), dp.value)); + return true; + } + if (dp.name.equals(name + "_count")) { + counts.put(dp.labels, (long) dp.value); + return true; + } + if (dp.name.equals(name + "_sum")) { + sums.put(dp.labels, dp.value); + return true; + } + return false; + } + + @Override + public Optional buildSnapshot(String name, @Nullable String help) { + if (quantiles.isEmpty()) { + return Optional.empty(); + } + var builder = SummarySnapshot.builder().name(name).help(help); + quantiles.asMap().forEach((labels, quantiles) -> { + builder.dataPoint( + SummarySnapshot.SummaryDataPointSnapshot.builder() + .labels(labels) + .quantiles(Quantiles.of(new ArrayList<>(quantiles))) + .sum(sums.getOrDefault(labels, Double.NaN)) + .count(counts.getOrDefault(labels, 0L)) + .build() + ); + }); + return Optional.of(builder.build()); + } + } + + private static Labels rmLabel(Labels labels, String labelToExclude) { + var builder = Labels.builder(); + labels.stream() + .filter(l -> !l.getName().equals(labelToExclude)) + .forEach(l -> builder.label(l.getName(), l.getValue())); + return builder.build(); + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java deleted file mode 100644 index 0c7b9c1ac..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/sink/KafkaSink.java +++ /dev/null @@ -1,78 +0,0 @@ -package io.kafbat.ui.service.metrics.sink; - -import static io.kafbat.ui.service.MessagesService.createProducer; -import static io.kafbat.ui.service.metrics.prometheus.PrometheusExpose.escapedLabelValue; -import static io.kafbat.ui.util.MetricsUtils.readPointValue; -import static io.kafbat.ui.util.MetricsUtils.toGoString; -import static org.apache.kafka.clients.producer.ProducerConfig.COMPRESSION_TYPE_CONFIG; - -import com.fasterxml.jackson.databind.json.JsonMapper; -import io.kafbat.ui.config.ClustersProperties; -import io.prometheus.metrics.model.snapshots.Label; -import io.prometheus.metrics.model.snapshots.MetricSnapshot; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.stream.Stream; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import reactor.core.publisher.Mono; - -/* - * Format of records copied from https://github.com/Telefonica/prometheus-kafka-adapter - */ -@RequiredArgsConstructor -class KafkaSink implements MetricsSink { - - record KafkaMetric(String timestamp, String value, String name, Map labels) { } - - private static final JsonMapper JSON_MAPPER = new JsonMapper(); - - private static final Map PRODUCER_ADDITIONAL_CONFIGS = Map.of(COMPRESSION_TYPE_CONFIG, "gzip"); - - private final String topic; - private final Producer producer; - - static KafkaSink create(ClustersProperties.Cluster cluster, String targetTopic) { - return new KafkaSink(targetTopic, createProducer(cluster, PRODUCER_ADDITIONAL_CONFIGS)); - } - - @Override - public Mono send(Stream metrics) { - return Mono.fromRunnable(() -> { - String ts = Instant.now() - .truncatedTo(ChronoUnit.SECONDS) - .atZone(ZoneOffset.UTC) - .format(DateTimeFormatter.ISO_DATE_TIME); - - metrics.flatMap(m -> createRecord(ts, m)).forEach(producer::send); - }); - } - - private Stream> createRecord(String ts, MetricSnapshot metric) { - String name = metric.getMetadata().getName(); - return metric.getDataPoints().stream() - .map(sample -> { - var lbls = new LinkedHashMap(); - lbls.put("__name__", name); - - for (Label label : sample.getLabels()) { - lbls.put(label.getName(), escapedLabelValue(label.getValue())); - } - - var km = new KafkaMetric(ts, toGoString(readPointValue(sample)), name, lbls); - return new ProducerRecord<>(topic, toJsonBytes(km)); - }); - } - - @SneakyThrows - private static byte[] toJsonBytes(KafkaMetric m) { - return JSON_MAPPER.writeValueAsBytes(m); - } - -} diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java index 4136f54bc..900882f1d 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/MetricsSink.java @@ -19,9 +19,6 @@ static MetricsSink create(ClustersProperties.Cluster cluster) { .flatMap(metrics -> Optional.ofNullable(metrics.getStore())) .flatMap(store -> Optional.ofNullable(store.getPrometheus())) .ifPresent(prometheusConf -> { - if (hasText(prometheusConf.getUrl()) && Boolean.TRUE.equals(prometheusConf.getRemoteWrite())) { - sinks.add(new PrometheusRemoteWriteSink(prometheusConf.getUrl(), cluster.getSsl())); - } if (hasText(prometheusConf.getPushGatewayUrl())) { sinks.add( PrometheusPushGatewaySink.create( @@ -31,25 +28,16 @@ static MetricsSink create(ClustersProperties.Cluster cluster) { )); } }); - - Optional.ofNullable(cluster.getMetrics()) - .flatMap(metrics -> Optional.ofNullable(metrics.getStore())) - .flatMap(store -> Optional.ofNullable(store.getKafka())) - .flatMap(kafka -> Optional.ofNullable(kafka.getTopic())) - .ifPresent(topic -> sinks.add(KafkaSink.create(cluster, topic))); - return compoundSink(sinks); } private static MetricsSink compoundSink(List sinks) { - return metricsStream -> { - var materialized = metricsStream.toList(); - return Flux.fromIterable(sinks) - .flatMap(sink -> sink.send(materialized.stream())) - .then(); - }; + return metricsFlux -> + Flux.fromIterable(sinks) + .flatMap(sink -> sink.send(metricsFlux)) + .then(); } - Mono send(Stream metrics); + Mono send(Flux metrics); } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java index 622858656..a94880e2d 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java @@ -7,10 +7,9 @@ import io.prometheus.metrics.model.snapshots.MetricSnapshots; import jakarta.annotation.Nullable; import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -34,13 +33,12 @@ static PrometheusPushGatewaySink create(String url, } @Override - public Mono send(Stream metrics) { - List metricsToPush = metrics.toList(); - if (metricsToPush.isEmpty()) { - return Mono.empty(); - } - return Mono.fromRunnable(() -> pushSync(metricsToPush)) - .subscribeOn(Schedulers.boundedElastic()); + public Mono send(Flux metrics) { + return metrics.collectList() + .filter(lst -> !lst.isEmpty()) + .doOnNext(this::pushSync) + .subscribeOn(Schedulers.boundedElastic()) + .then(); } @SneakyThrows diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java deleted file mode 100644 index b72690799..000000000 --- a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusRemoteWriteSink.java +++ /dev/null @@ -1,80 +0,0 @@ -package io.kafbat.ui.service.metrics.sink; - - -import io.kafbat.ui.config.ClustersProperties.TruststoreConfig; -import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; -import io.kafbat.ui.util.MetricsUtils; -import io.kafbat.ui.util.WebClientConfigurator; -import io.prometheus.metrics.model.snapshots.DataPointSnapshot; -import io.prometheus.metrics.model.snapshots.Label; -import io.prometheus.metrics.model.snapshots.MetricSnapshot; -import jakarta.annotation.Nullable; -import java.net.URI; -import java.util.stream.Stream; -import lombok.SneakyThrows; -import org.springframework.util.unit.DataSize; -import org.springframework.web.reactive.function.client.WebClient; -import org.xerial.snappy.Snappy; -import prometheus.Remote; -import prometheus.Types; -import reactor.core.publisher.Mono; - -class PrometheusRemoteWriteSink implements MetricsSink { - - private final URI writeEndpoint; - private final WebClient webClient; - - PrometheusRemoteWriteSink(String prometheusUrl, @Nullable TruststoreConfig truststoreConfig) { - this.writeEndpoint = URI.create(prometheusUrl).resolve("/api/v1/write"); - this.webClient = new WebClientConfigurator() - .configureSsl(truststoreConfig, null) - .configureBufferSize(DataSize.ofMegabytes(20)) - .build(); - } - - @SneakyThrows - @Override - public Mono send(Stream metrics) { - byte[] bytesToWrite = Snappy.compress(createWriteRequest(metrics).toByteArray()); - return webClient.post() - .uri(writeEndpoint) - .header("Content-Type", "application/x-protobuf") - .header("User-Agent", "promremote-kafbat-ui/0.1.0") - .header("Content-Encoding", "snappy") - .header("X-Prometheus-Remote-Write-Version", "0.1.0") - .bodyValue(bytesToWrite) - .retrieve() - .toBodilessEntity() - .then(); - } - - private static Remote.WriteRequest createWriteRequest(Stream metrics) { - long currentTs = System.currentTimeMillis(); - Remote.WriteRequest.Builder request = Remote.WriteRequest.newBuilder(); - metrics.forEach(mfs -> { - for (DataPointSnapshot dataPoint : mfs.getDataPoints()) { - Types.TimeSeries.Builder timeSeriesBuilder = Types.TimeSeries.newBuilder(); - timeSeriesBuilder.addLabels( - Types.Label.newBuilder().setName("__name__").setValue(mfs.getMetadata().getName()) - ); - for (Label label : dataPoint.getLabels()) { - timeSeriesBuilder.addLabels( - Types.Label.newBuilder() - .setName(label.getName()) - .setValue(PrometheusExpose.escapedLabelValue(label.getValue())) - ); - } - timeSeriesBuilder.addSamples( - Types.Sample.newBuilder() - .setValue(MetricsUtils.readPointValue(dataPoint)) - .setTimestamp(currentTs) - ); - request.addTimeseries(timeSeriesBuilder); - } - }); - //TODO: pass Metadata - return request.build(); - } - - -} diff --git a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java index a7753301c..48b02454a 100644 --- a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java +++ b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java @@ -12,12 +12,16 @@ import io.prometheus.metrics.model.snapshots.Labels; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.SummarySnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot.UnknownDataPointSnapshot; import java.util.Collection; import java.util.List; import java.util.stream.Stream; import org.apache.kafka.clients.MetadataSnapshot; public final class MetricsUtils { + + //TODO: rm public static double readPointValue(DataPointSnapshot dps) { return switch (dps) { case GaugeDataPointSnapshot guage -> guage.getValue(); @@ -26,6 +30,7 @@ public static double readPointValue(DataPointSnapshot dps) { }; } + //TODO: rm public static String toGoString(double d) { if (d == Double.POSITIVE_INFINITY) { return "+Inf"; @@ -38,7 +43,17 @@ public static String toGoString(double d) { public static MetricSnapshot appendLabel(MetricSnapshot md, String name, String value) { return switch (md) { - case GaugeSnapshot gauge -> new GaugeSnapshot(gauge.getMetadata(), gauge.getDataPoints() + case UnknownSnapshot unknown -> new UnknownSnapshot(unknown.getMetadata(), unknown.getDataPoints() + .stream().map(dp -> + new UnknownDataPointSnapshot( + dp.getValue(), + extendLabels(dp.getLabels(), name, value), + dp.getExemplar(), + dp.getScrapeTimestampMillis() + ) + ).toList() + ); + case GaugeSnapshot gauge -> new GaugeSnapshot(gauge.getMetadata(), gauge.getDataPoints() .stream().map(dp -> new GaugeDataPointSnapshot( dp.getValue(), @@ -88,6 +103,8 @@ public static MetricSnapshot concatDataPoints(MetricSnapshot d1, MetricSnapshot ).toList(); return switch (d1) { + case UnknownSnapshot u -> new UnknownSnapshot(u.getMetadata(), + (Collection)dataPoints); case GaugeSnapshot g -> new GaugeSnapshot(g.getMetadata(), (Collection) dataPoints); case CounterSnapshot c -> new CounterSnapshot(c.getMetadata(), @@ -100,8 +117,7 @@ public static MetricSnapshot concatDataPoints(MetricSnapshot d1, MetricSnapshot }; } - - public static Labels extendLabels(Labels labels, String name, String value) { + private static Labels extendLabels(Labels labels, String name, String value) { return labels.add(name, value); } } diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java index 19dc2997f..94257bc38 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScannerTest.java @@ -4,7 +4,7 @@ import static java.util.stream.Collectors.toMap; import static org.assertj.core.api.Assertions.assertThat; -import io.kafbat.ui.service.metrics.scrape.prometheus.PrometheusEndpointParser; +import io.kafbat.ui.service.metrics.scrape.prometheus.PrometheusTextFormatParser; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.math.BigDecimal; import java.util.List; @@ -69,7 +69,7 @@ private void populateWith(Map.Entry>... entries) { } private Map.Entry> nodeMetrics(Node n, String... prometheusMetrics) { - return Map.entry(n.id(), PrometheusEndpointParser.parse(stream(prometheusMetrics))); + return Map.entry(n.id(), new PrometheusTextFormatParser().parse(String.join("\n", prometheusMetrics))); } } diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java deleted file mode 100644 index 7c063a94b..000000000 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/PrometheusEndpointParserTest.java +++ /dev/null @@ -1,200 +0,0 @@ -package io.kafbat.ui.service.metrics.scrape; - -import static io.kafbat.ui.service.metrics.MetricsUtils.isTheSameMetric; -import static io.kafbat.ui.service.metrics.scrape.prometheus.PrometheusEndpointParser.parse; -import static java.lang.Double.POSITIVE_INFINITY; -import static org.assertj.core.api.Assertions.assertThat; - -import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; -import io.prometheus.metrics.core.metrics.Counter; -import io.prometheus.metrics.core.metrics.Gauge; -import io.prometheus.metrics.core.metrics.Histogram; -import io.prometheus.metrics.core.metrics.Info; -import io.prometheus.metrics.core.metrics.Summary; -import io.prometheus.metrics.model.registry.PrometheusRegistry; -import io.prometheus.metrics.model.snapshots.CounterSnapshot; -import io.prometheus.metrics.model.snapshots.GaugeSnapshot; -import io.prometheus.metrics.model.snapshots.Labels; -import io.prometheus.metrics.model.snapshots.MetricMetadata; -import io.prometheus.metrics.model.snapshots.MetricSnapshot; -import io.prometheus.metrics.model.snapshots.MetricSnapshots; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import org.junit.jupiter.api.Test; - -class PrometheusEndpointParserTest { - - @Test - void parsesAllGeneratedMetricTypes() { - MetricSnapshots original = generateMfs(); - String exposed = PrometheusExpose.constructHttpsResponse(original).getBody(); - List parsed = parse(exposed.lines()); - assertThat(parsed).containsExactlyElementsOf(original); - } - - @Test - void parsesMetricsFromPrometheusEndpointOutput() { - String expose = """ - # HELP http_requests_total The total number of HTTP requests. - # TYPE http_requests_total counter - http_requests_total{method="post",code="200",} 1027 1395066363000 - http_requests_total{method="post",code="400",} 3 1395066363000 - # Minimalistic line: - metric_without_timestamp_and_labels 12.47 - # A weird metric from before the epoch: - something_weird{problem="division by zero"} +Inf -3982045 - # TYPE something_untyped untyped - something_untyped{} -123123 - # TYPE unit_test_seconds counter - # UNIT unit_test_seconds seconds - # HELP unit_test_seconds Testing that unit parsed properly - unit_test_seconds_total 4.20072246e+06 - # HELP http_request_duration_seconds A histogram of the request duration. - # TYPE http_request_duration_seconds histogram - http_request_duration_seconds_bucket{le="0.05"} 24054 - http_request_duration_seconds_bucket{le="0.1"} 33444 - http_request_duration_seconds_bucket{le="0.2"} 100392 - http_request_duration_seconds_bucket{le="0.5"} 129389 - http_request_duration_seconds_bucket{le="1"} 133988 - http_request_duration_seconds_bucket{le="+Inf"} 144320 - http_request_duration_seconds_sum 53423 - http_request_duration_seconds_count 144320 - """; - List parsed = parse(expose.lines()); - - - - assertThat(parsed).anyMatch(p -> isTheSameMetric(p, - new CounterSnapshot( - new MetricMetadata("http_requests", "The total number of HTTP requests."), - List.of( - new CounterSnapshot.CounterDataPointSnapshot( - 1027, - Labels.of("method", "post", "code", "200"), - null, - 0 - ), - new CounterSnapshot.CounterDataPointSnapshot( - 3, - Labels.of("method", "post", "code", "400"), - null, - 0 - ) - ) - )) - ).anyMatch(p -> isTheSameMetric(p, - new GaugeSnapshot( - new MetricMetadata("metric_without_timestamp_and_labels", "metric_without_timestamp_and_labels"), - List.of( - new GaugeSnapshot.GaugeDataPointSnapshot(12.47, Labels.EMPTY, null) - ) - )) - ).anyMatch(p -> isTheSameMetric(p, - new GaugeSnapshot( - new MetricMetadata("something_weird", "something_weird"), - List.of( - new GaugeSnapshot.GaugeDataPointSnapshot(POSITIVE_INFINITY, - Labels.of("problem", "division by zero"), null) - ) - )) - -// new MetricFamilySamples( -// "something_untyped", -// Type.GAUGE, -// "something_untyped", -// List.of(new Sample("something_untyped", List.of(), List.of(), -123123)) -// ), -// new MetricFamilySamples( -// "unit_test_seconds", -// "seconds", -// Type.COUNTER, -// "Testing that unit parsed properly", -// List.of(new Sample("unit_test_seconds_total", List.of(), List.of(), 4.20072246e+06)) -// ), -// new MetricFamilySamples( -// "http_request_duration_seconds", -// Type.HISTOGRAM, -// "A histogram of the request duration.", -// List.of( -// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.05"), 24054), -// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.1"), 33444), -// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.2"), 100392), -// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("0.5"), 129389), -// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("1"), 133988), -// new Sample("http_request_duration_seconds_bucket", List.of("le"), List.of("+Inf"), 144320), -// new Sample("http_request_duration_seconds_sum", List.of(), List.of(), 53423), -// new Sample("http_request_duration_seconds_count", List.of(), List.of(), 144320) -// ) -// ) - ); - } - - private MetricSnapshots generateMfs() { - PrometheusRegistry collectorRegistry = new PrometheusRegistry(); - - Gauge.builder() - .name("test_gauge") - .help("help for gauge") - .register(collectorRegistry) - .set(42); - - Info.builder() - .name("test_info") - .help("help for info") - .labelNames("branch", "version", "revision") - .register(collectorRegistry) - .addLabelValues("HEAD", "1.2.3", "e0704b"); - - Counter.builder() - .name("counter_no_labels") - .help("help for counter no lbls") - .register(collectorRegistry) - .inc(111); - - var counterWithLbls = Counter.builder() - .name("counter_with_labels") - .help("help for counter with lbls") - .labelNames("lbl1", "lbl2") - .register(collectorRegistry); - - counterWithLbls.labelValues("v1", "v2").inc(234); - counterWithLbls.labelValues("v11", "v22").inc(345); - - var histogram = Histogram.builder() - .name("test_hist") - .help("help for hist") -// .linearBuckets(0.0, 1.0, 10) - .labelNames("lbl1", "lbl2") - .register(collectorRegistry); - - var summary = Summary.builder() - .name("test_summary") - .help("help for hist") - .labelNames("lbl1", "lbl2") - .register(collectorRegistry); - - for (int i = 0; i < 30; i++) { - var val = ThreadLocalRandom.current().nextDouble(10.0); - histogram.labelValues("v1", "v2").observe(val); - summary.labelValues("v1", "v2").observe(val); - } - -// //emulating unknown type -// collectorRegistry.register(new Collector() { -// @Override -// public List collect() { -// return List.of( -// new MetricFamilySamples( -// "test_unknown", -// Type.UNKNOWN, -// "help for unknown", -// List.of(new Sample("test_unknown", List.of("l1"), List.of("v1"), 23432.0)) -// ) -// ); -// } -// }); -// return Lists.newArrayList(Iterators.forEnumeration(collectorRegistry.scrape())); - return collectorRegistry.scrape(); - } - -} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java new file mode 100644 index 000000000..2e29cc4fe --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java @@ -0,0 +1,142 @@ +package io.kafbat.ui.service.metrics.scrape.prometheus; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.base.Charsets; +import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import java.io.ByteArrayOutputStream; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; + +class PrometheusTextFormatParserTest { + + @Test + void testCounter() { + String source = """ + # HELP kafka_network_requestmetrics_requests_total Total number of network requests + # TYPE kafka_network_requestmetrics_requests_total counter + kafka_network_requestmetrics_requests_total{request="FetchConsumer"} 138912.0 + kafka_network_requestmetrics_requests_total{request="Metadata"} 21001.0 + kafka_network_requestmetrics_requests_total{request="Produce"} 140321.0 + """; + test2waySerialization(source); + } + + @Test + void testGauge() { + String source = """ + # HELP kafka_controller_kafkacontroller_activecontrollercount Number of active controllers + # TYPE kafka_controller_kafkacontroller_activecontrollercount gauge + kafka_controller_kafkacontroller_activecontrollercount 1.0 + """; + test2waySerialization(source); + } + + @Test + void testHistogram() { + String source = """ + # HELP http_request_duration_seconds Request duration in seconds + # TYPE http_request_duration_seconds histogram + http_request_duration_seconds_bucket{method="GET",path="/hello",le="0.01"} 2 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="0.05"} 10 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="0.1"} 32 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="0.25"} 76 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="0.5"} 91 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="1.0"} 98 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="2.5"} 100 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="5.0"} 100 + http_request_duration_seconds_bucket{method="GET",path="/hello",le="+Inf"} 100 + http_request_duration_seconds_count{method="GET",path="/hello"} 100 + http_request_duration_seconds_sum{method="GET",path="/hello"} 22.57 + """; + test2waySerialization(source); + } + + @Test + void testSummary() { + String source = """ + # HELP kafka_network_requestmetrics_queue_time_ms Total time spent in request queue + # TYPE kafka_network_requestmetrics_queue_time_ms summary + kafka_network_requestmetrics_queue_time_ms{request="FetchConsumer",quantile="0.5"} 1.23 + kafka_network_requestmetrics_queue_time_ms{request="FetchConsumer",quantile="0.95"} 5.34 + kafka_network_requestmetrics_queue_time_ms{request="FetchConsumer",quantile="0.99"} 9.12 + kafka_network_requestmetrics_queue_time_ms_count{request="FetchConsumer"} 138912 + kafka_network_requestmetrics_queue_time_ms_sum{request="FetchConsumer"} 37812.3 + """; + test2waySerialization(source); + } + + @Test + void testUntyped() { + String source = """ + kafka_server_some_untyped_metric{topic="orders"} 138922 + """; + String expected = """ + # TYPE kafka_server_some_untypled_metric untyped + kafka_server_some_untyped_metric{topic="orders"} 138922.0 + """; + test2waySerialization(source, expected); + } + + @Test + void testVariousTypes() { + String source = """ + # HELP kafka_server_brokertopicmetrics_totalfetchrequests_total Total number of fetch requests + # TYPE kafka_server_brokertopicmetrics_totalfetchrequests_total counter + kafka_server_brokertopicmetrics_totalfetchrequests_total{topic="orders"} 138922.0 + + # some invalid comment here + kafka_server_some_untyped_metric{topic="orders"} 138922 + + # Minimalistic line: + metric_without_timestamp_and_labels 12.47 + + # HELP help_no_type Some metric with help, but no type + help_no_type{lbl="test1"} 1 + help_no_type{lbl="test2"} 2 + + # Escaping in label values: + msdos_file_access_time_seconds{path="C:\\\\DIR\\\\FILE.TXT",error="Cannot find file:\\n\\"FILE.TXT\\""} 1.458255915e9 + + # HELP kafka_controller_kafkacontroller_activecontrollercount Number of active controllers + # TYPE kafka_controller_kafkacontroller_activecontrollercount gauge + kafka_controller_kafkacontroller_activecontrollercount 1 + """; + + String expected = """ + # HELP help_no_type Some metric with help, but no type + # TYPE help_no_type untyped + help_no_type{lbl="test1"} 1.0 + help_no_type{lbl="test2"} 2.0 + # HELP kafka_controller_kafkacontroller_activecontrollercount Number of active controllers + # TYPE kafka_controller_kafkacontroller_activecontrollercount gauge + kafka_controller_kafkacontroller_activecontrollercount 1.0 + # HELP kafka_server_brokertopicmetrics_totalfetchrequests_total Total number of fetch requests + # TYPE kafka_server_brokertopicmetrics_totalfetchrequests_total counter + kafka_server_brokertopicmetrics_totalfetchrequests_total{topic="orders"} 138922.0 + # TYPE kafka_server_some_untyped_metric untyped + kafka_server_some_untyped_metric{topic="orders"} 138922.0 + # TYPE metric_without_timestamp_and_labels untyped + metric_without_timestamp_and_labels 12.47 + # TYPE msdos_file_access_time_seconds untyped + msdos_file_access_time_seconds{error="Cannot find file:\\n\\"FILE.TXT\\"",path="C:\\\\DIR\\\\FILE.TXT"} 1.458255915E9 + """; + + test2waySerialization(source, expected); + } + + private void test2waySerialization(String test) { + test2waySerialization(test, test); + } + + @SneakyThrows + private void test2waySerialization(String source, + String expected) { + var baos = new ByteArrayOutputStream(); + new PrometheusTextFormatWriter(false) + .write(baos, new MetricSnapshots(new PrometheusTextFormatParser().parse(source))); + assertThat(baos.toString(Charsets.UTF_8)).isEqualTo(expected); + } + +} From d107222a86615bd6bc3f9d43cd89f092c4f7d261 Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 10:14:23 +0500 Subject: [PATCH 04/29] checkstyle --- .../kafbat/ui/model/InternalLogDirStats.java | 1 + .../main/java/io/kafbat/ui/model/Metrics.java | 4 +- .../ui/service/KafkaClusterFactory.java | 16 ++--- .../service/graphs/PromQueryLangGrammar.java | 3 +- .../integration/odd/TopicsExporter.java | 7 +- .../kafbat/ui/service/metrics/RawMetric.java | 1 - .../ui/service/metrics/SummarizedMetrics.java | 64 ++++++++++++------- .../metrics/prometheus/PrometheusExpose.java | 17 ----- .../PrometheusTextFormatParser.java | 29 ++++++--- .../java/io/kafbat/ui/util/MetricsUtils.java | 22 ++----- .../integration/odd/TopicsExporterTest.java | 2 - .../ui/service/metrics/MetricsUtils.java | 20 +++--- .../prometheus/PrometheusExposeTest.java | 10 +-- .../PrometheusTextFormatParserTest.java | 1 + 14 files changed, 97 insertions(+), 100 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java b/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java index cb53ce527..d7e274f53 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java @@ -1,4 +1,5 @@ package io.kafbat.ui.model; + import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summarizingLong; diff --git a/api/src/main/java/io/kafbat/ui/model/Metrics.java b/api/src/main/java/io/kafbat/ui/model/Metrics.java index b21e523be..6c55f3172 100644 --- a/api/src/main/java/io/kafbat/ui/model/Metrics.java +++ b/api/src/main/java/io/kafbat/ui/model/Metrics.java @@ -1,5 +1,6 @@ package io.kafbat.ui.model; +import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import java.math.BigDecimal; import java.util.List; @@ -7,9 +8,6 @@ import lombok.Builder; import lombok.Value; -import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; - - @Builder @Value public class Metrics { diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index 1d0a1f5f9..eff085ef6 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -119,13 +119,13 @@ public Mono validate(ClustersProperties.Cluster clus ? validateKsql(() -> ksqlClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), - connectClientsConfigured(clusterProperties) ? - Flux.fromIterable(clusterProperties.getKafkaConnect()) - .flatMap(c -> - KafkaServicesValidation.validateConnect(() -> connectClient(clusterProperties, c)) - .map(r -> Tuples.of(c.getName(), r))) - .collectMap(Tuple2::getT1, Tuple2::getT2) - .map(Optional::of) + connectClientsConfigured(clusterProperties) + ? Flux.fromIterable(clusterProperties.getKafkaConnect()) + .flatMap(c -> + KafkaServicesValidation.validateConnect(() -> connectClient(clusterProperties, c)) + .map(r -> Tuples.of(c.getName(), r))) + .collectMap(Tuple2::getT1, Tuple2::getT2) + .map(Optional::of) : Mono.>>just(Optional.empty()), prometheusStorageConfigured(clusterProperties) @@ -137,7 +137,7 @@ public Mono validate(ClustersProperties.Cluster clus tuple.getT2().ifPresent(validation::schemaRegistry); tuple.getT3().ifPresent(validation::ksqldb); tuple.getT4().ifPresent(validation::kafkaConnects); - tuple.getT5().ifPresent(validation::prometheusStorage); + tuple.getT5().ifPresent(validation::prometheusStorage); return validation; }); } diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java index 466bd7e19..7d3db458c 100644 --- a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java +++ b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java @@ -16,8 +16,7 @@ static Optional validateExpression(String query) { parseExpression(query); return Optional.empty(); } catch (ParseCancellationException e) { - //TODO: add more descriptive msg - return Optional.of("Syntax error"); + return Optional.of("PromQL syntax error, " + e.getMessage()); } } diff --git a/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java b/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java index 46649606f..d34e0a613 100644 --- a/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java +++ b/api/src/main/java/io/kafbat/ui/service/integration/odd/TopicsExporter.java @@ -5,6 +5,7 @@ import io.kafbat.ui.service.StatisticsCache; import io.kafbat.ui.service.integration.odd.schema.DataSetFieldsExtractors; import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState; +import io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.TopicState; import io.kafbat.ui.sr.model.SchemaSubject; import java.net.URI; import java.util.List; @@ -50,7 +51,7 @@ Flux export(KafkaCluster cluster) { .items(topicsEntities)); } - private Mono createTopicDataEntity(KafkaCluster cluster, String topic, ScrapedClusterState.TopicState topicState) { + private Mono createTopicDataEntity(KafkaCluster cluster, String topic, TopicState topicState) { KafkaPath topicOddrnPath = Oddrn.topicOddrnPath(cluster, topic); return Mono.zip( @@ -75,7 +76,7 @@ private Mono createTopicDataEntity(KafkaCluster cluster, String topi ); } - private Map getNonDefaultConfigs(ScrapedClusterState.TopicState topicState) { + private Map getNonDefaultConfigs(TopicState topicState) { List config = topicState.configs(); if (config == null) { return Map.of(); @@ -85,7 +86,7 @@ private Map getNonDefaultConfigs(ScrapedClusterState.TopicState .collect(Collectors.toMap(ConfigEntry::name, ConfigEntry::value)); } - private Map getTopicMetadata(ScrapedClusterState.TopicState topicState) { + private Map getTopicMetadata(TopicState topicState) { TopicDescription topicDescription = topicState.description(); return ImmutableMap.builder() .put("partitions", topicDescription.partitions().size()) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java b/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java index b9e721fad..3b306872c 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/RawMetric.java @@ -2,7 +2,6 @@ import io.prometheus.metrics.core.metrics.Gauge; import io.prometheus.metrics.model.snapshots.MetricSnapshot; - import java.math.BigDecimal; import java.util.Arrays; import java.util.Collection; diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java index 265ff0e7a..63fac5130 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -1,7 +1,7 @@ package io.kafbat.ui.service.metrics; -import static io.prometheus.metrics.model.snapshots.CounterSnapshot.*; -import static io.prometheus.metrics.model.snapshots.GaugeSnapshot.*; +import static io.prometheus.metrics.model.snapshots.CounterSnapshot.CounterDataPointSnapshot; +import static io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; import static java.util.stream.Collectors.toMap; import com.google.common.collect.Streams; @@ -11,6 +11,8 @@ import io.prometheus.metrics.model.snapshots.GaugeSnapshot; import io.prometheus.metrics.model.snapshots.Labels; import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot.UnknownDataPointSnapshot; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Optional; @@ -27,22 +29,28 @@ public class SummarizedMetrics { public Stream asStream() { return Streams.concat( metrics.getInferredMetrics().asStream(), - metrics.getPerBrokerScrapedMetrics() - .values() - .stream() - .flatMap(Collection::stream) - .collect( - toMap( - mfs -> mfs.getMetadata().getName(), - Optional::of, SummarizedMetrics::summarizeMetricSnapshot, LinkedHashMap::new - ) - ).values() - .stream() - .filter(Optional::isPresent) - .map(Optional::get) + summarize( + metrics.getPerBrokerScrapedMetrics() + .values() + .stream() + .flatMap(Collection::stream) + ) ); } + private Stream summarize(Stream snapshots) { + return snapshots + .collect( + toMap( + mfs -> mfs.getMetadata().getName(), + Optional::of, SummarizedMetrics::summarizeMetricSnapshot, LinkedHashMap::new + ) + ).values() + .stream() + .filter(Optional::isPresent) + .map(Optional::get); + } + //returns Optional.empty if merging not supported for metric type @SuppressWarnings("unchecked") private static Optional summarizeMetricSnapshot(Optional snap1Opt, @@ -54,28 +62,38 @@ private static Optional summarizeMetricSnapshot(Optional pointFactory; Function valueGetter; Function, MetricSnapshot> builder; - if (snap1 instanceof CounterSnapshot) { + if (snap1 instanceof UnknownSnapshot) { + pointFactory = (l, v) -> UnknownDataPointSnapshot.builder() + .labels(l) + .value(v) + .build(); + valueGetter = (dp) -> ((UnknownDataPointSnapshot) dp).getValue(); + builder = (dps) -> + new UnknownSnapshot(snap1.getMetadata(), (Collection) dps); + } else if (snap1 instanceof CounterSnapshot) { pointFactory = (l, v) -> CounterDataPointSnapshot.builder() .labels(l) .value(v) .build(); - valueGetter = (dp) -> ((CounterDataPointSnapshot)dp).getValue(); + valueGetter = (dp) -> ((CounterDataPointSnapshot) dp).getValue(); builder = (dps) -> - new CounterSnapshot(snap1.getMetadata(), (Collection)dps); + new CounterSnapshot(snap1.getMetadata(), (Collection) dps); } else { - pointFactory = (l,v) -> GaugeDataPointSnapshot.builder() + pointFactory = (l, v) -> GaugeDataPointSnapshot.builder() .labels(l) .value(v) .build(); - valueGetter = (dp) -> ((GaugeDataPointSnapshot)dp).getValue(); + valueGetter = (dp) -> ((GaugeDataPointSnapshot) dp).getValue(); builder = (dps) -> - new GaugeSnapshot(snap1.getMetadata(), (Collection)dps); + new GaugeSnapshot(snap1.getMetadata(), (Collection) dps); } Collection points = diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java index f8d6f8820..fa2723e0e 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java @@ -77,21 +77,4 @@ public static ResponseEntity constructHttpsResponse(MetricSnapshots metr .headers(PROMETHEUS_EXPOSE_ENDPOINT_HEADERS) .body(buffer.toString(StandardCharsets.UTF_8)); } - - // copied from io.prometheus.client.exporter.common.TextFormat:writeEscapedLabelValue - //TODO: RM - public static String escapedLabelValue(String s) { - StringBuilder sb = new StringBuilder(s.length()); - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - switch (c) { - case '\\' -> sb.append("\\\\"); - case '\"' -> sb.append("\\\""); - case '\n' -> sb.append("\\n"); - default -> sb.append(c); - } - } - return sb.toString(); - } - } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 8076a73a1..fd06d765c 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -2,11 +2,21 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import io.prometheus.metrics.model.snapshots.*; +import io.prometheus.metrics.model.snapshots.ClassicHistogramBuckets; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; import io.prometheus.metrics.model.snapshots.CounterSnapshot.CounterDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; - +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; import io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import io.prometheus.metrics.model.snapshots.PrometheusNaming; +import io.prometheus.metrics.model.snapshots.Quantile; +import io.prometheus.metrics.model.snapshots.Quantiles; +import io.prometheus.metrics.model.snapshots.SummarySnapshot; +import io.prometheus.metrics.model.snapshots.UnknownSnapshot; import io.prometheus.metrics.model.snapshots.UnknownSnapshot.UnknownDataPointSnapshot; import java.util.ArrayList; import java.util.HashMap; @@ -24,16 +34,16 @@ * This class is designed to be the functional inverse of * {@code io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter}. */ -class PrometheusTextFormatParser { +public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( - "^([a-zA-Z_:][a-zA-Z0-9_:]*)" + // Metric name - "(?:\\{([^}]*)\\})?" + // Optional labels (content in group 2) - "\\s+" + - "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" + // Value (group 3) - "(?:\\s+([0-9]+))?$"); // Optional timestamp (group 4) + "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name + + "(?:\\{([^}]*)\\})?" // Optional labels (content in group 2) + + "\\s+" + + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Value (group 3) + + "(?:\\s+([0-9]+))?$"); // Optional timestamp (group 4) private static final Pattern HELP_PATTERN = Pattern.compile("^# HELP ([a-zA-Z_:][a-zA-Z0-9_:]*) (.*)"); @@ -45,7 +55,7 @@ class PrometheusTextFormatParser { private record ParsedDataPoint(String name, Labels labels, double value, Long scrapedAt) { } - List parse(String textFormat) { + public List parse(String textFormat) { List snapshots = new ArrayList<>(); var cxt = new ParsingContext(snapshots); textFormat.lines() @@ -192,6 +202,7 @@ void flushAndReset() { interface MetricDataPointsAccumulator { boolean add(ParsedDataPoint parsedDataPoint); + Optional buildSnapshot(String name, @Nullable String help); } diff --git a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java index 48b02454a..be40307f6 100644 --- a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java +++ b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java @@ -1,8 +1,8 @@ package io.kafbat.ui.util; -import static io.prometheus.metrics.model.snapshots.CounterSnapshot.*; -import static io.prometheus.metrics.model.snapshots.HistogramSnapshot.*; -import static io.prometheus.metrics.model.snapshots.SummarySnapshot.*; +import static io.prometheus.metrics.model.snapshots.CounterSnapshot.CounterDataPointSnapshot; +import static io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot; +import static io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot; import io.prometheus.metrics.model.snapshots.CounterSnapshot; import io.prometheus.metrics.model.snapshots.DataPointSnapshot; @@ -17,30 +17,18 @@ import java.util.Collection; import java.util.List; import java.util.stream.Stream; -import org.apache.kafka.clients.MetadataSnapshot; public final class MetricsUtils { - //TODO: rm public static double readPointValue(DataPointSnapshot dps) { return switch (dps) { + case UnknownDataPointSnapshot unknown -> unknown.getValue(); case GaugeDataPointSnapshot guage -> guage.getValue(); case CounterDataPointSnapshot counter -> counter.getValue(); default -> 0; }; } - //TODO: rm - public static String toGoString(double d) { - if (d == Double.POSITIVE_INFINITY) { - return "+Inf"; - } else if (d == Double.NEGATIVE_INFINITY) { - return "-Inf"; - } else { - return Double.toString(d); - } - } - public static MetricSnapshot appendLabel(MetricSnapshot md, String name, String value) { return switch (md) { case UnknownSnapshot unknown -> new UnknownSnapshot(unknown.getMetadata(), unknown.getDataPoints() @@ -104,7 +92,7 @@ public static MetricSnapshot concatDataPoints(MetricSnapshot d1, MetricSnapshot return switch (d1) { case UnknownSnapshot u -> new UnknownSnapshot(u.getMetadata(), - (Collection)dataPoints); + (Collection) dataPoints); case GaugeSnapshot g -> new GaugeSnapshot(g.getMetadata(), (Collection) dataPoints); case CounterSnapshot c -> new CounterSnapshot(c.getMetadata(), diff --git a/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java b/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java index 2c51703dc..c34ff742e 100644 --- a/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/integration/odd/TopicsExporterTest.java @@ -1,8 +1,6 @@ package io.kafbat.ui.service.integration.odd; -import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.TopicState; import static io.kafbat.ui.service.metrics.scrape.ScrapedClusterState.empty; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java index fd235b44c..e94be1bf4 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java @@ -1,7 +1,5 @@ package io.kafbat.ui.service.metrics; -import static org.assertj.core.api.Assertions.assertThat; - import io.prometheus.metrics.model.snapshots.CounterSnapshot; import io.prometheus.metrics.model.snapshots.DataPointSnapshot; import io.prometheus.metrics.model.snapshots.GaugeSnapshot; @@ -16,18 +14,20 @@ public static boolean isTheSameMetric(MetricSnapshot m1, MetricSnapshot m2) { MetricMetadata metadata2 = m2.getMetadata(); if ( - metadata1.getName().equals(metadata2.getName()) && - metadata1.getHelp().equals(metadata2.getHelp()) && - Optional.ofNullable( - metadata1.getUnit()).map(u -> u.equals(metadata2.getUnit()) - ).orElse(metadata2.getUnit() == null) + metadata1.getName().equals(metadata2.getName()) + && metadata1.getHelp().equals(metadata2.getHelp()) + && Optional.ofNullable( + metadata1.getUnit()).map(u -> u.equals(metadata2.getUnit()) + ).orElse(metadata2.getUnit() == null) ) { if (m1.getDataPoints().size() == m2.getDataPoints().size()) { for (int i = 0; i < m1.getDataPoints().size(); i++) { var m1dp = m1.getDataPoints().get(i); var m2dp = m2.getDataPoints().get(i); boolean same = isTheSameDataPoint(m1dp, m2dp); - if (!same) return false; + if (!same) { + return false; + } } return true; } @@ -41,11 +41,11 @@ public static boolean isTheSameDataPoint(DataPointSnapshot dp1, DataPointSnapsho if (Optional.ofNullable(dp1.getLabels()).map(l -> l.equals(dp2.getLabels())) .orElse(dp2.getLabels() == null)) { if (dp1 instanceof GaugeSnapshot.GaugeDataPointSnapshot g1) { - GaugeSnapshot.GaugeDataPointSnapshot g2 = (GaugeSnapshot.GaugeDataPointSnapshot)dp2; + GaugeSnapshot.GaugeDataPointSnapshot g2 = (GaugeSnapshot.GaugeDataPointSnapshot) dp2; return Double.compare(g1.getValue(), g2.getValue()) == 0; } if (dp1 instanceof CounterSnapshot.CounterDataPointSnapshot c1) { - CounterSnapshot.CounterDataPointSnapshot c2 = (CounterSnapshot.CounterDataPointSnapshot)dp2; + CounterSnapshot.CounterDataPointSnapshot c2 = (CounterSnapshot.CounterDataPointSnapshot) dp2; return Double.compare(c1.getValue(), c2.getValue()) == 0; } return true; diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java index 0cc945d66..927a9217f 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java @@ -20,13 +20,13 @@ class PrometheusExposeTest { void prepareMetricsForGlobalExposeAppendsClusterAndBrokerIdLabelsToMetrics() { var inferredMfs = new GaugeSnapshot(new MetricMetadata("infer", "help"), List.of( - new GaugeSnapshot.GaugeDataPointSnapshot(100, Labels.of("lbl1","lblVal1"), null))); + new GaugeSnapshot.GaugeDataPointSnapshot(100, Labels.of("lbl1", "lblVal1"), null))); var broker1Mfs = new GaugeSnapshot(new MetricMetadata("brok", "help"), List.of( - new GaugeSnapshot.GaugeDataPointSnapshot(101, Labels.of("broklbl1","broklblVal1"), null))); + new GaugeSnapshot.GaugeDataPointSnapshot(101, Labels.of("broklbl1", "broklblVal1"), null))); var broker2Mfs = new GaugeSnapshot(new MetricMetadata("brok", "help"), List.of( - new GaugeSnapshot.GaugeDataPointSnapshot(102, Labels.of("broklbl1","broklblVal1"), null))); + new GaugeSnapshot.GaugeDataPointSnapshot(102, Labels.of("broklbl1", "broklblVal1"), null))); List prepared = prepareMetricsForGlobalExpose( "testCluster", @@ -48,7 +48,7 @@ void prepareMetricsForGlobalExposeAppendsClusterAndBrokerIdLabelsToMetrics() { Labels.of( "cluster", "testCluster", "broker_id", "1", - "broklbl1","broklblVal1" + "broklbl1", "broklblVal1" ), null )) ))) @@ -57,7 +57,7 @@ void prepareMetricsForGlobalExposeAppendsClusterAndBrokerIdLabelsToMetrics() { Labels.of( "cluster", "testCluster", "broker_id", "2", - "broklbl1","broklblVal1" + "broklbl1", "broklblVal1" ), null )) ) diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java index 2e29cc4fe..0c26aa64c 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java @@ -80,6 +80,7 @@ void testUntyped() { } @Test + @SuppressWarnings("checkstyle:LineLength") void testVariousTypes() { String source = """ # HELP kafka_server_brokertopicmetrics_totalfetchrequests_total Total number of fetch requests From 8a9c401120f542d92a119e74cdae052c075c8132 Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 10:40:51 +0500 Subject: [PATCH 05/29] merge with main --- api/build.gradle | 1 - .../ui/mapper/DescribeLogDirsMapper.java | 3 +- contract/build.gradle | 7 - .../proto/prometheus/gogoproto/gogo.proto | 133 ------------- .../src/main/proto/prometheus/remote.proto | 88 --------- .../src/main/proto/prometheus/types.proto | 187 ------------------ 6 files changed, 1 insertion(+), 418 deletions(-) delete mode 100644 contract/src/main/proto/prometheus/gogoproto/gogo.proto delete mode 100644 contract/src/main/proto/prometheus/remote.proto delete mode 100644 contract/src/main/proto/prometheus/types.proto diff --git a/api/build.gradle b/api/build.gradle index 871ef6231..c0a5ed332 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -113,7 +113,6 @@ dependencies { generateGrammarSource { maxHeapSize = "64m" - arguments += ["-package", "ksql"] } tasks.withType(JavaCompile) { diff --git a/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java b/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java index 6064d0e57..dac046703 100644 --- a/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java +++ b/api/src/main/java/io/kafbat/ui/mapper/DescribeLogDirsMapper.java @@ -45,8 +45,7 @@ private BrokersLogdirsDTO toBrokerLogDirs(Integer broker, String dirName, private BrokerTopicLogdirsDTO toTopicLogDirs( Integer broker, String name, List> partitions) { - private BrokerTopicLogdirsDTO toTopicLogDirs(Integer broker, String name, - List> partitions) { + BrokerTopicLogdirsDTO topic = new BrokerTopicLogdirsDTO(); topic.setName(name); topic.setPartitions( diff --git a/contract/build.gradle b/contract/build.gradle index e1078e2f5..707a1942f 100644 --- a/contract/build.gradle +++ b/contract/build.gradle @@ -3,7 +3,6 @@ import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { id "java-library" alias(libs.plugins.openapi.generator) - alias(libs.plugins.protobuf) } def specDir = project.layout.projectDirectory.dir("src/main/resources/swagger/") @@ -12,7 +11,6 @@ def targetDir = project.layout.buildDirectory.dir("generated").get() dependencies { implementation libs.spring.starter.webflux implementation libs.spring.starter.validation - implementation libs.protobuf api libs.swagger.integration.jakarta api libs.jackson.databind.nullable api libs.jakarta.annotation.api @@ -129,10 +127,5 @@ sourceSets { } } -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${libs.versions.protobuf.get()}" } -} - - compileJava.dependsOn generateUiClient, generateBackendApi, generateConnectClient, generateSchemaRegistryClient, generatePrometheusClient processResources.dependsOn generateUiClient, generateBackendApi, generateConnectClient, generateSchemaRegistryClient, generatePrometheusClient diff --git a/contract/src/main/proto/prometheus/gogoproto/gogo.proto b/contract/src/main/proto/prometheus/gogoproto/gogo.proto deleted file mode 100644 index 2f0a3c76b..000000000 --- a/contract/src/main/proto/prometheus/gogoproto/gogo.proto +++ /dev/null @@ -1,133 +0,0 @@ -// Protocol Buffers for Go with Gadgets -// -// Copyright (c) 2013, The GoGo Authors. All rights reserved. -// http://github.com/gogo/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -package gogoproto; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "GoGoProtos"; -option go_package = "github.com/gogo/protobuf/gogoproto"; - -extend google.protobuf.EnumOptions { - optional bool goproto_enum_prefix = 62001; - optional bool goproto_enum_stringer = 62021; - optional bool enum_stringer = 62022; - optional string enum_customname = 62023; - optional bool enumdecl = 62024; -} - -extend google.protobuf.EnumValueOptions { - optional string enumvalue_customname = 66001; -} - -extend google.protobuf.FileOptions { - optional bool goproto_getters_all = 63001; - optional bool goproto_enum_prefix_all = 63002; - optional bool goproto_stringer_all = 63003; - optional bool verbose_equal_all = 63004; - optional bool face_all = 63005; - optional bool gostring_all = 63006; - optional bool populate_all = 63007; - optional bool stringer_all = 63008; - optional bool onlyone_all = 63009; - - optional bool equal_all = 63013; - optional bool description_all = 63014; - optional bool testgen_all = 63015; - optional bool benchgen_all = 63016; - optional bool marshaler_all = 63017; - optional bool unmarshaler_all = 63018; - optional bool stable_marshaler_all = 63019; - - optional bool sizer_all = 63020; - - optional bool goproto_enum_stringer_all = 63021; - optional bool enum_stringer_all = 63022; - - optional bool unsafe_marshaler_all = 63023; - optional bool unsafe_unmarshaler_all = 63024; - - optional bool goproto_extensions_map_all = 63025; - optional bool goproto_unrecognized_all = 63026; - optional bool gogoproto_import = 63027; - optional bool protosizer_all = 63028; - optional bool compare_all = 63029; - optional bool typedecl_all = 63030; - optional bool enumdecl_all = 63031; - - optional bool goproto_registration = 63032; -} - -extend google.protobuf.MessageOptions { - optional bool goproto_getters = 64001; - optional bool goproto_stringer = 64003; - optional bool verbose_equal = 64004; - optional bool face = 64005; - optional bool gostring = 64006; - optional bool populate = 64007; - optional bool stringer = 67008; - optional bool onlyone = 64009; - - optional bool equal = 64013; - optional bool description = 64014; - optional bool testgen = 64015; - optional bool benchgen = 64016; - optional bool marshaler = 64017; - optional bool unmarshaler = 64018; - optional bool stable_marshaler = 64019; - - optional bool sizer = 64020; - - optional bool unsafe_marshaler = 64023; - optional bool unsafe_unmarshaler = 64024; - - optional bool goproto_extensions_map = 64025; - optional bool goproto_unrecognized = 64026; - - optional bool protosizer = 64028; - optional bool compare = 64029; - - optional bool typedecl = 64030; -} - -extend google.protobuf.FieldOptions { - optional bool nullable = 65001; - optional bool embed = 65002; - optional string customtype = 65003; - optional string customname = 65004; - optional string jsontag = 65005; - optional string moretags = 65006; - optional string casttype = 65007; - optional string castkey = 65008; - optional string castvalue = 65009; - - optional bool stdtime = 65010; - optional bool stdduration = 65011; -} diff --git a/contract/src/main/proto/prometheus/remote.proto b/contract/src/main/proto/prometheus/remote.proto deleted file mode 100644 index 809d33f70..000000000 --- a/contract/src/main/proto/prometheus/remote.proto +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2016 Prometheus Team -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; -package prometheus; - -option go_package = "prompb"; - -import "prometheus/types.proto"; -import "prometheus/gogoproto/gogo.proto"; - -message WriteRequest { - repeated prometheus.TimeSeries timeseries = 1 [(gogoproto.nullable) = false]; - // Cortex uses this field to determine the source of the write request. - // We reserve it to avoid any compatibility issues. - reserved 2; - repeated prometheus.MetricMetadata metadata = 3 [(gogoproto.nullable) = false]; -} - -// ReadRequest represents a remote read request. -message ReadRequest { - repeated Query queries = 1; - - enum ResponseType { - // Server will return a single ReadResponse message with matched series that includes list of raw samples. - // It's recommended to use streamed response types instead. - // - // Response headers: - // Content-Type: "application/x-protobuf" - // Content-Encoding: "snappy" - SAMPLES = 0; - // Server will stream a delimited ChunkedReadResponse message that - // contains XOR or HISTOGRAM(!) encoded chunks for a single series. - // Each message is following varint size and fixed size bigendian - // uint32 for CRC32 Castagnoli checksum. - // - // Response headers: - // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" - // Content-Encoding: "" - STREAMED_XOR_CHUNKS = 1; - } - - // accepted_response_types allows negotiating the content type of the response. - // - // Response types are taken from the list in the FIFO order. If no response type in `accepted_response_types` is - // implemented by server, error is returned. - // For request that do not contain `accepted_response_types` field the SAMPLES response type will be used. - repeated ResponseType accepted_response_types = 2; -} - -// ReadResponse is a response when response_type equals SAMPLES. -message ReadResponse { - // In same order as the request's queries. - repeated QueryResult results = 1; -} - -message Query { - int64 start_timestamp_ms = 1; - int64 end_timestamp_ms = 2; - repeated prometheus.LabelMatcher matchers = 3; - prometheus.ReadHints hints = 4; -} - -message QueryResult { - // Samples within a time series must be ordered by time. - repeated prometheus.TimeSeries timeseries = 1; -} - -// ChunkedReadResponse is a response when response_type equals STREAMED_XOR_CHUNKS. -// We strictly stream full series after series, optionally split by time. This means that a single frame can contain -// partition of the single series, but once a new series is started to be streamed it means that no more chunks will -// be sent for previous one. Series are returned sorted in the same way TSDB block are internally. -message ChunkedReadResponse { - repeated prometheus.ChunkedSeries chunked_series = 1; - - // query_index represents an index of the query from ReadRequest.queries these chunks relates to. - int64 query_index = 2; -} diff --git a/contract/src/main/proto/prometheus/types.proto b/contract/src/main/proto/prometheus/types.proto deleted file mode 100644 index 4a077bdb7..000000000 --- a/contract/src/main/proto/prometheus/types.proto +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2017 Prometheus Team -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; -package prometheus; - -option go_package = "prompb"; - -import "prometheus/gogoproto/gogo.proto"; - -message MetricMetadata { - enum MetricType { - UNKNOWN = 0; - COUNTER = 1; - GAUGE = 2; - HISTOGRAM = 3; - GAUGEHISTOGRAM = 4; - SUMMARY = 5; - INFO = 6; - STATESET = 7; - } - - // Represents the metric type, these match the set from Prometheus. - // Refer to model/textparse/interface.go for details. - MetricType type = 1; - string metric_family_name = 2; - string help = 4; - string unit = 5; -} - -message Sample { - double value = 1; - // timestamp is in ms format, see model/timestamp/timestamp.go for - // conversion from time.Time to Prometheus timestamp. - int64 timestamp = 2; -} - -message Exemplar { - // Optional, can be empty. - repeated Label labels = 1 [(gogoproto.nullable) = false]; - double value = 2; - // timestamp is in ms format, see model/timestamp/timestamp.go for - // conversion from time.Time to Prometheus timestamp. - int64 timestamp = 3; -} - -// A native histogram, also known as a sparse histogram. -// Original design doc: -// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit -// The appendix of this design doc also explains the concept of float -// histograms. This Histogram message can represent both, the usual -// integer histogram as well as a float histogram. -message Histogram { - enum ResetHint { - UNKNOWN = 0; // Need to test for a counter reset explicitly. - YES = 1; // This is the 1st histogram after a counter reset. - NO = 2; // There was no counter reset between this and the previous Histogram. - GAUGE = 3; // This is a gauge histogram where counter resets don't happen. - } - - oneof count { // Count of observations in the histogram. - uint64 count_int = 1; - double count_float = 2; - } - double sum = 3; // Sum of observations in the histogram. - // The schema defines the bucket schema. Currently, valid numbers - // are -4 <= n <= 8. They are all for base-2 bucket schemas, where 1 - // is a bucket boundary in each case, and then each power of two is - // divided into 2^n logarithmic buckets. Or in other words, each - // bucket boundary is the previous boundary times 2^(2^-n). In the - // future, more bucket schemas may be added using numbers < -4 or > - // 8. - sint32 schema = 4; - double zero_threshold = 5; // Breadth of the zero bucket. - oneof zero_count { // Count in zero bucket. - uint64 zero_count_int = 6; - double zero_count_float = 7; - } - - // Negative Buckets. - repeated BucketSpan negative_spans = 8 [(gogoproto.nullable) = false]; - // Use either "negative_deltas" or "negative_counts", the former for - // regular histograms with integer counts, the latter for float - // histograms. - repeated sint64 negative_deltas = 9; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). - repeated double negative_counts = 10; // Absolute count of each bucket. - - // Positive Buckets. - repeated BucketSpan positive_spans = 11 [(gogoproto.nullable) = false]; - // Use either "positive_deltas" or "positive_counts", the former for - // regular histograms with integer counts, the latter for float - // histograms. - repeated sint64 positive_deltas = 12; // Count delta of each bucket compared to previous one (or to zero for 1st bucket). - repeated double positive_counts = 13; // Absolute count of each bucket. - - ResetHint reset_hint = 14; - // timestamp is in ms format, see model/timestamp/timestamp.go for - // conversion from time.Time to Prometheus timestamp. - int64 timestamp = 15; -} - -// A BucketSpan defines a number of consecutive buckets with their -// offset. Logically, it would be more straightforward to include the -// bucket counts in the Span. However, the protobuf representation is -// more compact in the way the data is structured here (with all the -// buckets in a single array separate from the Spans). -message BucketSpan { - sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative). - uint32 length = 2; // Length of consecutive buckets. -} - -// TimeSeries represents samples and labels for a single time series. -message TimeSeries { - // For a timeseries to be valid, and for the samples and exemplars - // to be ingested by the remote system properly, the labels field is required. - repeated Label labels = 1 [(gogoproto.nullable) = false]; - repeated Sample samples = 2 [(gogoproto.nullable) = false]; - repeated Exemplar exemplars = 3 [(gogoproto.nullable) = false]; - repeated Histogram histograms = 4 [(gogoproto.nullable) = false]; -} - -message Label { - string name = 1; - string value = 2; -} - -message Labels { - repeated Label labels = 1 [(gogoproto.nullable) = false]; -} - -// Matcher specifies a rule, which can match or set of labels or not. -message LabelMatcher { - enum Type { - EQ = 0; - NEQ = 1; - RE = 2; - NRE = 3; - } - Type type = 1; - string name = 2; - string value = 3; -} - -message ReadHints { - int64 step_ms = 1; // Query step size in milliseconds. - string func = 2; // String representation of surrounding function or aggregation. - int64 start_ms = 3; // Start time in milliseconds. - int64 end_ms = 4; // End time in milliseconds. - repeated string grouping = 5; // List of label names used in aggregation. - bool by = 6; // Indicate whether it is without or by. - int64 range_ms = 7; // Range vector selector range in milliseconds. -} - -// Chunk represents a TSDB chunk. -// Time range [min, max] is inclusive. -message Chunk { - int64 min_time_ms = 1; - int64 max_time_ms = 2; - - // We require this to match chunkenc.Encoding. - enum Encoding { - UNKNOWN = 0; - XOR = 1; - HISTOGRAM = 2; - FLOAT_HISTOGRAM = 3; - } - Encoding type = 3; - bytes data = 4; -} - -// ChunkedSeries represents single, encoded time series. -message ChunkedSeries { - // Labels should be sorted. - repeated Label labels = 1 [(gogoproto.nullable) = false]; - // Chunks will be in start time order and may overlap. - repeated Chunk chunks = 2 [(gogoproto.nullable) = false]; -} From 8426aacb1a56f62c789eca400a15c20f20939cd4 Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 10:48:26 +0500 Subject: [PATCH 06/29] tests fix --- .../scrape/prometheus/PrometheusTextFormatParserTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java index 0c26aa64c..2118bc00e 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java @@ -73,7 +73,7 @@ void testUntyped() { kafka_server_some_untyped_metric{topic="orders"} 138922 """; String expected = """ - # TYPE kafka_server_some_untypled_metric untyped + # TYPE kafka_server_some_untyped_metric untyped kafka_server_some_untyped_metric{topic="orders"} 138922.0 """; test2waySerialization(source, expected); From 6264ca7dd4b896a615383cec946dba415c9e4c9e Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 10:50:00 +0500 Subject: [PATCH 07/29] tests fix --- .../java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java index 56655eda0..618c711b1 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java @@ -279,10 +279,6 @@ private static Map populateDescriptors(Map loadSchemas(Optional> protobufFiles, Optional protobufFilesDir) { - if (true) { - return Map.of(); - } - if (protobufFilesDir.isPresent()) { if (protobufFiles.isPresent()) { log.warn("protobufFiles properties will be ignored, since protobufFilesDir provided"); From 41a4aba06fcc2e6e08deedab91271dde01b1b80e Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 11:27:20 +0500 Subject: [PATCH 08/29] proto dep version fix --- api/build.gradle | 4 +++- gradle/libs.versions.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/build.gradle b/api/build.gradle index c0a5ed332..249076006 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -79,7 +79,9 @@ dependencies { implementation libs.prometheus.metrics.core implementation libs.prometheus.metrics.textformats - implementation libs.prometheus.metrics.exporter.pushgateway + implementation (libs.prometheus.metrics.exporter.pushgateway) { + exclude group: 'com.google.protobuf', module: 'protobuf-java' because("PushGW lib pulls protobuf-java 4.x, which is incompatible with protobuf-java 3.x used by various dependencies of this project.") + } implementation libs.snappy // Annotation processors diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b0d5df90a..4e1c2d819 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -14,7 +14,7 @@ apache-commons-compress = '1.26.0' assertj = '3.25.3' avro = '1.11.4' byte-buddy = '1.14.19' -confluent = '7.9.0' +confluent = '7.9.2' confluent-ccs = '7.9.0-ccs' mapstruct = '1.6.2' From 2ac6f15c94d5f16ea1799fd2289b4a9a36f1d37e Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 11:37:55 +0500 Subject: [PATCH 09/29] minor test renaming --- .../PrometheusTextFormatParserTest.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java index 2118bc00e..b4b7120e5 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParserTest.java @@ -20,7 +20,7 @@ void testCounter() { kafka_network_requestmetrics_requests_total{request="Metadata"} 21001.0 kafka_network_requestmetrics_requests_total{request="Produce"} 140321.0 """; - test2waySerialization(source); + assertParseAndSerialize(source); } @Test @@ -30,7 +30,7 @@ void testGauge() { # TYPE kafka_controller_kafkacontroller_activecontrollercount gauge kafka_controller_kafkacontroller_activecontrollercount 1.0 """; - test2waySerialization(source); + assertParseAndSerialize(source); } @Test @@ -50,7 +50,7 @@ void testHistogram() { http_request_duration_seconds_count{method="GET",path="/hello"} 100 http_request_duration_seconds_sum{method="GET",path="/hello"} 22.57 """; - test2waySerialization(source); + assertParseAndSerialize(source); } @Test @@ -64,19 +64,20 @@ void testSummary() { kafka_network_requestmetrics_queue_time_ms_count{request="FetchConsumer"} 138912 kafka_network_requestmetrics_queue_time_ms_sum{request="FetchConsumer"} 37812.3 """; - test2waySerialization(source); + assertParseAndSerialize(source); } @Test void testUntyped() { String source = """ + # some comment that should be skipped kafka_server_some_untyped_metric{topic="orders"} 138922 """; String expected = """ # TYPE kafka_server_some_untyped_metric untyped kafka_server_some_untyped_metric{topic="orders"} 138922.0 """; - test2waySerialization(source, expected); + assertParseAndSerialize(source, expected); } @Test @@ -124,20 +125,21 @@ void testVariousTypes() { msdos_file_access_time_seconds{error="Cannot find file:\\n\\"FILE.TXT\\"",path="C:\\\\DIR\\\\FILE.TXT"} 1.458255915E9 """; - test2waySerialization(source, expected); + assertParseAndSerialize(source, expected); } - private void test2waySerialization(String test) { - test2waySerialization(test, test); + private void assertParseAndSerialize(String test) { + assertParseAndSerialize(test, test); } @SneakyThrows - private void test2waySerialization(String source, - String expected) { + private void assertParseAndSerialize(String source, + String expectedSerialized) { + var parsedMetrics = new MetricSnapshots(new PrometheusTextFormatParser().parse(source)); var baos = new ByteArrayOutputStream(); new PrometheusTextFormatWriter(false) - .write(baos, new MetricSnapshots(new PrometheusTextFormatParser().parse(source))); - assertThat(baos.toString(Charsets.UTF_8)).isEqualTo(expected); + .write(baos, parsedMetrics); + assertThat(baos.toString(Charsets.UTF_8)).isEqualTo(expectedSerialized); } } From 917fb19f8afaf8bf23c24adb828ff9b037588cee Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 17:01:00 +0500 Subject: [PATCH 10/29] StatisticsServiceTest, PrometheusMetricsRetrieverTest --- .../ui/model/MetricsScrapeProperties.java | 2 +- .../ui/service/metrics/SummarizedMetrics.java | 1 + .../inferred/InferredMetricsScraper.java | 4 +- .../PrometheusTextFormatParser.java | 2 +- .../sink/PrometheusPushGatewaySink.java | 1 - .../ui/service/StatisticsServiceTest.java | 77 +++++++++++++++++++ .../inferred/InferredMetricsScraperTest.java | 2 +- .../PrometheusMetricsRetrieverTest.java | 39 ++++++++++ 8 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java create mode 100644 api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java diff --git a/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java b/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java index 806a2e192..171e14e09 100644 --- a/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java +++ b/api/src/main/java/io/kafbat/ui/model/MetricsScrapeProperties.java @@ -12,7 +12,7 @@ import lombok.Value; @Value -@Builder(access = AccessLevel.PRIVATE) +@Builder public class MetricsScrapeProperties { public static final String JMX_METRICS_TYPE = "JMX"; public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java index 63fac5130..8e85a876e 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -21,6 +21,7 @@ import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +@Deprecated //used for api backward-compatibility @RequiredArgsConstructor public class SummarizedMetrics { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java index 3ef7ca8fd..13550e5a4 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -125,8 +125,8 @@ private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterSta state.description().partitions().size() ); state.endOffsets().forEach((partition, endOffset) -> registry.gauge( - "kafka_topic_partition_current_offset", - "Current Offset of a Broker at Topic/Partition", + "kafka_topic_partition_next_offset", + "Current (next) Offset of a Broker at Topic/Partition", List.of("topic", "partition"), List.of(topicName, String.valueOf(partition)), endOffset diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index fd06d765c..071e5f64d 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -101,7 +101,7 @@ private void parseMetricLine(String line, ParsingContext cxt) { String timestampString = m.group(4); cxt.dataPoint( new ParsedDataPoint( - metricName, + PrometheusNaming.sanitizeMetricName(metricName), Optional.ofNullable(labelsString).map(this::parseLabels).orElse(Labels.EMPTY), parseDouble(valueString), Optional.ofNullable(timestampString).map(Long::parseLong).orElse(0L))); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java index a94880e2d..16a66a931 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/sink/PrometheusPushGatewaySink.java @@ -25,7 +25,6 @@ static PrometheusPushGatewaySink create(String url, PushGateway.Builder builder = PushGateway.builder() .address(url); - if (hasText(username) && hasText(passw)) { builder.basicAuth(username, passw); } diff --git a/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java b/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java new file mode 100644 index 000000000..a301b7e93 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java @@ -0,0 +1,77 @@ +package io.kafbat.ui.service; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.kafbat.ui.AbstractIntegrationTest; +import io.kafbat.ui.model.CreateTopicMessageDTO; +import io.kafbat.ui.model.Statistics; +import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; +import java.util.List; +import java.util.UUID; +import org.apache.kafka.clients.admin.NewTopic; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class StatisticsServiceTest extends AbstractIntegrationTest { + + @Autowired + private MessagesService messagesService; + + @Autowired + private ClustersStorage clustersStorage; + + @Autowired + private StatisticsService statisticsService; + + @Test + void testInferredMetricsCollected() { + var newTopicName = "interred_metrics_" + UUID.randomUUID(); + createTopic(new NewTopic(newTopicName, 2, (short) 1)); + for (int i = 0; i < 4; i++) { + messagesService.sendMessage( + clustersStorage.getClusterByName(LOCAL).get(), + newTopicName, + new CreateTopicMessageDTO() + .key(UUID.randomUUID().toString()) + .value(UUID.randomUUID().toString()) + .partition(0) + .keySerde("String") + .valueSerde("String") + ).block(); + } + + Statistics updated = + statisticsService.updateCache(clustersStorage.getClusterByName(LOCAL).get()) + .block(); + + var kafkaTopicPartitionsGauge = getGaugeSnapshot( + updated.getMetrics().getInferredMetrics(), + "kafka_topic_partitions", + Labels.of("topic", newTopicName) + ); + assertThat(kafkaTopicPartitionsGauge.getValue()) + .isEqualTo(2); + + var kafkaTopicPartitionNextOffset = getGaugeSnapshot( + updated.getMetrics().getInferredMetrics(), + "kafka_topic_partition_next_offset", + Labels.of("topic", newTopicName, "partition", "0") + ); + assertThat(kafkaTopicPartitionNextOffset.getValue()) + .isEqualTo(4); + } + + private GaugeDataPointSnapshot getGaugeSnapshot(InferredMetrics inferredMetrics, + String metricName, + Labels labels) { + return inferredMetrics.asStream() + .filter(s -> s.getMetadata().getName().equals(metricName) && s instanceof GaugeSnapshot) + .flatMap(s -> ((List) s.getDataPoints()).stream()) + .filter(dp -> dp.getLabels().equals(labels)) + .findFirst() + .orElseThrow(); + } +} diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java index 191de675c..1141aa4bb 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraperTest.java @@ -100,7 +100,7 @@ void allExpectedMetricsScraped() { "broker_bytes", "topic_count", "kafka_topic_partitions", - "kafka_topic_partition_current_offset", + "kafka_topic_partition_next_offset", "kafka_topic_partition_oldest_offset", "kafka_topic_partition_in_sync_replica", "kafka_topic_partition_replicas", diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java new file mode 100644 index 000000000..516ec2176 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetrieverTest.java @@ -0,0 +1,39 @@ +package io.kafbat.ui.service.metrics.scrape.prometheus; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.kafbat.ui.container.PrometheusContainer; +import io.kafbat.ui.model.MetricsScrapeProperties; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +class PrometheusMetricsRetrieverTest { + + @Container + private static final PrometheusContainer PROMETHEUS = new PrometheusContainer(); + + @Test + void testPrometheusMetricsParsedFromEndpoint() { + List retrieved = new PrometheusMetricsRetriever( + MetricsScrapeProperties.builder() + .port(PROMETHEUS.getMappedPort(9090)) + .ssl(false) + .build() + ).retrieve(PROMETHEUS.getHost()).block(); + + assertThat(retrieved) + .map(m -> m.getMetadata().getName()) + .containsAll( + List.of( + "go_gc_cycles_automatic_gc_cycles", //counter + "go_gc_duration_seconds", //histogram + "go_gc_gogc_percent" //gauge + ) + ); + } + +} From 1901ab5c14446542274f85853263b4120895fe3d Mon Sep 17 00:00:00 2001 From: iliax Date: Mon, 21 Jul 2025 18:12:56 +0500 Subject: [PATCH 11/29] StatisticsServiceTest, PrometheusMetricsRetrieverTest fix --- .../scrape/prometheus/PrometheusMetricsRetriever.java | 5 ++++- .../scrape/prometheus/PrometheusTextFormatParser.java | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java index 13fe5f419..6a53ad9b1 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusMetricsRetriever.java @@ -48,6 +48,9 @@ Mono> retrieve(String host) { .bodyToMono(String.class) .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) .map(body -> new PrometheusTextFormatParser().parse(body)) - .onErrorResume(th -> Mono.just(List.of())); + .onErrorResume(th -> { + log.warn("Error while getting prometheus metrics from {}", host, th); + return Mono.just(List.of()); + }); } } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 071e5f64d..9ecd9fa41 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -101,7 +101,7 @@ private void parseMetricLine(String line, ParsingContext cxt) { String timestampString = m.group(4); cxt.dataPoint( new ParsedDataPoint( - PrometheusNaming.sanitizeMetricName(metricName), + metricName, Optional.ofNullable(labelsString).map(this::parseLabels).orElse(Labels.EMPTY), parseDouble(valueString), Optional.ofNullable(timestampString).map(Long::parseLong).orElse(0L))); @@ -180,7 +180,7 @@ void metricNameAndHelp(String metricName, String help) { void dataPoint(ParsedDataPoint parsedDataPoint) { if (currentMetricName == null) { - currentMetricName = parsedDataPoint.name; + currentMetricName = PrometheusNaming.sanitizeMetricName(parsedDataPoint.name); } if (dataPoints == null) { dataPoints = new UntypedDataPointsAccumulator(); From 6b68b82990f3ea14acbd3f4099ebe57d444ae248 Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 13:15:24 +0300 Subject: [PATCH 12/29] Fixed sonar errors --- .../service/graphs/PromQueryLangGrammar.java | 3 ++ .../ui/service/metrics/SummarizedMetrics.java | 2 +- .../metrics/scrape/IoRatesMetricsScanner.java | 15 +++++---- .../inferred/InferredMetricsScraper.java | 33 ++++++++++--------- .../PrometheusTextFormatParser.java | 21 +++++++----- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java index 7d3db458c..e8959c571 100644 --- a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java +++ b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryLangGrammar.java @@ -10,6 +10,9 @@ class PromQueryLangGrammar { + private PromQueryLangGrammar() { + } + // returns error msg, or empty if query is valid static Optional validateExpression(String query) { try { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java index 8e85a876e..cfe637e5a 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -21,7 +21,7 @@ import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -@Deprecated //used for api backward-compatibility +@Deprecated(forRemoval = false) //used for api backward-compatibility @RequiredArgsConstructor public class SummarizedMetrics { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java index ed1968104..899dfac17 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java @@ -12,11 +12,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Stream; // Scans external jmx/prometheus metric and tries to infer io rates class IoRatesMetricsScanner { + public static final String BROKER_TOPIC_METRICS_SUFFIX = "BrokerTopicMetrics"; + public static final String FIFTEEN_MINUTE_RATE_SUFFIX = "FifteenMinuteRate"; // per broker final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); final Map brokerBytesOutFifteenMinuteRate = new HashMap<>(); @@ -59,23 +60,23 @@ private void updateBrokerIOrates(int nodeId, String name, Labels labels, double if (!brokerBytesInFifteenMinuteRate.containsKey(nodeId) && labels.size() == 1 && "BytesInPerSec".equalsIgnoreCase(labels.getValue(0)) - && containsIgnoreCase(name, "BrokerTopicMetrics") - && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + && containsIgnoreCase(name, BROKER_TOPIC_METRICS_SUFFIX) + && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { brokerBytesInFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(value)); } if (!brokerBytesOutFifteenMinuteRate.containsKey(nodeId) && labels.size() == 1 && "BytesOutPerSec".equalsIgnoreCase(labels.getValue(0)) - && containsIgnoreCase(name, "BrokerTopicMetrics") - && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + && containsIgnoreCase(name, BROKER_TOPIC_METRICS_SUFFIX) + && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { brokerBytesOutFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(value)); } } private void updateTopicsIOrates(String name, Labels labels, double value) { if (labels.contains("topic") - && containsIgnoreCase(name, "BrokerTopicMetrics") - && endsWithIgnoreCase(name, "FifteenMinuteRate")) { + && containsIgnoreCase(name, BROKER_TOPIC_METRICS_SUFFIX) + && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { String topic = labels.get("topic"); if (labels.contains("name")) { var nameLblVal = labels.get("name"); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java index 13550e5a4..834fe9993 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -23,6 +23,9 @@ @RequiredArgsConstructor public class InferredMetricsScraper { + public static final String NODE_ID_TAG = "node_id"; + public static final String TOPIC_TAG = "topic"; + public static final String GROUP_TAG = "group"; private ScrapedClusterState prevState = null; public synchronized Mono scrape(ScrapedClusterState newState) { @@ -79,7 +82,7 @@ private static void fillNodesMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "broker_bytes_disk", "Written disk size in bytes of a broker", - List.of("node_id"), + List.of(NODE_ID_TAG), List.of(nodeId.toString()), state.segmentStats().getSegmentSize() ); @@ -89,7 +92,7 @@ private static void fillNodesMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "broker_bytes_usable", "Usable disk size in bytes of a broker", - List.of("node_id"), + List.of(NODE_ID_TAG), List.of(nodeId.toString()), state.logDirSpaceStats().usableBytes() ); @@ -98,7 +101,7 @@ private static void fillNodesMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "broker_bytes_total", "Total disk size in bytes of a broker", - List.of("node_id"), + List.of(NODE_ID_TAG), List.of(nodeId.toString()), state.logDirSpaceStats().totalBytes() ); @@ -120,21 +123,21 @@ private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "kafka_topic_partitions", "Number of partitions for this Topic", - List.of("topic"), + List.of(TOPIC_TAG), List.of(topicName), state.description().partitions().size() ); state.endOffsets().forEach((partition, endOffset) -> registry.gauge( "kafka_topic_partition_next_offset", "Current (next) Offset of a Broker at Topic/Partition", - List.of("topic", "partition"), + List.of(TOPIC_TAG, "partition"), List.of(topicName, String.valueOf(partition)), endOffset )); state.startOffsets().forEach((partition, startOffset) -> registry.gauge( "kafka_topic_partition_oldest_offset", "Oldest Offset of a Broker at Topic/Partition", - List.of("topic", "partition"), + List.of(TOPIC_TAG, "partition"), List.of(topicName, String.valueOf(partition)), startOffset )); @@ -142,21 +145,21 @@ private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "kafka_topic_partition_in_sync_replica", "Number of In-Sync Replicas for this Topic/Partition", - List.of("topic", "partition"), + List.of(TOPIC_TAG, "partition"), List.of(topicName, String.valueOf(p.partition())), p.isr().size() ); registry.gauge( "kafka_topic_partition_replicas", "Number of Replicas for this Topic/Partition", - List.of("topic", "partition"), + List.of(TOPIC_TAG, "partition"), List.of(topicName, String.valueOf(p.partition())), p.replicas().size() ); registry.gauge( "kafka_topic_partition_leader", "Leader Broker ID of this Topic/Partition (-1, if no leader)", - List.of("topic", "partition"), + List.of(TOPIC_TAG, "partition"), List.of(topicName, String.valueOf(p.partition())), Optional.ofNullable(p.leader()).map(Node::id).orElse(-1) ); @@ -165,7 +168,7 @@ private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "topic_bytes_disk", "Disk size in bytes of a topic", - List.of("topic"), + List.of(TOPIC_TAG), List.of(topicName), state.segmentStats().getSegmentSize() ); @@ -186,21 +189,21 @@ private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedC registry.gauge( "group_state", "State of the consumer group, value = ordinal of org.apache.kafka.common.ConsumerGroupState", - List.of("group"), + List.of(GROUP_TAG), List.of(groupName), state.description().state().ordinal() ); registry.gauge( "group_member_count", "Number of member assignments in the consumer group.", - List.of("group"), + List.of(GROUP_TAG), List.of(groupName), state.description().members().size() ); registry.gauge( "group_host_count", "Number of distinct hosts in the consumer group.", - List.of("group"), + List.of(GROUP_TAG), List.of(groupName), state.description().members().stream().map(MemberDescription::host).distinct().count() ); @@ -209,7 +212,7 @@ private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedC registry.gauge( "kafka_consumergroup_current_offset", "Current Offset of a ConsumerGroup at Topic/Partition", - List.of("consumergroup", "topic", "partition"), + List.of("consumergroup", TOPIC_TAG, "partition"), List.of(groupName, tp.topic(), String.valueOf(tp.partition())), committedOffset ); @@ -220,7 +223,7 @@ private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedC registry.gauge( "kafka_consumergroup_lag", "Current Approximate Lag of a ConsumerGroup at Topic/Partition", - List.of("consumergroup", "topic", "partition"), + List.of("consumergroup", TOPIC_TAG, "partition"), List.of(groupName, tp.topic(), String.valueOf(tp.partition())), endOffset - committedOffset //TODO: check +-1 )); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 9ecd9fa41..e0e0f7b15 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -51,6 +51,7 @@ public class PrometheusTextFormatParser { Pattern.compile("^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (counter|gauge|histogram|summary|untyped)"); private static final Pattern LABEL_PATTERN = Pattern.compile("([a-zA-Z_:][a-zA-Z0-9_:]*)=\"((?:\\\\\"|\\\\\\\\|\\\\n|[^\"])*)\""); + public static final String QUANTILE_LABEL = "quantile"; private record ParsedDataPoint(String name, Labels labels, double value, Long scrapedAt) { } @@ -258,18 +259,20 @@ public Optional buildSnapshot(String name, @Nullable String help } } - @RequiredArgsConstructor static class CounterDataPointsAccumulator extends UntypedDataPointsAccumulator { - final String name; - final List dataPoints = new ArrayList<>(); + final List counterDataPoints = new ArrayList<>(); + + public CounterDataPointsAccumulator(String name) { + this.name = name; + } @Override public boolean add(ParsedDataPoint dp) { if (!dp.name.equals(name + "_total")) { return false; } - dataPoints.add( + counterDataPoints.add( CounterDataPointSnapshot.builder() .labels(dp.labels).value(dp.value).scrapeTimestampMillis(dp.scrapedAt).build()); return true; @@ -277,11 +280,11 @@ public boolean add(ParsedDataPoint dp) { @Override public Optional buildSnapshot(String name, @Nullable String help) { - if (dataPoints.isEmpty()) { + if (counterDataPoints.isEmpty()) { return Optional.empty(); } var builder = CounterSnapshot.builder().name(name).help(help); - dataPoints.forEach(builder::dataPoint); + counterDataPoints.forEach(builder::dataPoint); return Optional.of(builder.build()); } } @@ -359,9 +362,9 @@ static class SummaryDataPointsAccumulator implements MetricDataPointsAccumulator @Override public boolean add(ParsedDataPoint dp) { - if (dp.name.equals(name) && dp.labels.contains("quantile")) { - var histLbls = rmLabel(dp.labels, "quantile"); - quantiles.put(histLbls, new Quantile(parseDouble(dp.labels.get("quantile")), dp.value)); + if (dp.name.equals(name) && dp.labels.contains(QUANTILE_LABEL)) { + var histLbls = rmLabel(dp.labels, QUANTILE_LABEL); + quantiles.put(histLbls, new Quantile(parseDouble(dp.labels.get(QUANTILE_LABEL)), dp.value)); return true; } if (dp.name.equals(name + "_count")) { From db1f493382dc982e0e2aa10272eb14bda8f76755 Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 13:20:17 +0300 Subject: [PATCH 13/29] Fixed sonar errors --- .../scrape/prometheus/PrometheusTextFormatParser.java | 4 ++-- api/src/main/java/io/kafbat/ui/util/MetricsUtils.java | 3 +++ .../test/java/io/kafbat/ui/service/metrics/MetricsUtils.java | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index e0e0f7b15..9cc1bb941 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -384,11 +384,11 @@ public Optional buildSnapshot(String name, @Nullable String help return Optional.empty(); } var builder = SummarySnapshot.builder().name(name).help(help); - quantiles.asMap().forEach((labels, quantiles) -> { + quantiles.asMap().forEach((labels, localQuantiles) -> { builder.dataPoint( SummarySnapshot.SummaryDataPointSnapshot.builder() .labels(labels) - .quantiles(Quantiles.of(new ArrayList<>(quantiles))) + .quantiles(Quantiles.of(new ArrayList<>(localQuantiles))) .sum(sums.getOrDefault(labels, Double.NaN)) .count(counts.getOrDefault(labels, 0L)) .build() diff --git a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java index be40307f6..3b4a22024 100644 --- a/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java +++ b/api/src/main/java/io/kafbat/ui/util/MetricsUtils.java @@ -20,6 +20,9 @@ public final class MetricsUtils { + private MetricsUtils() { + } + public static double readPointValue(DataPointSnapshot dps) { return switch (dps) { case UnknownDataPointSnapshot unknown -> unknown.getValue(); diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java index e94be1bf4..d43e21349 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java @@ -8,12 +8,15 @@ import java.util.Optional; public class MetricsUtils { + + private MetricsUtils() { + } + public static boolean isTheSameMetric(MetricSnapshot m1, MetricSnapshot m2) { if (m1.getClass().equals(m2.getClass())) { MetricMetadata metadata1 = m1.getMetadata(); MetricMetadata metadata2 = m2.getMetadata(); if ( - metadata1.getName().equals(metadata2.getName()) && metadata1.getHelp().equals(metadata2.getHelp()) && Optional.ofNullable( From 14a11e374c0304128c070816bc7fbdf58c473f40 Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 13:29:07 +0300 Subject: [PATCH 14/29] Fixed sonar errors --- .../service/metrics/scrape/jmx/JmxMetricsFormatter.java | 3 ++- .../scrape/prometheus/PrometheusTextFormatParser.java | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index a9f50b610..fbcaf9270 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -18,7 +18,8 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 private static final Pattern PROPERTY_PATTERN = Pattern.compile( - "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)"); + "([^,=:\\*\\?]+)=(\"(?>[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" + ); public static List constructMetricsList(ObjectName jmxMetric, MBeanAttributeInfo[] attributes, diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 9cc1bb941..1139d88aa 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -39,11 +39,12 @@ public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( - "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name - + "(?:\\{([^}]*)\\})?" // Optional labels (content in group 2) + "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Group 1: Metric name + + "(?:\\{(?>[^}]*)\\})?" // Group 2: Atomic label content + "\\s+" - + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Value (group 3) - + "(?:\\s+([0-9]+))?$"); // Optional timestamp (group 4) + + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Group 3: Value + + "(?:\\s+([0-9]+))?$"); // Group 4: Optional timestamp + private static final Pattern HELP_PATTERN = Pattern.compile("^# HELP ([a-zA-Z_:][a-zA-Z0-9_:]*) (.*)"); From bb3aee20a4cece508cfa2cf721468bf154767edb Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 13:31:58 +0300 Subject: [PATCH 15/29] Fixed sonar errors --- .../scrape/prometheus/PrometheusTextFormatParser.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 1139d88aa..ccd9f35a0 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -39,11 +39,12 @@ public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( - "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Group 1: Metric name - + "(?:\\{(?>[^}]*)\\})?" // Group 2: Atomic label content - + "\\s+" - + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Group 3: Value - + "(?:\\s+([0-9]+))?$"); // Group 4: Optional timestamp + "^([a-zA-Z_:][a-zA-Z0-9_:]*)" + // Metric name + "(?:\\{(?>[^}]*)\\})?" + // Optional labels (atomic group) + "\\s+" + + "(-?(?:Inf|NaN|(?:\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)))" + // Metric value + "(?:\\s+([0-9]+))?$" // Optional timestamp + ); // Group 4: Optional timestamp private static final Pattern HELP_PATTERN = From 1f6606d1d62a22507c7913ce388296c251286472 Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 13:33:22 +0300 Subject: [PATCH 16/29] Fixed sonar errors --- .../metrics/scrape/prometheus/PrometheusTextFormatParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index ccd9f35a0..c5bc4eb91 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -52,7 +52,7 @@ public class PrometheusTextFormatParser { private static final Pattern TYPE_PATTERN = Pattern.compile("^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (counter|gauge|histogram|summary|untyped)"); private static final Pattern LABEL_PATTERN = - Pattern.compile("([a-zA-Z_:][a-zA-Z0-9_:]*)=\"((?:\\\\\"|\\\\\\\\|\\\\n|[^\"])*)\""); + Pattern.compile("([a-zA-Z_:][a-zA-Z0-9_:]*)=\"((?>\\\\\"|\\\\\\\\|\\\\n|[^\"])*)\""); public static final String QUANTILE_LABEL = "quantile"; private record ParsedDataPoint(String name, Labels labels, double value, Long scrapedAt) { From 5043b4c249f3bc4b3178af848ce2c2b5f0da8b3c Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 13:44:37 +0300 Subject: [PATCH 17/29] Fixed sonar errors --- .../inferred/InferredMetricsScraper.java | 15 +++++----- .../scrape/jmx/JmxMetricsFormatter.java | 2 +- .../scrape/jmx/JmxMetricsRetriever.java | 28 ++++++++++++++----- .../PrometheusTextFormatParser.java | 10 +++---- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java index 834fe9993..5f523f2e3 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -26,6 +26,7 @@ public class InferredMetricsScraper { public static final String NODE_ID_TAG = "node_id"; public static final String TOPIC_TAG = "topic"; public static final String GROUP_TAG = "group"; + public static final String PARTITION_TAG = "partition"; private ScrapedClusterState prevState = null; public synchronized Mono scrape(ScrapedClusterState newState) { @@ -130,14 +131,14 @@ private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterSta state.endOffsets().forEach((partition, endOffset) -> registry.gauge( "kafka_topic_partition_next_offset", "Current (next) Offset of a Broker at Topic/Partition", - List.of(TOPIC_TAG, "partition"), + List.of(TOPIC_TAG, PARTITION_TAG), List.of(topicName, String.valueOf(partition)), endOffset )); state.startOffsets().forEach((partition, startOffset) -> registry.gauge( "kafka_topic_partition_oldest_offset", "Oldest Offset of a Broker at Topic/Partition", - List.of(TOPIC_TAG, "partition"), + List.of(TOPIC_TAG, PARTITION_TAG), List.of(topicName, String.valueOf(partition)), startOffset )); @@ -145,21 +146,21 @@ private static void fillTopicMetrics(MetricsRegistry registry, ScrapedClusterSta registry.gauge( "kafka_topic_partition_in_sync_replica", "Number of In-Sync Replicas for this Topic/Partition", - List.of(TOPIC_TAG, "partition"), + List.of(TOPIC_TAG, PARTITION_TAG), List.of(topicName, String.valueOf(p.partition())), p.isr().size() ); registry.gauge( "kafka_topic_partition_replicas", "Number of Replicas for this Topic/Partition", - List.of(TOPIC_TAG, "partition"), + List.of(TOPIC_TAG, PARTITION_TAG), List.of(topicName, String.valueOf(p.partition())), p.replicas().size() ); registry.gauge( "kafka_topic_partition_leader", "Leader Broker ID of this Topic/Partition (-1, if no leader)", - List.of(TOPIC_TAG, "partition"), + List.of(TOPIC_TAG, PARTITION_TAG), List.of(topicName, String.valueOf(p.partition())), Optional.ofNullable(p.leader()).map(Node::id).orElse(-1) ); @@ -212,7 +213,7 @@ private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedC registry.gauge( "kafka_consumergroup_current_offset", "Current Offset of a ConsumerGroup at Topic/Partition", - List.of("consumergroup", TOPIC_TAG, "partition"), + List.of("consumergroup", TOPIC_TAG, PARTITION_TAG), List.of(groupName, tp.topic(), String.valueOf(tp.partition())), committedOffset ); @@ -223,7 +224,7 @@ private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedC registry.gauge( "kafka_consumergroup_lag", "Current Approximate Lag of a ConsumerGroup at Topic/Partition", - List.of("consumergroup", TOPIC_TAG, "partition"), + List.of("consumergroup", TOPIC_TAG, PARTITION_TAG), List.of(groupName, tp.topic(), String.valueOf(tp.partition())), endOffset - committedOffset //TODO: check +-1 )); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index fbcaf9270..2b367d55a 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -18,7 +18,7 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 private static final Pattern PROPERTY_PATTERN = Pattern.compile( - "([^,=:\\*\\?]+)=(\"(?>[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" + "([^,=:\\*\\?]+)=(\"(?:\\\\.|[^\\\\\"])*\"|[^,=:\"]*)" ); public static List constructMetricsList(ObjectName jmxMetric, diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java index ef8614be9..81424138b 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java @@ -71,21 +71,35 @@ private void withJmxConnector(String jmxUrl, MetricsScrapeProperties scrapeProperties, Consumer consumer) { var env = prepareJmxEnvAndSetThreadLocal(scrapeProperties); - try (JMXConnector connector = JMXConnectorFactory.newJMXConnector(new JMXServiceURL(jmxUrl), env)) { - try { - connector.connect(env); - } catch (Exception exception) { - log.error("Error connecting to {}", jmxUrl, exception); + JMXServiceURL serviceUrl; + try { + serviceUrl = new JMXServiceURL(jmxUrl); + } catch (java.net.MalformedURLException e) { + log.error("Malformed JMX URL: {}", jmxUrl, e); + return; + } + try (JMXConnector connector = JMXConnectorFactory.newJMXConnector(serviceUrl, env)) { + if (!tryConnect(connector, env, jmxUrl)) { return; } consumer.accept(connector); - } catch (Exception e) { - log.error("Error getting jmx metrics from {}", jmxUrl, e); + } catch (Exception connectorException) { + log.error("Error getting jmx metrics from {}", jmxUrl, connectorException); } finally { JmxSslSocketFactory.clearThreadLocalContext(); } } + private boolean tryConnect(JMXConnector connector, Map env, String jmxUrl) { + try { + connector.connect(env); + return true; + } catch (Exception connectException) { + log.error("Error connecting to {}", jmxUrl, connectException); + return false; + } + } + private Map prepareJmxEnvAndSetThreadLocal(MetricsScrapeProperties scrapeProperties) { Map env = new HashMap<>(); if (isSslJmxEndpoint(scrapeProperties)) { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index c5bc4eb91..fe0509ad2 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -39,11 +39,11 @@ public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( - "^([a-zA-Z_:][a-zA-Z0-9_:]*)" + // Metric name - "(?:\\{(?>[^}]*)\\})?" + // Optional labels (atomic group) - "\\s+" + - "(-?(?:Inf|NaN|(?:\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)))" + // Metric value - "(?:\\s+([0-9]+))?$" // Optional timestamp + "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name + + "(?:\\{(?>[^}]*)\\})?" // Optional labels (atomic group) + + "\\s+" + + "(-?(?:Inf|NaN|(?:\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)))" // Metric value + + "(?:\\s+([0-9]+))?$" // Optional timestamp ); // Group 4: Optional timestamp From 69a755d49925fdc0b39680c616b6de40c052bc70 Mon Sep 17 00:00:00 2001 From: German Osin Date: Fri, 25 Jul 2025 14:50:16 +0300 Subject: [PATCH 18/29] Fix warnings --- api/build.gradle | 1 + .../ui/client/RetryingKafkaConnectClient.java | 5 +- .../ui/controller/MessagesController.java | 2 +- .../ui/controller/SchemasController.java | 4 +- .../ui/controller/TopicsController.java | 3 +- .../io/kafbat/ui/emitter/MessageFilters.java | 8 +- .../io/kafbat/ui/emitter/OffsetsInfo.java | 2 +- .../io/kafbat/ui/emitter/SeekOperations.java | 2 +- .../io/kafbat/ui/mapper/ClusterMapper.java | 1 - .../kafbat/ui/mapper/DynamicConfigMapper.java | 7 ++ .../ui/service/ConsumerGroupService.java | 4 +- .../ui/service/KafkaConnectService.java | 4 +- .../ui/service/PollingCursorsStorage.java | 2 +- .../ui/service/graphs/PromQueryTemplate.java | 4 +- .../ui/service/masking/policies/Mask.java | 13 +-- .../ui/service/masking/policies/Remove.java | 9 +- .../ui/service/masking/policies/Replace.java | 13 +-- .../ui/service/metrics/SummarizedMetrics.java | 2 +- .../metrics/scrape/IoRatesMetricsScanner.java | 16 ++-- .../scrape/jmx/JmxSslSocketFactory.java | 4 +- .../PrometheusTextFormatParser.java | 17 ++-- .../ui/util/DynamicConfigOperations.java | 2 +- .../util/jsonschema/JsonAvroConversion.java | 4 +- .../io/kafbat/ui/KafkaConsumerGroupTests.java | 2 +- .../kafbat/ui/emitter/MessageFiltersTest.java | 4 +- .../ui/service/StatisticsServiceTest.java | 1 + .../analyze/TopicAnalysisServiceTest.java | 4 +- .../ui/service/metrics/MetricsUtils.java | 83 +++++++++++-------- .../io/kafbat/ui/util/ContentUtilsTest.java | 2 +- gradle/libs.versions.toml | 2 + .../java/io/kafbat/ui/serde/api/Serde.java | 5 ++ 31 files changed, 136 insertions(+), 96 deletions(-) diff --git a/api/build.gradle b/api/build.gradle index 56d4d7aeb..c0cca8fd8 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -33,6 +33,7 @@ dependencies { implementation libs.apache.avro implementation libs.apache.commons + implementation libs.apache.commons.text implementation libs.apache.commons.pool2 implementation libs.apache.datasketches diff --git a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java index 30b664da1..8a03c3dee 100644 --- a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java +++ b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java @@ -1,5 +1,7 @@ package io.kafbat.ui.client; +import static org.apache.commons.lang3.Strings.CI; + import com.fasterxml.jackson.annotation.JsonProperty; import io.kafbat.ui.config.ClustersProperties; import io.kafbat.ui.connect.ApiClient; @@ -22,7 +24,6 @@ import java.util.Objects; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.http.ResponseEntity; import org.springframework.util.unit.DataSize; import org.springframework.web.client.RestClientException; @@ -58,7 +59,7 @@ private static Retry conflictCodeRetry() { if (e instanceof WebClientResponseException.InternalServerError exception) { final var errorMessage = getMessage(exception); - return StringUtils.equals(errorMessage, + return CI.equals(errorMessage, // From https://github.com/apache/kafka/blob/dfc07e0e0c6e737a56a5402644265f634402b864/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java#L2340 "Request cannot be completed because a rebalance is expected"); } diff --git a/api/src/main/java/io/kafbat/ui/controller/MessagesController.java b/api/src/main/java/io/kafbat/ui/controller/MessagesController.java index b99fa2aba..6f6b3e455 100644 --- a/api/src/main/java/io/kafbat/ui/controller/MessagesController.java +++ b/api/src/main/java/io/kafbat/ui/controller/MessagesController.java @@ -75,7 +75,7 @@ public Mono> executeSmartFilte .map(ResponseEntity::ok); } - @Deprecated + @Deprecated(forRemoval = true, since = "1.1.0") @Override public Mono>> getTopicMessages(String clusterName, String topicName, diff --git a/api/src/main/java/io/kafbat/ui/controller/SchemasController.java b/api/src/main/java/io/kafbat/ui/controller/SchemasController.java index 6f73d3525..a34110031 100644 --- a/api/src/main/java/io/kafbat/ui/controller/SchemasController.java +++ b/api/src/main/java/io/kafbat/ui/controller/SchemasController.java @@ -1,5 +1,7 @@ package io.kafbat.ui.controller; +import static org.apache.commons.lang3.Strings.CI; + import io.kafbat.ui.api.SchemasApi; import io.kafbat.ui.exception.ValidationException; import io.kafbat.ui.mapper.KafkaSrMapper; @@ -222,7 +224,7 @@ public Mono> getSchemas(String cluster int subjectToSkip = ((pageNum != null && pageNum > 0 ? pageNum : 1) - 1) * pageSize; List filteredSubjects = subjects .stream() - .filter(subj -> search == null || StringUtils.containsIgnoreCase(subj, search)) + .filter(subj -> search == null || CI.contains(subj, search)) .sorted().toList(); var totalPages = (filteredSubjects.size() / pageSize) + (filteredSubjects.size() % pageSize == 0 ? 0 : 1); diff --git a/api/src/main/java/io/kafbat/ui/controller/TopicsController.java b/api/src/main/java/io/kafbat/ui/controller/TopicsController.java index 208ca59cb..b72a4e395 100644 --- a/api/src/main/java/io/kafbat/ui/controller/TopicsController.java +++ b/api/src/main/java/io/kafbat/ui/controller/TopicsController.java @@ -7,6 +7,7 @@ import static io.kafbat.ui.model.rbac.permission.TopicAction.EDIT; import static io.kafbat.ui.model.rbac.permission.TopicAction.VIEW; import static java.util.stream.Collectors.toList; +import static org.apache.commons.lang3.Strings.CI; import io.kafbat.ui.api.TopicsApi; import io.kafbat.ui.mapper.ClusterMapper; @@ -190,7 +191,7 @@ public Mono> getTopics(String clusterName, List filtered = topics.stream() .filter(topic -> !topic.isInternal() || showInternal != null && showInternal) - .filter(topic -> search == null || StringUtils.containsIgnoreCase(topic.getName(), search)) + .filter(topic -> search == null || CI.contains(topic.getName(), search)) .sorted(comparator) .toList(); var totalPages = (filtered.size() / pageSize) diff --git a/api/src/main/java/io/kafbat/ui/emitter/MessageFilters.java b/api/src/main/java/io/kafbat/ui/emitter/MessageFilters.java index fbbc84ab3..1500a7696 100644 --- a/api/src/main/java/io/kafbat/ui/emitter/MessageFilters.java +++ b/api/src/main/java/io/kafbat/ui/emitter/MessageFilters.java @@ -1,6 +1,7 @@ package io.kafbat.ui.emitter; import static java.util.Collections.emptyMap; +import static org.apache.commons.lang3.Strings.CS; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -35,7 +36,6 @@ import javax.annotation.Nullable; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; @Slf4j @UtilityClass @@ -55,8 +55,8 @@ public static Predicate noop() { } public static Predicate containsStringFilter(String string) { - return msg -> StringUtils.contains(msg.getKey(), string) - || StringUtils.contains(msg.getValue(), string) || headersContains(msg, string); + return msg -> CS.contains(msg.getKey(), string) + || CS.contains(msg.getValue(), string) || headersContains(msg, string); } private static boolean headersContains(TopicMessageDTO msg, String searchString) { @@ -67,7 +67,7 @@ private static boolean headersContains(TopicMessageDTO msg, String searchString) } for (final var entry : headers.entrySet()) { - if (StringUtils.contains(entry.getKey(), searchString) || StringUtils.contains(entry.getValue(), searchString)) { + if (CS.contains(entry.getKey(), searchString) || CS.contains(entry.getValue(), searchString)) { return true; } } diff --git a/api/src/main/java/io/kafbat/ui/emitter/OffsetsInfo.java b/api/src/main/java/io/kafbat/ui/emitter/OffsetsInfo.java index 7f34e1708..1d0634c42 100644 --- a/api/src/main/java/io/kafbat/ui/emitter/OffsetsInfo.java +++ b/api/src/main/java/io/kafbat/ui/emitter/OffsetsInfo.java @@ -83,7 +83,7 @@ boolean assignedPartitionsFullyPolled() { long summaryOffsetsRange() { MutableLong cnt = new MutableLong(); nonEmptyPartitions.forEach(tp -> cnt.add(endOffsets.get(tp) - beginOffsets.get(tp))); - return cnt.getValue(); + return cnt.get().longValue(); } public Set allTargetPartitions() { diff --git a/api/src/main/java/io/kafbat/ui/emitter/SeekOperations.java b/api/src/main/java/io/kafbat/ui/emitter/SeekOperations.java index 87f29102c..b9135b495 100644 --- a/api/src/main/java/io/kafbat/ui/emitter/SeekOperations.java +++ b/api/src/main/java/io/kafbat/ui/emitter/SeekOperations.java @@ -55,7 +55,7 @@ public long summaryOffsetsRange() { public long offsetsProcessedFromSeek() { MutableLong count = new MutableLong(); offsetsForSeek.forEach((tp, initialOffset) -> count.add(consumer.position(tp) - initialOffset)); - return count.getValue(); + return count.get().longValue(); } // Get offsets to seek to. NOTE: offsets do not contain empty partitions offsets diff --git a/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java b/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java index c05ad8aca..28793be33 100644 --- a/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java +++ b/api/src/main/java/io/kafbat/ui/mapper/ClusterMapper.java @@ -61,7 +61,6 @@ public interface ClusterMapper { @Mapping(target = "zooKeeperStatus", ignore = true) ClusterStatsDTO toClusterStats(InternalClusterState clusterState); - @Deprecated default ClusterMetricsDTO toClusterMetrics(Metrics metrics) { return new ClusterMetricsDTO() .items(convert(new SummarizedMetrics(metrics).asStream()).toList()); diff --git a/api/src/main/java/io/kafbat/ui/mapper/DynamicConfigMapper.java b/api/src/main/java/io/kafbat/ui/mapper/DynamicConfigMapper.java index 59861d49e..ebe465660 100644 --- a/api/src/main/java/io/kafbat/ui/mapper/DynamicConfigMapper.java +++ b/api/src/main/java/io/kafbat/ui/mapper/DynamicConfigMapper.java @@ -1,15 +1,18 @@ package io.kafbat.ui.mapper; +import io.kafbat.ui.config.ClustersProperties; import io.kafbat.ui.model.ActionDTO; import io.kafbat.ui.model.ApplicationConfigPropertiesAuthOauth2ResourceServerDTO; import io.kafbat.ui.model.ApplicationConfigPropertiesAuthOauth2ResourceServerJwtDTO; import io.kafbat.ui.model.ApplicationConfigPropertiesAuthOauth2ResourceServerOpaquetokenDTO; import io.kafbat.ui.model.ApplicationConfigPropertiesDTO; +import io.kafbat.ui.model.ApplicationConfigPropertiesKafkaClustersInnerDTO; import io.kafbat.ui.model.ApplicationConfigPropertiesRbacRolesInnerPermissionsInnerDTO; import io.kafbat.ui.model.rbac.Permission; import io.kafbat.ui.util.DynamicConfigOperations; import java.util.Optional; import org.mapstruct.Mapper; +import org.mapstruct.Mapping; import org.springframework.boot.autoconfigure.security.oauth2.resource.OAuth2ResourceServerProperties; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; @@ -19,12 +22,16 @@ public interface DynamicConfigMapper { DynamicConfigOperations.PropertiesStructure fromDto(ApplicationConfigPropertiesDTO dto); + @Mapping(target = "kafka.clusters[].metrics.store", ignore = true) ApplicationConfigPropertiesDTO toDto(DynamicConfigOperations.PropertiesStructure propertiesStructure); default String map(Resource resource) { return resource.getFilename(); } + @Mapping(source = "metrics.store", target = "metrics.store", ignore = true) + ApplicationConfigPropertiesKafkaClustersInnerDTO map(ClustersProperties.Cluster cluster); + default Permission map(ApplicationConfigPropertiesRbacRolesInnerPermissionsInnerDTO perm) { Permission permission = new Permission(); permission.setResource(perm.getResource().getValue()); diff --git a/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java b/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java index 00ea5179a..5fda6d4ce 100644 --- a/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java +++ b/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java @@ -1,5 +1,7 @@ package io.kafbat.ui.service; +import static org.apache.commons.lang3.Strings.CI; + import com.google.common.collect.Streams; import com.google.common.collect.Table; import io.kafbat.ui.emitter.EnhancedConsumer; @@ -115,7 +117,7 @@ public Mono getConsumerGroupsPage( .map(listing -> search == null ? listing : listing.stream() - .filter(g -> StringUtils.containsIgnoreCase(g.groupId(), search)) + .filter(g -> CI.contains(g.groupId(), search)) .toList() ) .flatMapIterable(lst -> lst) diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index 31f552885..9dc03cf7d 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -1,5 +1,7 @@ package io.kafbat.ui.service; +import static org.apache.commons.lang3.Strings.CI; + import io.kafbat.ui.connect.api.KafkaConnectClientApi; import io.kafbat.ui.connect.model.ConnectorStatus; import io.kafbat.ui.connect.model.ConnectorStatusConnector; @@ -80,7 +82,7 @@ private Predicate matchesSearchTerm(@Nullable final String return c -> true; } return connector -> getStringsForSearch(connector) - .anyMatch(string -> StringUtils.containsIgnoreCase(string, search)); + .anyMatch(string -> CI.contains(string, search)); } private Stream getStringsForSearch(FullConnectorInfoDTO fullConnectorInfo) { diff --git a/api/src/main/java/io/kafbat/ui/service/PollingCursorsStorage.java b/api/src/main/java/io/kafbat/ui/service/PollingCursorsStorage.java index 2b760f010..206eeaef7 100644 --- a/api/src/main/java/io/kafbat/ui/service/PollingCursorsStorage.java +++ b/api/src/main/java/io/kafbat/ui/service/PollingCursorsStorage.java @@ -32,7 +32,7 @@ public Optional getCursor(String id) { } public String register(Cursor cursor) { - var id = RandomStringUtils.random(8, true, true); + var id = RandomStringUtils.secure().next(8, true, true); cursorsCache.put(id, cursor); return id; } diff --git a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java index 2aca2e7a3..060b7424e 100644 --- a/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java +++ b/api/src/main/java/io/kafbat/ui/service/graphs/PromQueryTemplate.java @@ -6,7 +6,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; class PromQueryTemplate { @@ -45,7 +45,7 @@ Optional validateSyntax() { } private String replaceParams(Map replacements) { - return new StrSubstitutor(replacements).replace(queryTemplate); + return new StringSubstitutor(replacements).replace(queryTemplate); } } diff --git a/api/src/main/java/io/kafbat/ui/service/masking/policies/Mask.java b/api/src/main/java/io/kafbat/ui/service/masking/policies/Mask.java index 97913164d..5a7210381 100644 --- a/api/src/main/java/io/kafbat/ui/service/masking/policies/Mask.java +++ b/api/src/main/java/io/kafbat/ui/service/masking/policies/Mask.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Preconditions; import java.util.List; +import java.util.Map; import java.util.function.UnaryOperator; class Mask extends MaskingPolicy { @@ -54,15 +55,15 @@ private static UnaryOperator createMasker(List maskingChars) { private JsonNode maskWithFieldsCheck(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); - node.fields().forEachRemaining(f -> { - String fieldName = f.getKey(); - JsonNode fieldVal = f.getValue(); + for (Map.Entry property : node.properties()) { + String fieldName = property.getKey(); + JsonNode fieldVal = property.getValue(); if (fieldShouldBeMasked(fieldName)) { obj.set(fieldName, maskNodeRecursively(fieldVal)); } else { obj.set(fieldName, maskWithFieldsCheck(fieldVal)); } - }); + } return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); @@ -75,7 +76,9 @@ private JsonNode maskWithFieldsCheck(JsonNode node) { private JsonNode maskNodeRecursively(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); - node.fields().forEachRemaining(f -> obj.set(f.getKey(), maskNodeRecursively(f.getValue()))); + for (Map.Entry property : node.properties()) { + obj.set(property.getKey(), maskNodeRecursively(property.getValue())); + } return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); diff --git a/api/src/main/java/io/kafbat/ui/service/masking/policies/Remove.java b/api/src/main/java/io/kafbat/ui/service/masking/policies/Remove.java index 90e01133b..af35900be 100644 --- a/api/src/main/java/io/kafbat/ui/service/masking/policies/Remove.java +++ b/api/src/main/java/io/kafbat/ui/service/masking/policies/Remove.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ContainerNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Map; class Remove extends MaskingPolicy { @@ -25,13 +26,13 @@ public ContainerNode applyToJsonContainer(ContainerNode node) { private JsonNode removeFields(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); - node.fields().forEachRemaining(f -> { - String fieldName = f.getKey(); - JsonNode fieldVal = f.getValue(); + for (Map.Entry property : node.properties()) { + String fieldName = property.getKey(); + JsonNode fieldVal = property.getValue(); if (!fieldShouldBeMasked(fieldName)) { obj.set(fieldName, removeFields(fieldVal)); } - }); + } return obj; } else if (node.isArray()) { var arr = ((ArrayNode) node).arrayNode(node.size()); diff --git a/api/src/main/java/io/kafbat/ui/service/masking/policies/Replace.java b/api/src/main/java/io/kafbat/ui/service/masking/policies/Replace.java index 89bf6304e..cc5ff72c3 100644 --- a/api/src/main/java/io/kafbat/ui/service/masking/policies/Replace.java +++ b/api/src/main/java/io/kafbat/ui/service/masking/policies/Replace.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Preconditions; +import java.util.Map; class Replace extends MaskingPolicy { @@ -31,15 +32,15 @@ public ContainerNode applyToJsonContainer(ContainerNode node) { private JsonNode replaceWithFieldsCheck(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); - node.fields().forEachRemaining(f -> { - String fieldName = f.getKey(); - JsonNode fieldVal = f.getValue(); + for (Map.Entry property : node.properties()) { + String fieldName = property.getKey(); + JsonNode fieldVal = property.getValue(); if (fieldShouldBeMasked(fieldName)) { obj.set(fieldName, replaceRecursive(fieldVal)); } else { obj.set(fieldName, replaceWithFieldsCheck(fieldVal)); } - }); + } return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); @@ -53,7 +54,9 @@ private JsonNode replaceWithFieldsCheck(JsonNode node) { private JsonNode replaceRecursive(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); - node.fields().forEachRemaining(f -> obj.set(f.getKey(), replaceRecursive(f.getValue()))); + for (Map.Entry property : node.properties()) { + obj.set(property.getKey(), replaceRecursive(property.getValue())); + } return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java index cfe637e5a..ef1782a71 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -21,7 +21,7 @@ import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -@Deprecated(forRemoval = false) //used for api backward-compatibility +@Deprecated(forRemoval = true, since = "1.4.0") //used for api backward-compatibility @RequiredArgsConstructor public class SummarizedMetrics { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java index 899dfac17..cc912f978 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/IoRatesMetricsScanner.java @@ -1,7 +1,7 @@ package io.kafbat.ui.service.metrics.scrape; -import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; -import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; +import static org.apache.commons.lang3.Strings.CI; +import static org.apache.commons.lang3.Strings.CS; import io.kafbat.ui.model.Metrics; import io.prometheus.metrics.model.snapshots.GaugeSnapshot; @@ -60,23 +60,23 @@ private void updateBrokerIOrates(int nodeId, String name, Labels labels, double if (!brokerBytesInFifteenMinuteRate.containsKey(nodeId) && labels.size() == 1 && "BytesInPerSec".equalsIgnoreCase(labels.getValue(0)) - && containsIgnoreCase(name, BROKER_TOPIC_METRICS_SUFFIX) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { + && CI.contains(name, BROKER_TOPIC_METRICS_SUFFIX) + && CI.endsWith(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { brokerBytesInFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(value)); } if (!brokerBytesOutFifteenMinuteRate.containsKey(nodeId) && labels.size() == 1 && "BytesOutPerSec".equalsIgnoreCase(labels.getValue(0)) - && containsIgnoreCase(name, BROKER_TOPIC_METRICS_SUFFIX) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { + && CI.contains(name, BROKER_TOPIC_METRICS_SUFFIX) + && CI.endsWith(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { brokerBytesOutFifteenMinuteRate.put(nodeId, BigDecimal.valueOf(value)); } } private void updateTopicsIOrates(String name, Labels labels, double value) { if (labels.contains("topic") - && containsIgnoreCase(name, BROKER_TOPIC_METRICS_SUFFIX) - && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { + && CI.contains(name, BROKER_TOPIC_METRICS_SUFFIX) + && CI.endsWith(name, FIFTEEN_MINUTE_RATE_SUFFIX)) { String topic = labels.get("topic"); if (labels.contains("name")) { var nameLblVal = labels.get("name"); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java index a477a2e0b..bd20cb2e1 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxSslSocketFactory.java @@ -165,7 +165,7 @@ public Socket createSocket(String host, int port) throws IOException { return defaultSocketFactory.createSocket(host, port); } - /// FOLLOWING METHODS WON'T BE USED DURING JMX INTERACTION, IMPLEMENTING THEM JUST FOR CONSISTENCY ->>>>> + // THE FOLLOWING METHODS WON'T BE USED DURING JMX INTERACTION, IMPLEMENTING THEM JUST FOR CONSISTENCY ->>>>> @Override public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException { @@ -177,7 +177,7 @@ public Socket createSocket(Socket s, String host, int port, boolean autoClose) t @Override public Socket createSocket(String host, int port, InetAddress localHost, int localPort) - throws IOException, UnknownHostException { + throws IOException { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().createSocket(host, port, localHost, localPort); } diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index fe0509ad2..0419233fd 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -39,12 +39,11 @@ public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( - "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name - + "(?:\\{(?>[^}]*)\\})?" // Optional labels (atomic group) - + "\\s+" - + "(-?(?:Inf|NaN|(?:\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)))" // Metric value - + "(?:\\s+([0-9]+))?$" // Optional timestamp - ); // Group 4: Optional timestamp + "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name + + "(?:\\{([^}]*)\\})?" // Optional labels (content in group 2) + + "\\s+" + + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Value (group 3) + + "(?:\\s+([0-9]+))?$"); // Group 4: Optional timestamp private static final Pattern HELP_PATTERN = @@ -329,11 +328,11 @@ public Optional buildSnapshot(String name, @Nullable String help return Optional.empty(); } var builder = HistogramSnapshot.builder().name(name).help(help); - buckets.asMap().forEach((labels, buckets) -> { - buckets = buckets.stream().sorted().toList(); + buckets.asMap().forEach((labels, localBuckets) -> { + localBuckets = localBuckets.stream().sorted().toList(); long prevCount = 0; var nonCumulativeBuckets = new ArrayList(); - for (Bucket b : buckets) { + for (Bucket b : localBuckets) { nonCumulativeBuckets.add(new Bucket(b.le, b.count - prevCount)); prevCount = b.count; } diff --git a/api/src/main/java/io/kafbat/ui/util/DynamicConfigOperations.java b/api/src/main/java/io/kafbat/ui/util/DynamicConfigOperations.java index 0686de2c4..1bc741ae4 100644 --- a/api/src/main/java/io/kafbat/ui/util/DynamicConfigOperations.java +++ b/api/src/main/java/io/kafbat/ui/util/DynamicConfigOperations.java @@ -221,7 +221,7 @@ protected NodeTuple representJavaBeanProperty(Object javaBean, @Data @Builder - // field name should be in sync with @ConfigurationProperties annotation + // the field name should be in sync with @ConfigurationProperties annotation public static class PropertiesStructure { private ClustersProperties kafka; diff --git a/api/src/main/java/io/kafbat/ui/util/jsonschema/JsonAvroConversion.java b/api/src/main/java/io/kafbat/ui/util/jsonschema/JsonAvroConversion.java index 76baf0072..4df8a637e 100644 --- a/api/src/main/java/io/kafbat/ui/util/jsonschema/JsonAvroConversion.java +++ b/api/src/main/java/io/kafbat/ui/util/jsonschema/JsonAvroConversion.java @@ -75,7 +75,7 @@ private static Object convert(JsonNode node, Schema avroSchema) { assertJsonType(node, JsonNodeType.OBJECT); var map = new LinkedHashMap(); var valueSchema = avroSchema.getValueType(); - node.fields().forEachRemaining(f -> map.put(f.getKey(), convert(f.getValue(), valueSchema))); + node.properties().forEach(f -> map.put(f.getKey(), convert(f.getValue(), valueSchema))); yield map; } case ARRAY -> { @@ -101,7 +101,7 @@ private static Object convert(JsonNode node, Schema avroSchema) { } assertJsonType(node, JsonNodeType.OBJECT); - var elements = Lists.newArrayList(node.fields()); + var elements = Lists.newArrayList(node.properties()); if (elements.size() != 1) { throw new JsonAvroConversionException( "UNION field value should be an object with single field == type name"); diff --git a/api/src/test/java/io/kafbat/ui/KafkaConsumerGroupTests.java b/api/src/test/java/io/kafbat/ui/KafkaConsumerGroupTests.java index c23ea5fb0..89b3000aa 100644 --- a/api/src/test/java/io/kafbat/ui/KafkaConsumerGroupTests.java +++ b/api/src/test/java/io/kafbat/ui/KafkaConsumerGroupTests.java @@ -215,7 +215,7 @@ private Closeable startConsumerGroups(int count, String consumerGroupPrefix) { String topicName = createTopicWithRandomName(); var consumers = Stream.generate(() -> { - String groupId = consumerGroupPrefix + RandomStringUtils.randomAlphabetic(5); + String groupId = consumerGroupPrefix + RandomStringUtils.secure().nextAlphabetic(5); val consumer = createTestConsumerWithGroupId(groupId); consumer.subscribe(List.of(topicName)); consumer.poll(Duration.ofMillis(100)); diff --git a/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java b/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java index 33414ef05..c9aa0bdfa 100644 --- a/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java +++ b/api/src/test/java/io/kafbat/ui/emitter/MessageFiltersTest.java @@ -183,8 +183,8 @@ void filterSpeedIsAtLeast5kPerSec() { List toFilter = new ArrayList<>(); for (int i = 0; i < 5_000; i++) { - String name = i % 2 == 0 ? "user1" : RandomStringUtils.randomAlphabetic(10); - String randString = RandomStringUtils.randomAlphabetic(30); + String name = i % 2 == 0 ? "user1" : RandomStringUtils.secure().nextAlphabetic(10); + String randString = RandomStringUtils.secure().nextAlphabetic(30); String jsonContent = String.format( "{ \"name\" : { \"randomStr\": \"%s\", \"first\" : \"%s\"} }", randString, name); diff --git a/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java b/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java index a301b7e93..9005ba2c3 100644 --- a/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java +++ b/api/src/test/java/io/kafbat/ui/service/StatisticsServiceTest.java @@ -64,6 +64,7 @@ void testInferredMetricsCollected() { .isEqualTo(4); } + @SuppressWarnings("unchecked") private GaugeDataPointSnapshot getGaugeSnapshot(InferredMetrics inferredMetrics, String metricName, Labels labels) { diff --git a/api/src/test/java/io/kafbat/ui/service/analyze/TopicAnalysisServiceTest.java b/api/src/test/java/io/kafbat/ui/service/analyze/TopicAnalysisServiceTest.java index 1c32b4215..93ed7a8e6 100644 --- a/api/src/test/java/io/kafbat/ui/service/analyze/TopicAnalysisServiceTest.java +++ b/api/src/test/java/io/kafbat/ui/service/analyze/TopicAnalysisServiceTest.java @@ -50,8 +50,8 @@ private void fillTopic(String topic, int cnt) { producer.send( new ProducerRecord<>( topic, - RandomStringUtils.randomAlphabetic(5), - RandomStringUtils.randomAlphabetic(10))); + RandomStringUtils.secure().nextAlphabetic(5), + RandomStringUtils.secure().nextAlphabetic(10))); } } } diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java index d43e21349..53261c4af 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/MetricsUtils.java @@ -3,8 +3,10 @@ import io.prometheus.metrics.model.snapshots.CounterSnapshot; import io.prometheus.metrics.model.snapshots.DataPointSnapshot; import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; import io.prometheus.metrics.model.snapshots.MetricMetadata; import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import java.util.Map; import java.util.Optional; public class MetricsUtils { @@ -12,49 +14,58 @@ public class MetricsUtils { private MetricsUtils() { } + private static boolean areMetadataEqual(MetricMetadata metadata1, MetricMetadata metadata2) { + return metadata1.getName().equals(metadata2.getName()) + && metadata1.getHelp().equals(metadata2.getHelp()) + && Optional.ofNullable(metadata1.getUnit()) + .map(u -> u.equals(metadata2.getUnit())) + .orElse(metadata2.getUnit() == null); + } + public static boolean isTheSameMetric(MetricSnapshot m1, MetricSnapshot m2) { - if (m1.getClass().equals(m2.getClass())) { - MetricMetadata metadata1 = m1.getMetadata(); - MetricMetadata metadata2 = m2.getMetadata(); - if ( - metadata1.getName().equals(metadata2.getName()) - && metadata1.getHelp().equals(metadata2.getHelp()) - && Optional.ofNullable( - metadata1.getUnit()).map(u -> u.equals(metadata2.getUnit()) - ).orElse(metadata2.getUnit() == null) - ) { - if (m1.getDataPoints().size() == m2.getDataPoints().size()) { - for (int i = 0; i < m1.getDataPoints().size(); i++) { - var m1dp = m1.getDataPoints().get(i); - var m2dp = m2.getDataPoints().get(i); - boolean same = isTheSameDataPoint(m1dp, m2dp); - if (!same) { - return false; - } - } - return true; - } + if (!m1.getClass().equals(m2.getClass())) { + return false; + } + MetricMetadata metadata1 = m1.getMetadata(); + MetricMetadata metadata2 = m2.getMetadata(); + if (!areMetadataEqual(metadata1, metadata2)) { + return false; + } + var dataPoints1 = m1.getDataPoints(); + var dataPoints2 = m2.getDataPoints(); + if (dataPoints1.size() != dataPoints2.size()) { + return false; + } + for (int i = 0; i < dataPoints1.size(); i++) { + if (!isTheSameDataPoint(dataPoints1.get(i), dataPoints2.get(i))) { + return false; } } - return false; + return true; + } + + private static boolean areLabelsEqual(Labels labels1, Labels labels2) { + return Optional.ofNullable(labels1) + .map(l -> l.equals(labels2)) + .orElse(labels2 == null); } public static boolean isTheSameDataPoint(DataPointSnapshot dp1, DataPointSnapshot dp2) { - if (dp1.getClass().equals(dp2.getClass())) { - if (Optional.ofNullable(dp1.getLabels()).map(l -> l.equals(dp2.getLabels())) - .orElse(dp2.getLabels() == null)) { - if (dp1 instanceof GaugeSnapshot.GaugeDataPointSnapshot g1) { - GaugeSnapshot.GaugeDataPointSnapshot g2 = (GaugeSnapshot.GaugeDataPointSnapshot) dp2; - return Double.compare(g1.getValue(), g2.getValue()) == 0; - } - if (dp1 instanceof CounterSnapshot.CounterDataPointSnapshot c1) { - CounterSnapshot.CounterDataPointSnapshot c2 = (CounterSnapshot.CounterDataPointSnapshot) dp2; - return Double.compare(c1.getValue(), c2.getValue()) == 0; - } - return true; - } + if (!dp1.getClass().equals(dp2.getClass())) { + return false; + } + if (!areLabelsEqual(dp1.getLabels(), dp2.getLabels())) { + return false; + } + if (dp1 instanceof GaugeSnapshot.GaugeDataPointSnapshot gauge1) { + var gauge2 = (GaugeSnapshot.GaugeDataPointSnapshot) dp2; + return Double.compare(gauge1.getValue(), gauge2.getValue()) == 0; + } + if (dp1 instanceof CounterSnapshot.CounterDataPointSnapshot counter1) { + var counter2 = (CounterSnapshot.CounterDataPointSnapshot) dp2; + return Double.compare(counter1.getValue(), counter2.getValue()) == 0; } - return false; + return true; } } diff --git a/api/src/test/java/io/kafbat/ui/util/ContentUtilsTest.java b/api/src/test/java/io/kafbat/ui/util/ContentUtilsTest.java index fa2926e3d..fecdb40e7 100644 --- a/api/src/test/java/io/kafbat/ui/util/ContentUtilsTest.java +++ b/api/src/test/java/io/kafbat/ui/util/ContentUtilsTest.java @@ -56,7 +56,7 @@ void testHeaderValueShort() { @Test void testHeaderValueLongStringUtf8() { - String testValue = RandomStringUtils.random(10000, true, false); + String testValue = RandomStringUtils.secure().next(10000, true, false); assertEquals(testValue, ContentUtils.convertToString(testValue.getBytes(StandardCharsets.UTF_8))); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 37349cd77..f504370d3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,6 +6,7 @@ aws-msk-auth = '2.3.0' azure-identity = '1.15.4' apache-commons-lang3 = '3.18.0' +apache-commons-text = '1.13.1' apache-commons-io = '2.18.0' apache-commons-pool2 = '2.12.1' apache-datasketches = '3.1.0' @@ -80,6 +81,7 @@ jackson-databind-nullable = { module = 'org.openapitools:jackson-databind-nullab kafka-clients = { module = 'org.apache.kafka:kafka-clients', version.ref = 'confluent-ccs' } apache-commons = { module = 'org.apache.commons:commons-lang3', version.ref = 'apache-commons-lang3' } +apache-commons-text = { module = 'org.apache.commons:commons-text', version.ref = 'apache-commons-text' } apache-commons-compress = { module = 'org.apache.commons:commons-compress', version.ref = 'apache-commons-compress' } apache-commons-io = { module = 'commons-io:commons-io', version.ref = 'apache-commons-io' } apache-commons-pool2 = { module = 'org.apache.commons:commons-pool2', version.ref = 'apache-commons-pool2' } diff --git a/serde-api/src/main/java/io/kafbat/ui/serde/api/Serde.java b/serde-api/src/main/java/io/kafbat/ui/serde/api/Serde.java index b9f812b62..5dc1057a2 100644 --- a/serde-api/src/main/java/io/kafbat/ui/serde/api/Serde.java +++ b/serde-api/src/main/java/io/kafbat/ui/serde/api/Serde.java @@ -123,6 +123,11 @@ interface Serializer { */ byte[] serialize(String input); + /** + * Serializes input string to bytes. Uses provided headers for additional information. + * @param input string entered by user into UI text field.
Note: this input is not formatted in any way. + * @return serialized bytes. Can be null if input is null or empty string. + */ default byte[] serialize(String input, Headers headers) { return serialize(input); } From fa2ffe35040b638c81578536ffd2ceb5dc95dae3 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 15:40:03 +0300 Subject: [PATCH 19/29] SonarQube fixes --- .../ui/service/KafkaClusterFactory.java | 5 -- .../ui/service/KafkaConnectService.java | 1 - .../ui/service/metrics/SummarizedMetrics.java | 4 ++ .../metrics/scrape/ScrapedClusterState.java | 51 ++++++++++--------- .../inferred/InferredMetricsScraper.java | 2 +- .../scrape/jmx/JmxMetricsFormatter.java | 7 ++- .../PrometheusTextFormatParser.java | 2 +- 7 files changed, 37 insertions(+), 35 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index eff085ef6..f0af81ae9 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -249,9 +249,4 @@ private ReactiveFailover ksqlClient(ClustersProperties.Cluster cl private List parseUrlList(String url) { return Stream.of(url.split(",")).map(String::trim).filter(s -> !s.isBlank()).toList(); } - - private boolean metricsConfigured(ClustersProperties.Cluster clusterProperties) { - return clusterProperties.getMetrics() != null; - } - } diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index 9dc03cf7d..fd4d912da 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -33,7 +33,6 @@ import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java index ef1782a71..feadbfdce 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/SummarizedMetrics.java @@ -21,6 +21,10 @@ import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +/** + * Will be replaced in the next versions. + * @deprecated Since 1.4.0 + **/ @Deprecated(forRemoval = true, since = "1.4.0") //used for api backward-compatibility @RequiredArgsConstructor public class SummarizedMetrics { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java index 5ae280c64..e5d8c059c 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.Builder; import lombok.RequiredArgsConstructor; @@ -123,41 +124,41 @@ public static Mono scrape(ClusterDescription clusterDescrip create( clusterDescription, phase1.getT1(), - phase1.getT3(), - phase1.getT4(), - phase2.getT1(), - phase2.getT2(), + topicStateMap(phase1.getT1(), phase1.getT3(), phase1.getT4(), phase2.getT1(), phase2.getT2()), phase2.getT3(), phase2.getT4() ))); } + private static Map topicStateMap( + InternalLogDirStats segmentStats, + Map topicDescriptions, + Map> topicConfigs, + Map latestOffsets, + Map earliestOffsets) { + + return topicDescriptions.entrySet().stream().map(entry -> new TopicState( + entry.getKey(), + entry.getValue(), + topicConfigs.getOrDefault(entry.getKey(), List.of()), + filterTopic(entry.getKey(), earliestOffsets), + filterTopic(entry.getKey(), latestOffsets), + segmentStats.getTopicStats().get(entry.getKey()), + Optional.ofNullable(segmentStats.getPartitionsStats()) + .map(topicForFilter -> filterTopic(entry.getKey(), topicForFilter)) + .orElse(null) + )).collect(Collectors.toMap( + TopicState::name, + Function.identity() + )); + } + private static ScrapedClusterState create(ClusterDescription clusterDescription, InternalLogDirStats segmentStats, - Map topicDescriptions, - Map> topicConfigs, - Map latestOffsets, - Map earliestOffsets, + Map topicStates, Map consumerDescriptions, Table consumerOffsets) { - - Map topicStates = new HashMap<>(); - topicDescriptions.forEach((name, desc) -> - topicStates.put( - name, - new TopicState( - name, - desc, - topicConfigs.getOrDefault(name, List.of()), - filterTopic(name, earliestOffsets), - filterTopic(name, latestOffsets), - segmentStats.getTopicStats().get(name), - Optional.ofNullable(segmentStats.getPartitionsStats()) - .map(topicForFilter -> filterTopic(name, topicForFilter)) - .orElse(null) - ))); - Map consumerGroupsStates = new HashMap<>(); consumerDescriptions.forEach((name, desc) -> consumerGroupsStates.put( diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java index 5f523f2e3..0bdba0a6a 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/inferred/InferredMetricsScraper.java @@ -226,7 +226,7 @@ private static void fillConsumerGroupsMetrics(MetricsRegistry registry, ScrapedC "Current Approximate Lag of a ConsumerGroup at Topic/Partition", List.of("consumergroup", TOPIC_TAG, PARTITION_TAG), List.of(groupName, tp.topic(), String.valueOf(tp.partition())), - endOffset - committedOffset //TODO: check +-1 + endOffset - committedOffset )); }); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 2b367d55a..063520984 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -21,9 +21,12 @@ public class JmxMetricsFormatter { "([^,=:\\*\\?]+)=(\"(?:\\\\.|[^\\\\\"])*\"|[^,=:\"]*)" ); + private JmxMetricsFormatter() { + } + public static List constructMetricsList(ObjectName jmxMetric, - MBeanAttributeInfo[] attributes, - Object[] attrValues) { + MBeanAttributeInfo[] attributes, + Object[] attrValues) { String domain = fixIllegalChars(jmxMetric.getDomain()); LinkedHashMap labels = getLabelsMap(jmxMetric); String firstLabel = labels.keySet().iterator().next(); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 0419233fd..39bfc0271 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -43,7 +43,7 @@ public class PrometheusTextFormatParser { + "(?:\\{([^}]*)\\})?" // Optional labels (content in group 2) + "\\s+" + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Value (group 3) - + "(?:\\s+([0-9]+))?$"); // Group 4: Optional timestamp + + "(?:\\s+(\\d+))?$"); // Group 4: Optional timestamp private static final Pattern HELP_PATTERN = From 9e4f180980bf0b0ea3d5a7baa365aee21a983ee9 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 15:56:09 +0300 Subject: [PATCH 20/29] SonarQube fixes --- .../ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java | 7 ++++++- .../scrape/prometheus/PrometheusTextFormatParser.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 063520984..65903c690 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -18,7 +18,7 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 private static final Pattern PROPERTY_PATTERN = Pattern.compile( - "([^,=:\\*\\?]+)=(\"(?:\\\\.|[^\\\\\"])*\"|[^,=:\"]*)" + "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" ); private JmxMetricsFormatter() { @@ -73,6 +73,11 @@ private static Optional convertNumericValue(Object value) { private static LinkedHashMap getLabelsMap(ObjectName mbeanName) { LinkedHashMap keyProperties = new LinkedHashMap<>(); String properties = mbeanName.getKeyPropertyListString(); + + if (properties.length() > 1024) { + throw new IllegalArgumentException("MBean key property list too long: " + properties); + } + Matcher match = PROPERTY_PATTERN.matcher(properties); while (match.lookingAt()) { String labelName = fixIllegalChars(match.group(1)); // label names should be fixed diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 39bfc0271..f8fe0029b 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -40,9 +40,9 @@ public class PrometheusTextFormatParser { // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name - + "(?:\\{([^}]*)\\})?" // Optional labels (content in group 2) + + "(?:\\{([^}]*)})?" // Optional labels (content in group 2) + "\\s+" - + "(-?(?:Inf|NaN|(?:\\d*\\.?\\d+(?:[eE][+-]?\\d+)?)))" // Value (group 3) + + "(-?(?:Inf|NaN|\\d*\\.?\\d+(?:[eE][+-]?\\d+)?))" // Value (group 3) + "(?:\\s+(\\d+))?$"); // Group 4: Optional timestamp From ff469623b66362b4a6ded5eca32416118a02c2b6 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 16:05:29 +0300 Subject: [PATCH 21/29] SonarQube fixes --- .../scrape/jmx/JmxMetricsFormatter.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 65903c690..3524607bd 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -17,7 +17,7 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 - private static final Pattern PROPERTY_PATTERN = Pattern.compile( + private static final Pattern PROPERTY_PATTERN = Pattern.compile( // NOSONAR "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" ); @@ -78,17 +78,20 @@ private static LinkedHashMap getLabelsMap(ObjectName mbeanName) throw new IllegalArgumentException("MBean key property list too long: " + properties); } - Matcher match = PROPERTY_PATTERN.matcher(properties); - while (match.lookingAt()) { - String labelName = fixIllegalChars(match.group(1)); // label names should be fixed - String labelValue = match.group(2); - keyProperties.put(labelName, labelValue); - properties = properties.substring(match.end()); - if (properties.startsWith(",")) { - properties = properties.substring(1); + if (!properties.isBlank()) { + Matcher match = PROPERTY_PATTERN.matcher(properties); + while (match.lookingAt()) { + String labelName = fixIllegalChars(match.group(1)); // label names should be fixed + String labelValue = match.group(2); + keyProperties.put(labelName, labelValue); + properties = properties.substring(match.end()); + if (properties.startsWith(",")) { + properties = properties.substring(1); + } + match.reset(properties); } - match.reset(properties); } + return keyProperties; } From d06c3b0e7c7d2380b43f8791bf05349d16e2fa5f Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 16:07:20 +0300 Subject: [PATCH 22/29] SonarQube fixes --- .../ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java | 4 ---- .../metrics/scrape/prometheus/PrometheusTextFormatParser.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 3524607bd..3fb65bdc3 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -74,10 +74,6 @@ private static LinkedHashMap getLabelsMap(ObjectName mbeanName) LinkedHashMap keyProperties = new LinkedHashMap<>(); String properties = mbeanName.getKeyPropertyListString(); - if (properties.length() > 1024) { - throw new IllegalArgumentException("MBean key property list too long: " + properties); - } - if (!properties.isBlank()) { Matcher match = PROPERTY_PATTERN.matcher(properties); while (match.lookingAt()) { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index f8fe0029b..15bdfd285 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -38,7 +38,7 @@ public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp - private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( + private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( // NOSONAR "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name + "(?:\\{([^}]*)})?" // Optional labels (content in group 2) + "\\s+" From 61c4dcf8afc8f2d8d03471c28a9240fe426f7564 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 16:09:18 +0300 Subject: [PATCH 23/29] SonarQube fixes --- .../ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 3fb65bdc3..950f17f03 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -17,8 +17,8 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 - private static final Pattern PROPERTY_PATTERN = Pattern.compile( // NOSONAR - "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" + private static final Pattern PROPERTY_PATTERN = Pattern.compile( + "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" // NOSONAR ); private JmxMetricsFormatter() { From bc60ea52ad0666a8f0b561cacb3a8135a0d9f3fd Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 16:10:29 +0300 Subject: [PATCH 24/29] SonarQube fixes --- .../ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index 950f17f03..c0b4ace33 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -17,7 +17,7 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 - private static final Pattern PROPERTY_PATTERN = Pattern.compile( + private static final Pattern PROPERTY_PATTERN = Pattern.compile( // NOSONAR "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" // NOSONAR ); From 894935f9721e1aeced06aeaf5ba882fecd0181fe Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 28 Jul 2025 16:54:33 +0300 Subject: [PATCH 25/29] Expose metrics per cluster & added graphs enabled feature --- .../kafbat/ui/config/ClustersProperties.java | 2 + .../PrometheusExposeController.java | 17 ++++- .../io/kafbat/ui/model/ClusterFeature.java | 3 +- .../io/kafbat/ui/service/FeatureService.java | 4 ++ .../ui/service/KafkaClusterFactory.java | 27 ++++++-- .../scrape/jmx/JmxMetricsFormatter.java | 2 +- .../PrometheusTextFormatParser.java | 2 +- .../main/resources/swagger/kafbat-ui-api.yaml | 68 ++++++++++++------- 8 files changed, 92 insertions(+), 33 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java index 7aff0af7b..47bba4e04 100644 --- a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java +++ b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java @@ -37,6 +37,8 @@ public class ClustersProperties { PollingProperties polling = new PollingProperties(); + MetricsStorage defaultMetricsStorage = new MetricsStorage(); + @Data public static class Cluster { @NotBlank(message = "field name for for cluster could not be blank") diff --git a/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java b/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java index 97c5084b1..7933471f3 100644 --- a/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java +++ b/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java @@ -4,6 +4,8 @@ import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.service.StatisticsCache; import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; @@ -18,7 +20,7 @@ public class PrometheusExposeController extends AbstractController implements Pr private final StatisticsCache statisticsCache; @Override - public Mono> getAllMetrics(ServerWebExchange exchange) { + public Mono> exposeAllMetrics(ServerWebExchange exchange) { return Mono.just( PrometheusExpose.exposeAllMetrics( clustersStorage.getKafkaClusters() @@ -29,4 +31,17 @@ public Mono> getAllMetrics(ServerWebExchange exchange) { ); } + @Override + public Mono> exposeClusterMetrics(String clusterName, + ServerWebExchange exchange) { + Optional cluster = clustersStorage.getClusterByName(clusterName); + if (cluster.isPresent() && cluster.get().isExposeMetricsViaPrometheusEndpoint()) { + return Mono.just(PrometheusExpose.exposeAllMetrics( + Map.of(clusterName, statisticsCache.get(cluster.get()).getMetrics()) + )); + } else { + return Mono.just(ResponseEntity.notFound().build()); + } + } + } diff --git a/api/src/main/java/io/kafbat/ui/model/ClusterFeature.java b/api/src/main/java/io/kafbat/ui/model/ClusterFeature.java index 6a88534e0..bdd66ea04 100644 --- a/api/src/main/java/io/kafbat/ui/model/ClusterFeature.java +++ b/api/src/main/java/io/kafbat/ui/model/ClusterFeature.java @@ -7,5 +7,6 @@ public enum ClusterFeature { TOPIC_DELETION, KAFKA_ACL_VIEW, KAFKA_ACL_EDIT, - CLIENT_QUOTA_MANAGEMENT + CLIENT_QUOTA_MANAGEMENT, + GRAPHS_ENABLED } diff --git a/api/src/main/java/io/kafbat/ui/service/FeatureService.java b/api/src/main/java/io/kafbat/ui/service/FeatureService.java index 59a23236b..8fb19d065 100644 --- a/api/src/main/java/io/kafbat/ui/service/FeatureService.java +++ b/api/src/main/java/io/kafbat/ui/service/FeatureService.java @@ -36,6 +36,10 @@ public Mono> getAvailableFeatures(ReactiveAdminClient admin features.add(Mono.just(ClusterFeature.KSQL_DB)); } + if (cluster.getPrometheusStorageClient() != null) { + features.add(Mono.just(ClusterFeature.GRAPHS_ENABLED)); + } + if (cluster.getSchemaRegistryClient() != null) { features.add(Mono.just(ClusterFeature.SCHEMA_REGISTRY)); } diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index f0af81ae9..e55bb1734 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -87,9 +87,18 @@ public KafkaCluster create(ClustersProperties properties, if (ksqlConfigured(clusterProperties)) { builder.ksqlClient(ksqlClient(clusterProperties)); } + if (prometheusStorageConfigured(properties.getDefaultMetricsStorage())) { + builder.prometheusStorageClient( + prometheusStorageClient(properties.getDefaultMetricsStorage(), clusterProperties.getSsl()) + ); + } if (prometheusStorageConfigured(clusterProperties)) { - builder.prometheusStorageClient(prometheusStorageClient(clusterProperties)); + builder.prometheusStorageClient(prometheusStorageClient( + clusterProperties.getMetrics().getStore(), + clusterProperties.getSsl()) + ); } + builder.originalProperties(clusterProperties); return builder.build(); } @@ -129,7 +138,8 @@ public Mono validate(ClustersProperties.Cluster clus : Mono.>>just(Optional.empty()), prometheusStorageConfigured(clusterProperties) - ? validatePrometheusStore(() -> prometheusStorageClient(clusterProperties)).map(Optional::of) + ? validatePrometheusStore(() -> prometheusStorageClient( + clusterProperties.getMetrics().getStore(), clusterProperties.getSsl())).map(Optional::of) : Mono.>just(Optional.empty()) ).map(tuple -> { var validation = new ClusterConfigValidationDTO(); @@ -156,13 +166,14 @@ private Properties convertProperties(Map propertiesMap) { return properties; } - private ReactiveFailover prometheusStorageClient(ClustersProperties.Cluster cluster) { + private ReactiveFailover prometheusStorageClient( + ClustersProperties.MetricsStorage storage, ClustersProperties.TruststoreConfig ssl) { WebClient webClient = new WebClientConfigurator() - .configureSsl(cluster.getSsl(), null) + .configureSsl(ssl, null) .configureBufferSize(webClientMaxBuffSize) .build(); return ReactiveFailover.create( - parseUrlList(cluster.getMetrics().getStore().getPrometheus().getUrl()), + parseUrlList(storage.getPrometheus().getUrl()), url -> new PrometheusClientApi(new io.kafbat.ui.prometheus.ApiClient(webClient).setBasePath(url)), ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, "No live schemaRegistry instances available", @@ -173,6 +184,12 @@ private ReactiveFailover prometheusStorageClient(ClustersPr private boolean prometheusStorageConfigured(ClustersProperties.Cluster cluster) { return Optional.ofNullable(cluster.getMetrics()) .flatMap(m -> Optional.ofNullable(m.getStore())) + .map(this::prometheusStorageConfigured) + .orElse(false); + } + + private boolean prometheusStorageConfigured(ClustersProperties.MetricsStorage storage) { + return Optional.ofNullable(storage) .flatMap(s -> Optional.ofNullable(s.getPrometheus())) .map(p -> StringUtils.hasText(p.getUrl())) .orElse(false); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java index c0b4ace33..c445434cb 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsFormatter.java @@ -17,7 +17,7 @@ public class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 - private static final Pattern PROPERTY_PATTERN = Pattern.compile( // NOSONAR + private static final Pattern PROPERTY_PATTERN = Pattern.compile(// NOSONAR "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)" // NOSONAR ); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java index 15bdfd285..9b7571cf6 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/prometheus/PrometheusTextFormatParser.java @@ -38,7 +38,7 @@ public class PrometheusTextFormatParser { // Regex to capture metric name, optional labels, value, and optional timestamp. // Groups: 1=name, 2=labels (content), 3=value, 4=timestamp - private static final Pattern METRIC_LINE_PATTERN = Pattern.compile( // NOSONAR + private static final Pattern METRIC_LINE_PATTERN = Pattern.compile(// NOSONAR "^([a-zA-Z_:][a-zA-Z0-9_:]*)" // Metric name + "(?:\\{([^}]*)})?" // Optional labels (content in group 2) + "\\s+" diff --git a/contract/src/main/resources/swagger/kafbat-ui-api.yaml b/contract/src/main/resources/swagger/kafbat-ui-api.yaml index 045a00e33..a72456ad9 100644 --- a/contract/src/main/resources/swagger/kafbat-ui-api.yaml +++ b/contract/src/main/resources/swagger/kafbat-ui-api.yaml @@ -205,8 +205,28 @@ paths: get: tags: - PrometheusExpose - summary: getAllMetrics - operationId: getAllMetrics + summary: exposeAllMetrics + operationId: exposeAllMetrics + responses: + 200: + description: OK + content: + application/text: + schema: + type: string + + /metrics/{clusterName}: + get: + tags: + - PrometheusExpose + summary: exposeClusterMetrics + operationId: exposeClusterMetrics + parameters: + - name: clusterName + in: path + required: true + schema: + type: string responses: 200: description: OK @@ -2540,6 +2560,7 @@ components: - KAFKA_ACL_VIEW # get ACLs listing - KAFKA_ACL_EDIT # create & delete ACLs - CLIENT_QUOTA_MANAGEMENT + - GRAPHS_ENABLED required: - id - name @@ -4494,6 +4515,8 @@ components: type: integer internalTopicPrefix: type: string + defaultMetricsStorage: + $ref: '#/components/schemas/ClusterMetricsStoreConfig' clusters: type: array items: @@ -4585,28 +4608,7 @@ components: prometheusExpose: type: boolean store: - type: object - properties: - prometheus: - type: object - properties: - url: - type: string - remoteWrite: - type: boolean - pushGatewayUrl: - type: string - pushGatewayUsername: - type: string - pushGatewayPassword: - type: string - pushGatewayJobName: - type: string - kafka: - type: object - properties: - topic: - type: string + $ref: '#/components/schemas/ClusterMetricsStoreConfig' properties: type: object additionalProperties: true @@ -4688,3 +4690,21 @@ components: type: object additionalProperties: type: string + ClusterMetricsStoreConfig: + type: object + properties: + prometheus: + type: object + properties: + url: + type: string + remoteWrite: + type: boolean + pushGatewayUrl: + type: string + pushGatewayUsername: + type: string + pushGatewayPassword: + type: string + pushGatewayJobName: + type: string From da5119c1085240e418405f492f559637efa066ae Mon Sep 17 00:00:00 2001 From: German Osin Date: Thu, 7 Aug 2025 14:47:01 +0300 Subject: [PATCH 26/29] fixed checkstyle --- api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java index c8c96d786..b3e5aa058 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaConnectService.java @@ -2,7 +2,6 @@ import static org.apache.commons.lang3.Strings.CI; - import com.github.benmanes.caffeine.cache.AsyncCache; import com.github.benmanes.caffeine.cache.Caffeine; import io.kafbat.ui.config.ClustersProperties; From ae7107847621109d458d27a03be019c37e636e9e Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 11 Aug 2025 14:32:17 +0300 Subject: [PATCH 27/29] Fixed comments --- .../ui/controller/PrometheusExposeController.java | 6 +++--- .../io/kafbat/ui/model/InternalLogDirStats.java | 2 +- .../java/io/kafbat/ui/model/KafkaCluster.java | 4 ++-- .../io/kafbat/ui/service/KafkaClusterFactory.java | 4 ++-- .../io/kafbat/ui/service/MessagesService.java | 3 +-- .../java/io/kafbat/ui/service/TopicsService.java | 6 ++++-- ...sExpose.java => PrometheusMetricsExposer.java} | 4 ++-- ...{MetricsScrapping.java => MetricsScraper.java} | 13 ++++++------- .../metrics/scrape/jmx/JmxMetricsRetriever.java | 4 ++-- .../ui/util/KafkaClientSslPropertiesUtil.java | 1 - .../java/io/kafbat/ui/util/SslPropertiesUtil.java | 15 --------------- .../metrics/prometheus/PrometheusExposeTest.java | 2 +- 12 files changed, 24 insertions(+), 40 deletions(-) rename api/src/main/java/io/kafbat/ui/service/metrics/prometheus/{PrometheusExpose.java => PrometheusMetricsExposer.java} (97%) rename api/src/main/java/io/kafbat/ui/service/metrics/scrape/{MetricsScrapping.java => MetricsScraper.java} (89%) diff --git a/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java b/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java index 7933471f3..26bceb8e0 100644 --- a/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java +++ b/api/src/main/java/io/kafbat/ui/controller/PrometheusExposeController.java @@ -3,7 +3,7 @@ import io.kafbat.ui.api.PrometheusExposeApi; import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.service.StatisticsCache; -import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import io.kafbat.ui.service.metrics.prometheus.PrometheusMetricsExposer; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -22,7 +22,7 @@ public class PrometheusExposeController extends AbstractController implements Pr @Override public Mono> exposeAllMetrics(ServerWebExchange exchange) { return Mono.just( - PrometheusExpose.exposeAllMetrics( + PrometheusMetricsExposer.exposeAllMetrics( clustersStorage.getKafkaClusters() .stream() .filter(KafkaCluster::isExposeMetricsViaPrometheusEndpoint) @@ -36,7 +36,7 @@ public Mono> exposeClusterMetrics(String clusterName, ServerWebExchange exchange) { Optional cluster = clustersStorage.getClusterByName(clusterName); if (cluster.isPresent() && cluster.get().isExposeMetricsViaPrometheusEndpoint()) { - return Mono.just(PrometheusExpose.exposeAllMetrics( + return Mono.just(PrometheusMetricsExposer.exposeAllMetrics( Map.of(clusterName, statisticsCache.get(cluster.get()).getMetrics()) )); } else { diff --git a/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java b/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java index d7e274f53..caa47bc2b 100644 --- a/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java +++ b/api/src/main/java/io/kafbat/ui/model/InternalLogDirStats.java @@ -85,7 +85,7 @@ private static Map calculateSpaceStats( Map totalBytes = new HashMap<>(); Map usableBytes = new HashMap<>(); logDirStats.forEach((logDir, descr) -> { - if (descr.error() == null) { + if (descr.error() != null) { return; } descr.totalBytes().ifPresent(b -> totalBytes.merge(logDir, b, Long::sum)); diff --git a/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java b/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java index 227364eb2..8fad56a49 100644 --- a/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java +++ b/api/src/main/java/io/kafbat/ui/model/KafkaCluster.java @@ -6,7 +6,7 @@ import io.kafbat.ui.prometheus.api.PrometheusClientApi; import io.kafbat.ui.service.ksql.KsqlApiClient; import io.kafbat.ui.service.masking.DataMasking; -import io.kafbat.ui.service.metrics.scrape.MetricsScrapping; +import io.kafbat.ui.service.metrics.scrape.MetricsScraper; import io.kafbat.ui.sr.api.KafkaSrClientApi; import io.kafbat.ui.util.ReactiveFailover; import java.util.Map; @@ -32,7 +32,7 @@ public class KafkaCluster { private final boolean exposeMetricsViaPrometheusEndpoint; private final DataMasking masking; private final PollingSettings pollingSettings; - private final MetricsScrapping metricsScrapping; + private final MetricsScraper metricsScrapping; private final ReactiveFailover schemaRegistryClient; private final Map> connectsClients; private final ReactiveFailover ksqlClient; diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index e55bb1734..526c9701f 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -17,7 +17,7 @@ import io.kafbat.ui.prometheus.api.PrometheusClientApi; import io.kafbat.ui.service.ksql.KsqlApiClient; import io.kafbat.ui.service.masking.DataMasking; -import io.kafbat.ui.service.metrics.scrape.MetricsScrapping; +import io.kafbat.ui.service.metrics.scrape.MetricsScraper; import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; import io.kafbat.ui.sr.ApiClient; import io.kafbat.ui.sr.api.KafkaSrClientApi; @@ -76,7 +76,7 @@ public KafkaCluster create(ClustersProperties properties, builder.exposeMetricsViaPrometheusEndpoint(exposeMetricsViaPrometheusEndpoint(clusterProperties)); builder.masking(DataMasking.create(clusterProperties.getMasking())); builder.pollingSettings(PollingSettings.create(clusterProperties, properties)); - builder.metricsScrapping(MetricsScrapping.create(clusterProperties, jmxMetricsRetriever)); + builder.metricsScrapping(MetricsScraper.create(clusterProperties, jmxMetricsRetriever)); if (schemaRegistryConfigured(clusterProperties)) { builder.schemaRegistryClient(schemaRegistryClient(clusterProperties)); diff --git a/api/src/main/java/io/kafbat/ui/service/MessagesService.java b/api/src/main/java/io/kafbat/ui/service/MessagesService.java index 36a0be1ce..3bf3d8330 100644 --- a/api/src/main/java/io/kafbat/ui/service/MessagesService.java +++ b/api/src/main/java/io/kafbat/ui/service/MessagesService.java @@ -24,7 +24,6 @@ import io.kafbat.ui.serdes.ConsumerRecordDeserializer; import io.kafbat.ui.serdes.ProducerRecordCreator; import io.kafbat.ui.util.KafkaClientSslPropertiesUtil; -import io.kafbat.ui.util.SslPropertiesUtil; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -205,7 +204,7 @@ public static KafkaProducer createProducer(KafkaCluster cluster, public static KafkaProducer createProducer(ClustersProperties.Cluster cluster, Map additionalProps) { Properties properties = new Properties(); - SslPropertiesUtil.addKafkaSslProperties(cluster.getSsl(), properties); + KafkaClientSslPropertiesUtil.addKafkaSslProperties(cluster.getSsl(), properties); properties.putAll(cluster.getProperties()); properties.putAll(cluster.getProducerProperties()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); diff --git a/api/src/main/java/io/kafbat/ui/service/TopicsService.java b/api/src/main/java/io/kafbat/ui/service/TopicsService.java index 034e1b9b9..becfd55ad 100644 --- a/api/src/main/java/io/kafbat/ui/service/TopicsService.java +++ b/api/src/main/java/io/kafbat/ui/service/TopicsService.java @@ -132,8 +132,10 @@ private List createList(List orderedNames, configs.getOrDefault(t, List.of()), partitionsOffsets, metrics, - Optional.ofNullable(clusterState.getTopicStates().get(t)).map(s -> s.segmentStats()).orElse(null), - Optional.ofNullable(clusterState.getTopicStates().get(t)).map(s -> s.partitionsSegmentStats()).orElse(null), + Optional.ofNullable(clusterState.getTopicStates().get(t)).map(TopicState::segmentStats) + .orElse(null), + Optional.ofNullable(clusterState.getTopicStates().get(t)).map(TopicState::partitionsSegmentStats) + .orElse(Map.of()), clustersProperties.getInternalTopicPrefix() )) .collect(toList()); diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusMetricsExposer.java similarity index 97% rename from api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java rename to api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusMetricsExposer.java index fa2723e0e..a90e65247 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExpose.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/prometheus/PrometheusMetricsExposer.java @@ -21,7 +21,7 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; -public final class PrometheusExpose { +public final class PrometheusMetricsExposer { private static final String CLUSTER_EXPOSE_LBL_NAME = "cluster"; private static final String BROKER_EXPOSE_LBL_NAME = "broker_id"; @@ -33,7 +33,7 @@ public final class PrometheusExpose { PROMETHEUS_EXPOSE_ENDPOINT_HEADERS.set(CONTENT_TYPE, OpenMetricsTextFormatWriter.CONTENT_TYPE); } - private PrometheusExpose() { + private PrometheusMetricsExposer() { } public static ResponseEntity exposeAllMetrics(Map clustersMetrics) { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScraper.java similarity index 89% rename from api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java rename to api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScraper.java index 0271a75db..8c5ff3bba 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScrapping.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/MetricsScraper.java @@ -7,7 +7,7 @@ import io.kafbat.ui.config.ClustersProperties.MetricsConfig; import io.kafbat.ui.model.Metrics; import io.kafbat.ui.model.MetricsScrapeProperties; -import io.kafbat.ui.service.metrics.prometheus.PrometheusExpose; +import io.kafbat.ui.service.metrics.prometheus.PrometheusMetricsExposer; import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetrics; import io.kafbat.ui.service.metrics.scrape.inferred.InferredMetricsScraper; import io.kafbat.ui.service.metrics.scrape.jmx.JmxMetricsRetriever; @@ -17,7 +17,6 @@ import io.prometheus.metrics.model.snapshots.MetricSnapshot; import jakarta.annotation.Nullable; import java.util.Collection; -import java.util.stream.Stream; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,7 +26,7 @@ @Slf4j @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class MetricsScrapping { +public class MetricsScraper { private final String clusterName; private final MetricsSink sink; @@ -35,8 +34,8 @@ public class MetricsScrapping { @Nullable private final BrokerMetricsScraper brokerMetricsScraper; - public static MetricsScrapping create(Cluster cluster, - JmxMetricsRetriever jmxMetricsRetriever) { + public static MetricsScraper create(Cluster cluster, + JmxMetricsRetriever jmxMetricsRetriever) { BrokerMetricsScraper scraper = null; MetricsConfig metricsConfig = cluster.getMetrics(); if (cluster.getMetrics() != null) { @@ -47,7 +46,7 @@ public static MetricsScrapping create(Cluster cluster, scraper = new PrometheusScraper(scrapeProperties); } } - return new MetricsScrapping( + return new MetricsScraper( cluster.getName(), MetricsSink.create(cluster), new InferredMetricsScraper(), @@ -79,7 +78,7 @@ private Flux prepareMetricsForSending(Metrics metrics) { //need to be "cold" because sinks can resubscribe multiple times return Flux.defer(() -> Flux.fromStream( - PrometheusExpose.prepareMetricsForGlobalExpose(clusterName, metrics))); + PrometheusMetricsExposer.prepareMetricsForGlobalExpose(clusterName, metrics))); } private Mono scrapeBrokers(Collection nodes) { diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java index 81424138b..e5984725d 100644 --- a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java +++ b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/jmx/JmxMetricsRetriever.java @@ -45,7 +45,7 @@ public void close() { public Mono> retrieveFromNode(MetricsScrapeProperties scrapeProperties, Node node) { if (isSslJmxEndpoint(scrapeProperties) && !SSL_JMX_SUPPORTED) { - log.warn("Cluster has jmx ssl configured, but it is not supported by app"); + log.warn("Cluster has jmx ssl configured, but it is not supported by the app"); return Mono.just(List.of()); } return Mono.fromSupplier(() -> retrieveSync(scrapeProperties, node)) @@ -60,7 +60,7 @@ private boolean isSslJmxEndpoint(MetricsScrapeProperties scrapeProperties) { @SneakyThrows private List retrieveSync(MetricsScrapeProperties scrapeProperties, Node node) { String jmxUrl = JMX_URL + node.host() + ":" + scrapeProperties.getPort() + "/" + JMX_SERVICE_TYPE; - log.debug("Collection JMX metrics for {}", jmxUrl); + log.debug("Collecting JMX metrics for {}", jmxUrl); List result = new ArrayList<>(); withJmxConnector(jmxUrl, scrapeProperties, jmxConnector -> getMetricsFromJmx(jmxConnector, result)); log.debug("{} metrics collected for {}", result.size(), jmxUrl); diff --git a/api/src/main/java/io/kafbat/ui/util/KafkaClientSslPropertiesUtil.java b/api/src/main/java/io/kafbat/ui/util/KafkaClientSslPropertiesUtil.java index 324e2e4d0..384888aa1 100644 --- a/api/src/main/java/io/kafbat/ui/util/KafkaClientSslPropertiesUtil.java +++ b/api/src/main/java/io/kafbat/ui/util/KafkaClientSslPropertiesUtil.java @@ -31,5 +31,4 @@ public static void addKafkaSslProperties(@Nullable ClustersProperties.Truststore } } - } diff --git a/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java b/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java index fda959a2b..52c20e1d1 100644 --- a/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java +++ b/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java @@ -1,23 +1,8 @@ package io.kafbat.ui.util; -import io.kafbat.ui.config.ClustersProperties; -import java.util.Properties; -import javax.annotation.Nullable; -import org.apache.kafka.common.config.SslConfigs; - public final class SslPropertiesUtil { private SslPropertiesUtil() { } - public static void addKafkaSslProperties(@Nullable ClustersProperties.TruststoreConfig truststoreConfig, - Properties sink) { - if (truststoreConfig != null && truststoreConfig.getTruststoreLocation() != null) { - sink.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststoreConfig.getTruststoreLocation()); - if (truststoreConfig.getTruststorePassword() != null) { - sink.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, truststoreConfig.getTruststorePassword()); - } - } - } - } diff --git a/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java b/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java index 927a9217f..2868e4765 100644 --- a/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java +++ b/api/src/test/java/io/kafbat/ui/service/metrics/prometheus/PrometheusExposeTest.java @@ -1,7 +1,7 @@ package io.kafbat.ui.service.metrics.prometheus; import static io.kafbat.ui.service.metrics.MetricsUtils.isTheSameMetric; -import static io.kafbat.ui.service.metrics.prometheus.PrometheusExpose.prepareMetricsForGlobalExpose; +import static io.kafbat.ui.service.metrics.prometheus.PrometheusMetricsExposer.prepareMetricsForGlobalExpose; import static org.assertj.core.api.Assertions.assertThat; import io.kafbat.ui.model.Metrics; From 95739bc3fa872f3a854b8dc00bb10ca4d15bc830 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 11 Aug 2025 14:33:51 +0300 Subject: [PATCH 28/29] Fixed comments --- api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java index 526c9701f..5a0c5201a 100644 --- a/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java +++ b/api/src/main/java/io/kafbat/ui/service/KafkaClusterFactory.java @@ -176,7 +176,7 @@ private ReactiveFailover prometheusStorageClient( parseUrlList(storage.getPrometheus().getUrl()), url -> new PrometheusClientApi(new io.kafbat.ui.prometheus.ApiClient(webClient).setBasePath(url)), ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, - "No live schemaRegistry instances available", + "No live Prometheus instances available", ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS ); } From 6c3abc4065769946320631b80ac16db4186a8c26 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 11 Aug 2025 15:04:54 +0300 Subject: [PATCH 29/29] Fixed comments --- .../main/java/io/kafbat/ui/util/SslPropertiesUtil.java | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java diff --git a/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java b/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java deleted file mode 100644 index 52c20e1d1..000000000 --- a/api/src/main/java/io/kafbat/ui/util/SslPropertiesUtil.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.kafbat.ui.util; - -public final class SslPropertiesUtil { - - private SslPropertiesUtil() { - } - -}