diff --git a/api/build.gradle b/api/build.gradle index dfbbf8201..64abce25f 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -58,6 +58,8 @@ dependencies { implementation libs.lucene.queryparser implementation libs.lucene.analysis.common + implementation libs.fastcsv + implementation libs.opendatadiscovery.oddrn implementation(libs.opendatadiscovery.client) { exclude group: 'org.springframework.boot', module: 'spring-boot-starter-webflux' diff --git a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java index e521f85a9..c232a8acb 100644 --- a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java +++ b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java @@ -45,6 +45,16 @@ public class ClustersProperties { AdminClient adminClient = new AdminClient(); + Csv csv = new Csv(); + + @Data + public static class Csv { + String lineDelimeter = "crlf"; + char quoteCharacter = '"'; + String quoteStrategy = "required"; + char fieldSeparator = ','; + } + @Data public static class AdminClient { Integer timeout; diff --git a/api/src/main/java/io/kafbat/ui/controller/AbstractController.java b/api/src/main/java/io/kafbat/ui/controller/AbstractController.java index be29d4a4b..605199f6c 100644 --- a/api/src/main/java/io/kafbat/ui/controller/AbstractController.java +++ b/api/src/main/java/io/kafbat/ui/controller/AbstractController.java @@ -4,9 +4,16 @@ import io.kafbat.ui.model.KafkaCluster; import io.kafbat.ui.model.rbac.AccessContext; import io.kafbat.ui.service.ClustersStorage; +import io.kafbat.ui.service.CsvWriterService; import io.kafbat.ui.service.audit.AuditService; import io.kafbat.ui.service.rbac.AccessControlService; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.Signal; @@ -15,6 +22,7 @@ public abstract class AbstractController { protected ClustersStorage clustersStorage; protected AccessControlService accessControlService; protected AuditService auditService; + protected CsvWriterService csvWriterService; protected KafkaCluster getCluster(String name) { return clustersStorage.getClusterByName(name) @@ -44,4 +52,22 @@ public void setAccessControlService(AccessControlService accessControlService) { public void setAuditService(AuditService auditService) { this.auditService = auditService; } + + public , R> Mono> responseToCsv(ResponseEntity response) { + return responseToCsv(response, (t) -> t); + } + + public Mono> responseToCsv(ResponseEntity response, Function> extract) { + if (response.getStatusCode().is2xxSuccessful()) { + return mapToCsv(extract.apply(response.getBody())).map(ResponseEntity::ok); + } else { + return Mono.just(ResponseEntity.status(response.getStatusCode()).body( + Optional.ofNullable(response.getBody()).map(Object::toString).orElse("") + )); + } + } + + protected Mono mapToCsv(Flux body) { + return csvWriterService.write(body); + } } diff --git a/api/src/main/java/io/kafbat/ui/controller/AclsController.java b/api/src/main/java/io/kafbat/ui/controller/AclsController.java index 190b1081c..5d146218a 100644 --- a/api/src/main/java/io/kafbat/ui/controller/AclsController.java +++ b/api/src/main/java/io/kafbat/ui/controller/AclsController.java @@ -63,6 +63,8 @@ public Mono> deleteAcl(String clusterName, Mono>> listAcls(String clusterName, KafkaAclResourceTypeDTO resourceTypeDto, @@ -96,19 +98,14 @@ public Mono>> listAcls(String clusterName, } @Override - public Mono> getAclAsCsv(String clusterName, ServerWebExchange exchange) { - AccessContext context = AccessContext.builder() - .cluster(clusterName) - .aclActions(AclAction.VIEW) - .operationName("getAclAsCsv") - .build(); - - return validateAccess(context).then( - aclsService.getAclAsCsvString(getCluster(clusterName)) - .map(ResponseEntity::ok) - .flatMap(Mono::just) - .doOnEach(sig -> audit(context, sig)) - ); + public Mono> getAclAsCsv(String clusterName, + KafkaAclResourceTypeDTO resourceType, + String resourceName, + KafkaAclNamePatternTypeDTO namePatternType, + String search, Boolean fts, + ServerWebExchange exchange) { + return listAcls(clusterName, resourceType, resourceName, namePatternType, search, fts, exchange) + .flatMap(this::responseToCsv); } @Override diff --git a/api/src/main/java/io/kafbat/ui/controller/BrokersController.java b/api/src/main/java/io/kafbat/ui/controller/BrokersController.java index 9c76fdc5e..64786ceab 100644 --- a/api/src/main/java/io/kafbat/ui/controller/BrokersController.java +++ b/api/src/main/java/io/kafbat/ui/controller/BrokersController.java @@ -46,6 +46,12 @@ public Mono>> getBrokers(String clusterName, .doOnEach(sig -> audit(context, sig)); } + @Override + public Mono> getBrokersCsv(String clusterName, + ServerWebExchange exchange) { + return getBrokers(clusterName, exchange).flatMap(this::responseToCsv); + } + @Override public Mono> getBrokersMetrics(String clusterName, Integer id, ServerWebExchange exchange) { diff --git a/api/src/main/java/io/kafbat/ui/controller/ConsumerGroupsController.java b/api/src/main/java/io/kafbat/ui/controller/ConsumerGroupsController.java index 9e954575d..0b0341522 100644 --- a/api/src/main/java/io/kafbat/ui/controller/ConsumerGroupsController.java +++ b/api/src/main/java/io/kafbat/ui/controller/ConsumerGroupsController.java @@ -22,6 +22,7 @@ import io.kafbat.ui.service.mcp.McpTool; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.function.Supplier; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -95,6 +96,8 @@ public Mono> getConsumerGroup(String clu .doOnEach(sig -> audit(context, sig)); } + + @Override public Mono>> getTopicConsumerGroups(String clusterName, String topicName, @@ -120,6 +123,8 @@ public Mono>> getTopicConsumerGroups(Strin .doOnEach(sig -> audit(context, sig)); } + + @Override public Mono> getConsumerGroupsPage( String clusterName, @@ -138,10 +143,43 @@ public Mono> getConsumerGroupsPage .build(); return validateAccess(context).then( - consumerGroupService.getConsumerGroupsPage( + consumerGroupService.getConsumerGroups( + getCluster(clusterName), + OptionalInt.of( + Optional.ofNullable(page).filter(i -> i > 0).orElse(1) + ), + OptionalInt.of( + Optional.ofNullable(perPage).filter(i -> i > 0).orElse(defaultConsumerGroupsPageSize) + ), + search, + fts, + Optional.ofNullable(orderBy).orElse(ConsumerGroupOrderingDTO.NAME), + Optional.ofNullable(sortOrderDto).orElse(SortOrderDTO.ASC) + ) + .map(this::convertPage) + .map(ResponseEntity::ok) + ).doOnEach(sig -> audit(context, sig)); + } + + + @Override + public Mono> getConsumerGroupsCsv(String clusterName, Integer page, + Integer perPage, String search, + ConsumerGroupOrderingDTO orderBy, + SortOrderDTO sortOrderDto, Boolean fts, + ServerWebExchange exchange) { + + var context = AccessContext.builder() + .cluster(clusterName) + // consumer group access validation is within the service + .operationName("getConsumerGroupsPage") + .build(); + + return validateAccess(context).then( + consumerGroupService.getConsumerGroups( getCluster(clusterName), - Optional.ofNullable(page).filter(i -> i > 0).orElse(1), - Optional.ofNullable(perPage).filter(i -> i > 0).orElse(defaultConsumerGroupsPageSize), + OptionalInt.empty(), + OptionalInt.empty(), search, fts, Optional.ofNullable(orderBy).orElse(ConsumerGroupOrderingDTO.NAME), @@ -149,6 +187,7 @@ public Mono> getConsumerGroupsPage ) .map(this::convertPage) .map(ResponseEntity::ok) + .flatMap(r -> responseToCsv(r, (g) -> Flux.fromIterable(g.getConsumerGroups()))) ).doOnEach(sig -> audit(context, sig)); } @@ -194,7 +233,12 @@ public Mono> resetConsumerGroupOffsets(String clusterName, ); } Map offsets = reset.getPartitionsOffsets().stream() - .collect(toMap(PartitionOffsetDTO::getPartition, PartitionOffsetDTO::getOffset)); + .collect( + toMap( + PartitionOffsetDTO::getPartition, + d -> Optional.ofNullable(d.getOffset()).orElse(0L) + ) + ); return offsetsResetService.resetToOffsets(cluster, group, reset.getTopic(), offsets); default: return Mono.error( diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index 559d9ae6c..b271d19fd 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -17,6 +17,7 @@ import io.kafbat.ui.model.NewConnectorDTO; import io.kafbat.ui.model.SortOrderDTO; import io.kafbat.ui.model.TaskDTO; +import io.kafbat.ui.model.TopicsResponseDTO; import io.kafbat.ui.model.rbac.AccessContext; import io.kafbat.ui.model.rbac.permission.ConnectAction; import io.kafbat.ui.service.KafkaConnectService; @@ -56,6 +57,13 @@ public Mono>> getConnects(String clusterName, return Mono.just(ResponseEntity.ok(availableConnects)); } + @Override + public Mono> getConnectsCsv(String clusterName, Boolean withStats, + ServerWebExchange exchange) { + return getConnects(clusterName, withStats, exchange) + .flatMap(this::responseToCsv); + } + @Override public Mono>> getConnectors(String clusterName, String connectName, ServerWebExchange exchange) { @@ -157,6 +165,15 @@ public Mono>> getAllConnectors( .doOnEach(sig -> audit(context, sig)); } + @Override + public Mono> getAllConnectorsCsv(String clusterName, String search, + ConnectorColumnsToSortDTO orderBy, + SortOrderDTO sortOrder, Boolean fts, + ServerWebExchange exchange) { + return getAllConnectors(clusterName, search, orderBy, sortOrder, fts, exchange) + .flatMap(this::responseToCsv); + } + @Override public Mono>> getConnectorConfig(String clusterName, String connectName, diff --git a/api/src/main/java/io/kafbat/ui/controller/TopicsController.java b/api/src/main/java/io/kafbat/ui/controller/TopicsController.java index 51af387c6..7d440719d 100644 --- a/api/src/main/java/io/kafbat/ui/controller/TopicsController.java +++ b/api/src/main/java/io/kafbat/ui/controller/TopicsController.java @@ -29,7 +29,6 @@ import io.kafbat.ui.model.TopicUpdateDTO; import io.kafbat.ui.model.TopicsResponseDTO; import io.kafbat.ui.model.rbac.AccessContext; -import io.kafbat.ui.model.rbac.permission.ConnectAction; import io.kafbat.ui.service.KafkaConnectService; import io.kafbat.ui.service.TopicsService; import io.kafbat.ui.service.analyze.TopicAnalysisService; @@ -186,7 +185,7 @@ public Mono> getTopics(String clusterName, .operationName("getTopics") .build(); - return topicsService.getTopicsForPagination(getCluster(clusterName), search, showInternal, fts) + return topicsService.getTopics(getCluster(clusterName), search, showInternal, fts) .flatMap(topics -> accessControlService.filterViewableTopics(topics, clusterName)) .flatMap(topics -> { int pageSize = perPage != null && perPage > 0 ? perPage : DEFAULT_PAGE_SIZE; @@ -219,6 +218,28 @@ public Mono> getTopics(String clusterName, .doOnEach(sig -> audit(context, sig)); } + @Override + public Mono> getTopicsCsv(String clusterName, Boolean showInternal, + String search, TopicColumnsToSortDTO orderBy, + SortOrderDTO sortOrder, Boolean fts, + ServerWebExchange exchange) { + + AccessContext context = AccessContext.builder() + .cluster(clusterName) + .operationName("getTopicsCsv") + .build(); + + ClustersProperties.ClusterFtsProperties ftsProperties = clustersProperties.getFts(); + Comparator comparatorForTopic = getComparatorForTopic(orderBy, ftsProperties.use(fts)); + + return topicsService + .getTopics(getCluster(clusterName), search, showInternal, fts) + .flatMap(topics -> accessControlService.filterViewableTopics(topics, clusterName)) + .map(topics -> topics.stream().sorted(comparatorForTopic).toList()) + .flatMap(topics -> responseToCsv(ResponseEntity.ok(Flux.fromIterable(topics)))) + .doOnEach(sig -> audit(context, sig)); + } + @Override public Mono> updateTopic( String clusterName, String topicName, @Valid Mono topicUpdate, diff --git a/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java b/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java index ce8f6a09c..9a4e56d6b 100644 --- a/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java +++ b/api/src/main/java/io/kafbat/ui/service/ConsumerGroupService.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.Properties; import java.util.Set; import java.util.function.ToIntFunction; @@ -70,6 +71,31 @@ private Mono> getConsumerGroups( }); } + public Mono getConsumerGroups( + KafkaCluster cluster, + OptionalInt pageNum, + OptionalInt perPage, + @Nullable String search, + Boolean fts, + ConsumerGroupOrderingDTO orderBy, + SortOrderDTO sortOrderDto) { + return adminClientService.get(cluster).flatMap(ac -> + ac.listConsumerGroups() + .map(listing -> filterGroups(listing, search, fts)) + .flatMapIterable(lst -> lst) + .filterWhen(cg -> accessControlService.isConsumerGroupAccessible(cg.groupId(), cluster.getName())) + .collectList() + .flatMap(allGroups -> + loadSortedDescriptions(ac, allGroups, pageNum, perPage, orderBy, sortOrderDto) + .flatMap(descriptions -> getConsumerGroups(ac, descriptions) + .map(page -> + ConsumerGroupsPage.from(page, allGroups.size(), pageNum, perPage) + ) + ) + ) + ); + } + public Mono> getConsumerGroupsForTopic(KafkaCluster cluster, String topic) { return adminClientService.get(cluster) @@ -142,33 +168,20 @@ private boolean isConsumerGroupRelatesToTopic(String topic, } public record ConsumerGroupsPage(List consumerGroups, int totalPages) { + public static ConsumerGroupsPage from(List groups, + int totalSize, + OptionalInt pageNum, + OptionalInt perPage) { + return new ConsumerGroupsPage(groups, + (totalSize / perPage.orElse(totalSize)) + (totalSize % perPage.orElse(totalSize) == 0 ? 0 : 1) + ); + } } private record GroupWithDescr(InternalConsumerGroup icg, ConsumerGroupDescription cgd) { } - public Mono getConsumerGroupsPage( - KafkaCluster cluster, - int pageNum, - int perPage, - @Nullable String search, - Boolean fts, - ConsumerGroupOrderingDTO orderBy, - SortOrderDTO sortOrderDto) { - return adminClientService.get(cluster).flatMap(ac -> - ac.listConsumerGroups() - .map(listing -> filterGroups(listing, search, fts) - ) - .flatMapIterable(lst -> lst) - .filterWhen(cg -> accessControlService.isConsumerGroupAccessible(cg.groupId(), cluster.getName())) - .collectList() - .flatMap(allGroups -> - loadSortedDescriptions(ac, allGroups, pageNum, perPage, orderBy, sortOrderDto) - .flatMap(descriptions -> getConsumerGroups(ac, descriptions) - .map(page -> new ConsumerGroupsPage( - page, - (allGroups.size() / perPage) + (allGroups.size() % perPage == 0 ? 0 : 1)))))); - } + private Collection filterGroups(Collection groups, String search, Boolean useFts) { @@ -180,8 +193,8 @@ private Collection filterGroups(Collection> loadSortedDescriptions(ReactiveAdminClient ac, List groups, - int pageNum, - int perPage, + OptionalInt pageNum, + OptionalInt perPage, ConsumerGroupOrderingDTO orderBy, SortOrderDTO sortOrderDto) { return switch (orderBy) { @@ -232,8 +245,8 @@ private Mono> loadSortedDescriptions(ReactiveAdmi private Mono> loadDescriptionsByListings(ReactiveAdminClient ac, List listings, Comparator comparator, - int pageNum, - int perPage, + OptionalInt pageNum, + OptionalInt perPage, SortOrderDTO sortOrderDto) { List sortedGroups = sortAndPaginate(listings, comparator, pageNum, perPage, sortOrderDto) .map(ConsumerGroupListing::groupId) @@ -244,13 +257,19 @@ private Mono> loadDescriptionsByListings(Reactive private Stream sortAndPaginate(Collection collection, Comparator comparator, - int pageNum, - int perPage, + OptionalInt pageNum, + OptionalInt perPage, SortOrderDTO sortOrderDto) { - return collection.stream() - .sorted(sortOrderDto == SortOrderDTO.ASC ? comparator : comparator.reversed()) - .skip((long) (pageNum - 1) * perPage) - .limit(perPage); + Stream sorted = collection.stream() + .sorted(sortOrderDto == SortOrderDTO.ASC ? comparator : comparator.reversed()); + + if (pageNum.isPresent() && perPage.isPresent()) { + return sorted + .skip((long) (pageNum.getAsInt() - 1) * perPage.getAsInt()) + .limit(perPage.getAsInt()); + } else { + return sorted; + } } private Mono> describeConsumerGroups( @@ -304,8 +323,8 @@ private Mono> loadDescriptionsByInternalConsumerG ReactiveAdminClient ac, List groups, Comparator comparator, - int pageNum, - int perPage, + OptionalInt pageNum, + OptionalInt perPage, SortOrderDTO sortOrderDto) { var groupNames = groups.stream().map(ConsumerGroupListing::groupId).toList(); diff --git a/api/src/main/java/io/kafbat/ui/service/CsvWriterService.java b/api/src/main/java/io/kafbat/ui/service/CsvWriterService.java new file mode 100644 index 000000000..794455104 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/CsvWriterService.java @@ -0,0 +1,74 @@ +package io.kafbat.ui.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import de.siegmar.fastcsv.writer.CsvWriter; +import de.siegmar.fastcsv.writer.LineDelimiter; +import de.siegmar.fastcsv.writer.QuoteStrategies; +import io.kafbat.ui.config.ClustersProperties; +import java.io.StringWriter; +import java.util.List; +import java.util.Map; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class CsvWriterService { + private final ObjectMapper om; + private final ClustersProperties.Csv properties; + + public CsvWriterService(ObjectMapper om, + ClustersProperties properties) { + this.om = om; + this.properties = properties.getCsv(); + } + + private CsvWriter writer(StringWriter sw) { + return CsvWriter.builder() + .fieldSeparator(properties.getFieldSeparator()) + .quoteCharacter(properties.getQuoteCharacter()) + .quoteStrategy(QuoteStrategies.valueOf(properties.getQuoteStrategy().toUpperCase())) + .lineDelimiter(LineDelimiter.valueOf(properties.getLineDelimeter().toUpperCase())) + .build(sw); + } + + public Mono write(Flux items) { + return items.collectList().map(this::write); + } + + + public String write(List items) { + final StringWriter sw = new StringWriter(); + final CsvWriter writer = writer(sw); + + if (!items.isEmpty()) { + writer.writeRecord(mapHeader(items.getFirst())); + for (T item : items) { + writer.writeRecord(mapRecord(item)); + } + } + return sw.toString(); + } + + private List mapHeader(T item) { + JsonNode jsonNode = om.valueToTree(item); + if (jsonNode.isObject()) { + return jsonNode.properties().stream() + .map(Map.Entry::getKey) + .toList(); + } else { + return List.of(jsonNode.asText()); + } + } + + private List mapRecord(T item) { + JsonNode jsonNode = om.valueToTree(item); + if (jsonNode.isObject()) { + return jsonNode.properties().stream() + .map(Map.Entry::getValue) + .map(JsonNode::asText) + .toList(); + } else { + return List.of(jsonNode.asText()); + } + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/TopicsService.java b/api/src/main/java/io/kafbat/ui/service/TopicsService.java index 4bb5a8fbf..d9e800b05 100644 --- a/api/src/main/java/io/kafbat/ui/service/TopicsService.java +++ b/api/src/main/java/io/kafbat/ui/service/TopicsService.java @@ -467,8 +467,7 @@ public Mono cloneTopic( ); } - public Mono> getTopicsForPagination(KafkaCluster cluster, String search, Boolean showInternal, - Boolean fts) { + public Mono> getTopics(KafkaCluster cluster, String search, Boolean showInternal, Boolean fts) { Statistics stats = statisticsCache.get(cluster); ScrapedClusterState clusterState = stats.getClusterState(); boolean useFts = clustersProperties.getFts().use(fts); diff --git a/api/src/main/java/io/kafbat/ui/service/acl/AclsService.java b/api/src/main/java/io/kafbat/ui/service/acl/AclsService.java index 9986d603e..a65f6eec8 100644 --- a/api/src/main/java/io/kafbat/ui/service/acl/AclsService.java +++ b/api/src/main/java/io/kafbat/ui/service/acl/AclsService.java @@ -86,12 +86,6 @@ private List filter(List acls, String principalSearch, B return filter.find(principalSearch); } - public Mono getAclAsCsvString(KafkaCluster cluster) { - return adminClientService.get(cluster) - .flatMap(c -> c.listAcls(ResourcePatternFilter.ANY)) - .map(AclCsv::transformToCsvString); - } - public Mono syncAclWithAclCsv(KafkaCluster cluster, String csv) { return adminClientService.get(cluster) .flatMap(ac -> ac.listAcls(ResourcePatternFilter.ANY).flatMap(existingAclList -> { diff --git a/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java b/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java index 0ffa5633f..8b858e6ba 100644 --- a/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java +++ b/api/src/test/java/io/kafbat/ui/service/TopicsServicePaginationTest.java @@ -110,9 +110,9 @@ private void init(Map topicsInCache) { when(reactiveAdminClient.listTopics(anyBoolean())).thenReturn(Mono.just(topicsInCache.keySet())); when(clustersStorage.getClusterByName(isA(String.class))) .thenReturn(Optional.of(kafkaCluster)); - when(mockTopicsService.getTopicsForPagination(isA(KafkaCluster.class), any(), any(), any())) + when(mockTopicsService.getTopics(isA(KafkaCluster.class), any(), any(), any())) .thenAnswer(a -> - topicsService.getTopicsForPagination( + topicsService.getTopics( a.getArgument(0), a.getArgument(1), a.getArgument(2), diff --git a/api/src/test/java/io/kafbat/ui/service/mcp/McpSpecificationGeneratorTest.java b/api/src/test/java/io/kafbat/ui/service/mcp/McpSpecificationGeneratorTest.java index 3c8334da5..198f03e9e 100644 --- a/api/src/test/java/io/kafbat/ui/service/mcp/McpSpecificationGeneratorTest.java +++ b/api/src/test/java/io/kafbat/ui/service/mcp/McpSpecificationGeneratorTest.java @@ -43,7 +43,7 @@ void testConvertController() { List specifications = MCP_SPECIFICATION_GENERATOR.convertTool(topicsController); - assertThat(specifications).hasSize(15); + assertThat(specifications).hasSize(16); List tools = List.of( new McpSchema.Tool( "recreateTopic", diff --git a/contract-typespec/api/acls.tsp b/contract-typespec/api/acls.tsp index e3b8a41e8..74f7cffe9 100644 --- a/contract-typespec/api/acls.tsp +++ b/contract-typespec/api/acls.tsp @@ -24,7 +24,14 @@ interface AclApi { @summary("getAclAsCsv") @get @operationId("getAclAsCsv") - getAclAsCsv(@path clusterName: string): string; + getAclAsCsv( + @path clusterName: string, + @query resourceType?: KafkaAclResourceType, + @query resourceName?: string, + @query namePatternType?: KafkaAclNamePatternType, + @query search?: string, + @query fts?: boolean + ): CsvResponse; @route("/csv") @summary("syncAclsCsv") diff --git a/contract-typespec/api/brokers.tsp b/contract-typespec/api/brokers.tsp index 29a2ceca5..b19695fcb 100644 --- a/contract-typespec/api/brokers.tsp +++ b/contract-typespec/api/brokers.tsp @@ -13,6 +13,12 @@ interface BrokersApi { @summary("getBrokers") getBrokers(@path clusterName: string): Broker[]; + @get + @operationId("getBrokersCsv") + @summary("getBrokersCsv") + @route("/csv") + getBrokersCsv(@path clusterName: string): CsvResponse; + @get @route("/{id}/configs") @operationId("getBrokerConfig") diff --git a/contract-typespec/api/consumer-groups.tsp b/contract-typespec/api/consumer-groups.tsp index 8e13d588e..fb4860923 100644 --- a/contract-typespec/api/consumer-groups.tsp +++ b/contract-typespec/api/consumer-groups.tsp @@ -23,6 +23,20 @@ interface ConsumerGroupsApi { @query fts?: boolean ): ConsumerGroupsPageResponse; + @get + @route("/csv") + @operationId("getConsumerGroupsCsv") + @summary("getConsumerGroupsPage") + getConsumerGroupsCsv( + @path clusterName: string, + @query page?: int32, + @query perPage?: int32, + @query search?: string, + @query orderBy?: ConsumerGroupOrdering, + @query sortOrder?: SortOrder, + @query fts?: boolean + ): CsvResponse; + @get @route("/{id}") @operationId("getConsumerGroup") diff --git a/contract-typespec/api/kafka-connect.tsp b/contract-typespec/api/kafka-connect.tsp index aa45b4aef..db5632068 100644 --- a/contract-typespec/api/kafka-connect.tsp +++ b/contract-typespec/api/kafka-connect.tsp @@ -14,6 +14,12 @@ interface ConnectInstancesApi { @summary("getConnects") getConnects(@path clusterName: string, @query withStats?: boolean): Connect[]; + @get + @operationId("getConnectsCsv") + @summary("getConnects") + @route("/csv") + getConnectsCsv(@path clusterName: string, @query withStats?: boolean): CsvResponse; + @get @route("/{connectName}/plugins") @summary("get connector plugins") @@ -49,6 +55,18 @@ interface ConnectorsApi { @query sortOrder?: SortOrder, @query fts?: boolean ): FullConnectorInfo[]; + + @get + @operationId("getAllConnectorsCsv") + @summary("getAllConnectorsCsv") + @route("/csv") + getAllConnectorsCsv( + @path clusterName: string, + @query search?: string, + @query orderBy?: ConnectorColumnsToSort, + @query sortOrder?: SortOrder, + @query fts?: boolean + ): CsvResponse; } // /api/clusters/{clusterName}/connects/{connectName}/connectors diff --git a/contract-typespec/api/responses.tsp b/contract-typespec/api/responses.tsp index c784aff70..4befce8df 100644 --- a/contract-typespec/api/responses.tsp +++ b/contract-typespec/api/responses.tsp @@ -70,3 +70,8 @@ model FieldError { @doc("Field format violations description (ex. [\"size must be between 0 and 20\", \"must be a well-formed email address\"])") restrictions?: string[]; } + +model CsvResponse is Response<200> { + @header contentType: "text/csv"; + @body _: string; +} diff --git a/contract-typespec/api/topics.tsp b/contract-typespec/api/topics.tsp index 27b13c57a..d61b8190f 100644 --- a/contract-typespec/api/topics.tsp +++ b/contract-typespec/api/topics.tsp @@ -24,6 +24,19 @@ interface TopicsApi { @query fts?: boolean ): TopicsResponse; + @get + @operationId("getTopicsCsv") + @summary("getTopics") + @route("csv") + getTopicsCsv( + @path clusterName: string, + @query showInternal?: boolean, + @query search?: string, + @query orderBy?: TopicColumnsToSort, + @query sortOrder?: SortOrder, + @query fts?: boolean + ): CsvResponse; + @post @operationId("createTopic") @summary("createTopic") diff --git a/frontend/package.json b/frontend/package.json index 0d3881493..4f98cad2b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,6 +27,7 @@ "react-error-boundary": "4.0.13", "react-hook-form": "7.54.2", "react-hot-toast": "2.4.1", + "react-innertext": "1.1.5", "react-is": "18.2.0", "react-multi-select-component": "4.3.4", "react-router-dom": "6.23.0", diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index be267d752..6c245f201 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -90,6 +90,9 @@ importers: react-hot-toast: specifier: 2.4.1 version: 2.4.1(csstype@3.1.2)(react-dom@18.2.0(react@18.2.0))(react@18.2.0) + react-innertext: + specifier: 1.1.5 + version: 1.1.5(@types/react@18.2.79)(react@18.2.0) react-is: specifier: 18.2.0 version: 18.2.0 @@ -3739,6 +3742,12 @@ packages: react: '>=16' react-dom: '>=16' + react-innertext@1.1.5: + resolution: {integrity: sha512-PWAqdqhxhHIv80dT9znP2KvS+hfkbRovFp4zFYHFFlOoQLRiawIic81gKb3U1wEyJZgMwgs3JoLtwryASRWP3Q==} + peerDependencies: + '@types/react': '>=0.0.0 <=99' + react: '>=0.0.0 <=99' + react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} @@ -8703,6 +8712,11 @@ snapshots: transitivePeerDependencies: - csstype + react-innertext@1.1.5(@types/react@18.2.79)(react@18.2.0): + dependencies: + '@types/react': 18.2.79 + react: 18.2.0 + react-is@16.13.1: {} react-is@17.0.2: {} diff --git a/frontend/src/components/Brokers/BrokersList/BrokersList.tsx b/frontend/src/components/Brokers/BrokersList/BrokersList.tsx index effba3b83..fb4dd7b5e 100644 --- a/frontend/src/components/Brokers/BrokersList/BrokersList.tsx +++ b/frontend/src/components/Brokers/BrokersList/BrokersList.tsx @@ -7,9 +7,11 @@ import { clusterBrokerPath } from 'lib/paths'; import { useBrokers } from 'lib/hooks/api/brokers'; import { useClusterStats } from 'lib/hooks/api/clusters'; import ResourcePageHeading from 'components/common/ResourcePageHeading/ResourcePageHeading'; +import { DownloadCsvButton } from 'components/common/DownloadCsvButton/DownloadCsvButton'; +import { brokersApiClient } from 'lib/api'; -import { BrokersMetrics } from './BrokersMetrics/BrokersMetrics'; import { getBrokersTableColumns, getBrokersTableRows } from './lib'; +import { BrokersMetrics } from './BrokersMetrics/BrokersMetrics'; const BrokersList: React.FC = () => { const navigate = useNavigate(); @@ -43,9 +45,18 @@ const BrokersList: React.FC = () => { const columns = useMemo(() => getBrokersTableColumns(), []); + const fetchCsv = async () => { + return brokersApiClient.getBrokersCsv({ clusterName }); + }; + return ( <> - + + + { }), columnHelper.accessor('replicasSkew', { header: SkewHeader, + meta: { csv: 'Replicas skew' }, cell: Cell.Skew, }), columnHelper.accessor('partitionsLeader', { header: 'Leaders' }), diff --git a/frontend/src/components/Connect/Clusters/ui/List/Cells/ConnectorsCell.tsx b/frontend/src/components/Connect/Clusters/ui/List/Cells/ConnectorsCell.tsx index 2319cc07d..8a2475a94 100644 --- a/frontend/src/components/Connect/Clusters/ui/List/Cells/ConnectorsCell.tsx +++ b/frontend/src/components/Connect/Clusters/ui/List/Cells/ConnectorsCell.tsx @@ -2,15 +2,23 @@ import AlertBadge from 'components/common/AlertBadge/AlertBadge'; import { Connect } from 'generated-sources'; import React from 'react'; -type Props = { connect: Connect }; -const ConnectorsCell = ({ connect }: Props) => { +export const getConnectorsCountText = (connect: Connect) => { const count = connect.connectorsCount ?? 0; const failedCount = connect.failedConnectorsCount ?? 0; - const text = `${count - failedCount}/${count}`; - if (count === 0) { - return null; - } + return { + count, + failedCount, + text: `${count - failedCount}/${count}`, + }; +}; + +type Props = { connect: Connect }; + +const ConnectorsCell = ({ connect }: Props) => { + const { count, failedCount, text } = getConnectorsCountText(connect); + + if (count === 0) return null; if (failedCount > 0) { return ( diff --git a/frontend/src/components/Connect/Clusters/ui/List/Cells/TasksCell.tsx b/frontend/src/components/Connect/Clusters/ui/List/Cells/TasksCell.tsx index 041237e0a..855a80408 100644 --- a/frontend/src/components/Connect/Clusters/ui/List/Cells/TasksCell.tsx +++ b/frontend/src/components/Connect/Clusters/ui/List/Cells/TasksCell.tsx @@ -2,15 +2,20 @@ import AlertBadge from 'components/common/AlertBadge/AlertBadge'; import { Connect } from 'generated-sources'; import React from 'react'; -type Props = { connect: Connect }; -const TasksCell = ({ connect }: Props) => { +export const getTasksCountText = (connect: Connect) => { const count = connect.tasksCount ?? 0; const failedCount = connect.failedTasksCount ?? 0; const text = `${count - failedCount}/${count}`; - if (!count) { - return null; - } + return { count, failedCount, text }; +}; + +type Props = { connect: Connect }; + +const TasksCell = ({ connect }: Props) => { + const { count, failedCount, text } = getTasksCountText(connect); + + if (!count) return null; if (failedCount > 0) { return ( diff --git a/frontend/src/components/Connect/Clusters/ui/List/List.tsx b/frontend/src/components/Connect/Clusters/ui/List/List.tsx index f0cd4e693..e8ad8813a 100644 --- a/frontend/src/components/Connect/Clusters/ui/List/List.tsx +++ b/frontend/src/components/Connect/Clusters/ui/List/List.tsx @@ -7,29 +7,31 @@ import { useNavigate } from 'react-router-dom'; import { clusterConnectorsPath } from 'lib/paths'; import { createColumnHelper } from '@tanstack/react-table'; -import ConnectorsCell from './Cells/ConnectorsCell'; +import ConnectorsCell, { getConnectorsCountText } from './Cells/ConnectorsCell'; import NameCell from './Cells/NameCell'; -import TasksCell from './Cells/TasksCell'; +import TasksCell, { getTasksCountText } from './Cells/TasksCell'; const helper = createColumnHelper(); export const columns = [ - helper.accessor('name', { cell: NameCell, size: 600 }), + helper.accessor('name', { header: 'Name', cell: NameCell, size: 600 }), helper.accessor('version', { header: 'Version', cell: ({ getValue }) => getValue(), enableSorting: true, }), - helper.display({ + helper.accessor('connectorsCount', { header: 'Connectors', id: 'connectors', cell: (props) => , size: 100, + meta: { csvFn: (row) => getConnectorsCountText(row).text }, }), - helper.display({ + helper.accessor('tasksCount', { header: 'Running tasks', id: 'tasks', cell: (props) => , size: 100, + meta: { csvFn: (row) => getTasksCountText(row).text }, }), ]; diff --git a/frontend/src/components/Connect/Connect.tsx b/frontend/src/components/Connect/Connect.tsx index 48787f5ae..a61f1435f 100644 --- a/frontend/src/components/Connect/Connect.tsx +++ b/frontend/src/components/Connect/Connect.tsx @@ -1,11 +1,11 @@ import React from 'react'; -import { Navigate, Routes, Route, NavLink } from 'react-router-dom'; +import { Navigate, NavLink, Route, Routes } from 'react-router-dom'; import { clusterConnectorsPath, + clusterConnectorsRelativePath, ClusterNameRoute, kafkaConnectClustersPath, kafkaConnectClustersRelativePath, - clusterConnectorsRelativePath, } from 'lib/paths'; import useAppParams from 'lib/hooks/useAppParams'; import Navbar from 'components/common/Navigation/Navbar.styled'; diff --git a/frontend/src/components/Connect/Header/Header.tsx b/frontend/src/components/Connect/Header/Header.tsx index 75b19a6a8..ce788a23e 100644 --- a/frontend/src/components/Connect/Header/Header.tsx +++ b/frontend/src/components/Connect/Header/Header.tsx @@ -2,22 +2,69 @@ import { ActionButton } from 'components/common/ActionComponent'; import ResourcePageHeading from 'components/common/ResourcePageHeading/ResourcePageHeading'; import Tooltip from 'components/common/Tooltip/Tooltip'; import ClusterContext from 'components/contexts/ClusterContext'; -import { ResourceType, Action } from 'generated-sources'; -import { useConnects } from 'lib/hooks/api/kafkaConnect'; +import { + Action, + ConnectorColumnsToSort, + ResourceType, + SortOrder, +} from 'generated-sources'; import useAppParams from 'lib/hooks/useAppParams'; -import { clusterConnectorNewPath, ClusterNameRoute } from 'lib/paths'; +import { + clusterConnectorNewPath, + clusterConnectorsRelativePath, + ClusterNameRoute, + kafkaConnectClustersRelativePath, +} from 'lib/paths'; import React from 'react'; +import { DownloadCsvButton } from 'components/common/DownloadCsvButton/DownloadCsvButton'; +import { kafkaConnectApiClient } from 'lib/api'; +import { connects } from 'lib/fixtures/kafkaConnect'; +import { useSearchParams } from 'react-router-dom'; +import useFts from 'components/common/Fts/useFts'; + +type ConnectPage = + | typeof kafkaConnectClustersRelativePath + | typeof clusterConnectorsRelativePath; const Header = () => { const { isReadOnly } = React.useContext(ClusterContext); - const { clusterName } = useAppParams(); - const { data: connects = [] } = useConnects(clusterName, true); + const { '*': currentPath, clusterName } = useAppParams< + ClusterNameRoute & { ['*']: ConnectPage } + >(); + const [searchParams] = useSearchParams(); + const { isFtsEnabled } = useFts('connects'); + + const fetchCsv = async () => { + if (currentPath === kafkaConnectClustersRelativePath) { + return kafkaConnectApiClient.getConnectsCsv({ + clusterName, + withStats: true, + }); + } + + if (currentPath === clusterConnectorsRelativePath) { + return kafkaConnectApiClient.getAllConnectorsCsv({ + clusterName, + search: searchParams.get('q') || undefined, + fts: isFtsEnabled, + orderBy: + (searchParams.get('sortBy') as ConnectorColumnsToSort) || undefined, + sortOrder: + (searchParams.get('sortDirection')?.toUpperCase() as SortOrder) || + undefined, + }); + } + + return ''; + }; + return ( {!isReadOnly && ( { placement="left" /> )} + + ); }; diff --git a/frontend/src/components/Connect/Header/__tests__/Header.spec.tsx b/frontend/src/components/Connect/Header/__tests__/Header.spec.tsx index 8f1bd93d1..cf7d3ca2f 100644 --- a/frontend/src/components/Connect/Header/__tests__/Header.spec.tsx +++ b/frontend/src/components/Connect/Header/__tests__/Header.spec.tsx @@ -52,11 +52,11 @@ describe('Kafka Connect header', () => { (useConnects as jest.Mock).mockImplementation(() => ({ data: [], })); - renderComponent({ isReadOnly: false }); + renderComponent({ isReadOnly: true }); const btn = screen.queryByRole('button', { name: 'Create Connector' }); - expect(btn).toBeDisabled(); + expect(btn).not.toBeInTheDocument(); }); }); }); diff --git a/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/cells/RunningTasksCell.tsx b/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/cells/RunningTasksCell.tsx index fc829df49..9f7ba53bf 100644 --- a/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/cells/RunningTasksCell.tsx +++ b/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/cells/RunningTasksCell.tsx @@ -3,19 +3,23 @@ import { FullConnectorInfo } from 'generated-sources'; import { CellContext } from '@tanstack/react-table'; import AlertBadge from 'components/common/AlertBadge/AlertBadge'; -const RunningTasksCell: React.FC> = ({ - row, -}) => { - const { tasksCount, failedTasksCount } = row.original; +export const getRunningTasksCountText = (connector: FullConnectorInfo) => { + const { tasksCount, failedTasksCount } = connector; const failedCount = failedTasksCount ?? 0; const count = tasksCount ?? 0; const text = `${count - failedCount}/${count}`; - if (!tasksCount) { - return null; - } + return { count, failedCount, text }; +}; + +const RunningTasksCell: React.FC> = ({ + row, +}) => { + const { count, failedCount, text } = getRunningTasksCountText(row.original); + + if (!count) return null; if (failedCount > 0) { return ( diff --git a/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/columns.tsx b/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/columns.tsx index 1fc65eff6..7cc92b3d3 100644 --- a/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/columns.tsx +++ b/frontend/src/components/Connect/List/ConnectorsTable/connectorsColumns/columns.tsx @@ -5,7 +5,9 @@ import { TagCell } from 'components/common/NewTable'; import { KafkaConnectLinkCell } from './cells/KafkaConnectLinkCell'; import TopicsCell from './cells/TopicsCell'; -import RunningTasksCell from './cells/RunningTasksCell'; +import RunningTasksCell, { + getRunningTasksCountText, +} from './cells/RunningTasksCell'; import ActionsCell from './cells/ActionsCell'; export const connectorsColumns: ColumnDef[] = [ @@ -43,7 +45,10 @@ export const connectorsColumns: ColumnDef[] = [ accessorKey: 'topics', cell: TopicsCell, enableColumnFilter: true, - meta: { filterVariant: 'multi-select' }, + meta: { + filterVariant: 'multi-select', + csvFn: (row) => (row.topics ? row.topics.join(', ') : '-'), + }, filterFn: 'arrIncludesSome', enableResizing: true, }, @@ -51,14 +56,16 @@ export const connectorsColumns: ColumnDef[] = [ header: 'Status', accessorKey: 'status.state', cell: TagCell, - meta: { filterVariant: 'multi-select' }, + meta: { filterVariant: 'multi-select', csvFn: (row) => row.status.state }, filterFn: 'arrIncludesSome', }, { id: 'running_task', + accessorKey: 'tasksCount', header: 'Running Tasks', cell: RunningTasksCell, size: 120, + meta: { csvFn: (row) => getRunningTasksCountText(row).text }, }, { header: '', diff --git a/frontend/src/components/Connect/List/List.tsx b/frontend/src/components/Connect/List/List.tsx deleted file mode 100644 index b8abbbcf7..000000000 --- a/frontend/src/components/Connect/List/List.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import React from 'react'; -import { ConnectorsTable } from 'components/Connect/List/ConnectorsTable/ConnectorsTable'; -import { FullConnectorInfo } from 'generated-sources'; - -interface ConnectorsListProps { - connectors: FullConnectorInfo[]; -} - -const List: React.FC = ({ connectors }) => { - return ; -}; - -export default List; diff --git a/frontend/src/components/Connect/List/ListPage.tsx b/frontend/src/components/Connect/List/ListPage.tsx index dab3fe3f1..040e3663d 100644 --- a/frontend/src/components/Connect/List/ListPage.tsx +++ b/frontend/src/components/Connect/List/ListPage.tsx @@ -11,7 +11,7 @@ import { FullConnectorInfo } from 'generated-sources'; import { FilteredConnectorsProvider } from 'components/Connect/model/FilteredConnectorsProvider'; import * as S from './ListPage.styled'; -import List from './List'; +import { ConnectorsTable } from './ConnectorsTable/ConnectorsTable'; import ConnectorsStatistics from './Statistics/Statistics'; const emptyConnectors: FullConnectorInfo[] = []; @@ -36,7 +36,7 @@ const ListPage: React.FC = () => { /> }> - + ); diff --git a/frontend/src/components/Connect/List/__tests__/List.spec.tsx b/frontend/src/components/Connect/List/__tests__/List.spec.tsx deleted file mode 100644 index a489f2fb7..000000000 --- a/frontend/src/components/Connect/List/__tests__/List.spec.tsx +++ /dev/null @@ -1,168 +0,0 @@ -import React from 'react'; -import { connectors } from 'lib/fixtures/kafkaConnect'; -import ClusterContext, { - ContextProps, - initialValue, -} from 'components/contexts/ClusterContext'; -import List from 'components/Connect/List/List'; -import { screen, waitFor } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; -import { render, WithRoute } from 'lib/testHelpers'; -import { clusterConnectConnectorPath, clusterConnectorsPath } from 'lib/paths'; -import { - useConnectors, - useDeleteConnector, - useResetConnectorOffsets, - useUpdateConnectorState, -} from 'lib/hooks/api/kafkaConnect'; -import { FullConnectorInfo } from 'generated-sources'; -import { FilteredConnectorsProvider } from 'components/Connect/model/FilteredConnectorsProvider'; - -const mockedUsedNavigate = jest.fn(); -const mockDelete = jest.fn(); -const mockResetOffsets = jest.fn(); - -jest.mock('react-router-dom', () => ({ - ...jest.requireActual('react-router-dom'), - useNavigate: () => mockedUsedNavigate, -})); - -jest.mock('lib/hooks/api/kafkaConnect', () => ({ - useConnectors: jest.fn(), - useDeleteConnector: jest.fn(), - useUpdateConnectorState: jest.fn(), - useResetConnectorOffsets: jest.fn(), -})); - -const clusterName = 'local'; - -const renderComponent = ( - contextValue: ContextProps = initialValue, - data: FullConnectorInfo[] = connectors -) => - render( - - - - - - - , - { initialEntries: [clusterConnectorsPath(clusterName)] } - ); - -describe('Connectors List', () => { - describe('when the connectors are loaded', () => { - beforeEach(() => { - const restartConnector = jest.fn(); - (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ - mutateAsync: restartConnector, - })); - }); - - it('renders', async () => { - renderComponent(); - expect(screen.getByRole('table')).toBeInTheDocument(); - expect(screen.getAllByRole('row').length).toEqual(4); - }); - - it('opens broker when row clicked', async () => { - renderComponent(); - screen.debug(); - expect(screen.getByText('hdfs-source-connector')).toHaveAttribute( - 'href', - clusterConnectConnectorPath( - clusterName, - 'first', - 'hdfs-source-connector' - ) - ); - }); - }); - - describe('when table is empty', () => { - beforeEach(() => { - (useConnectors as jest.Mock).mockImplementation(() => ({ - data: [], - })); - }); - - it('renders empty table', async () => { - renderComponent(undefined, []); - expect(screen.getByRole('table')).toBeInTheDocument(); - expect( - screen.getByRole('row', { name: 'No connectors found' }) - ).toBeInTheDocument(); - }); - }); - - describe('when delete modal is open', () => { - beforeEach(() => { - (useConnectors as jest.Mock).mockImplementation(() => ({ - data: connectors, - })); - (useDeleteConnector as jest.Mock).mockImplementation(() => ({ - mutateAsync: mockDelete, - })); - }); - - it('calls deleteConnector on confirm', async () => { - renderComponent(); - const deleteButton = screen.getAllByText('Delete')[0]; - await waitFor(() => userEvent.click(deleteButton)); - - const submitButton = screen.getAllByRole('button', { - name: 'Confirm', - })[0]; - await userEvent.click(submitButton); - expect(mockDelete).toHaveBeenCalledWith(); - }); - - it('closes the modal when cancel button is clicked', async () => { - renderComponent(); - const deleteButton = screen.getAllByText('Delete')[0]; - await waitFor(() => userEvent.click(deleteButton)); - - const cancelButton = screen.getAllByRole('button', { - name: 'Cancel', - })[0]; - await waitFor(() => userEvent.click(cancelButton)); - expect(cancelButton).not.toBeInTheDocument(); - }); - }); - - describe('when reset connector offsets modal is open', () => { - beforeEach(() => { - (useConnectors as jest.Mock).mockImplementation(() => ({ - data: connectors, - })); - (useResetConnectorOffsets as jest.Mock).mockImplementation(() => ({ - mutateAsync: mockResetOffsets, - })); - }); - - it('calls resetConnectorOffsets on confirm', async () => { - renderComponent(); - const resetButton = screen.getAllByText('Reset Offsets')[2]; - await waitFor(() => userEvent.click(resetButton)); - - const submitButton = screen.getAllByRole('button', { - name: 'Confirm', - })[0]; - await userEvent.click(submitButton); - expect(mockResetOffsets).toHaveBeenCalledWith(); - }); - - it('closes the modal when cancel button is clicked', async () => { - renderComponent(); - const resetButton = screen.getAllByText('Reset Offsets')[2]; - await waitFor(() => userEvent.click(resetButton)); - - const cancelButton = screen.getAllByRole('button', { - name: 'Cancel', - })[0]; - await waitFor(() => userEvent.click(cancelButton)); - expect(cancelButton).not.toBeInTheDocument(); - }); - }); -}); diff --git a/frontend/src/components/Connect/List/__tests__/ListPage.spec.tsx b/frontend/src/components/Connect/List/__tests__/ListPage.spec.tsx index 77196a1be..18c4a8ab3 100644 --- a/frontend/src/components/Connect/List/__tests__/ListPage.spec.tsx +++ b/frontend/src/components/Connect/List/__tests__/ListPage.spec.tsx @@ -10,10 +10,20 @@ import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectorsPath } from 'lib/paths'; import { useConnectors, useConnects } from 'lib/hooks/api/kafkaConnect'; -jest.mock('components/Connect/List/List', () => () => ( -
Connectors List
+jest.mock('components/Connect/List/ConnectorsTable/ConnectorsTable', () => ({ + ConnectorsTable: () =>
Connectors List
, +})); + +jest.mock('components/Connect/List/Statistics/Statistics', () => () => ( +
Statistics
)); +jest.mock('components/common/Fts/Fts', () => () =>
Fts
); + +jest.mock('components/common/Fts/useFts', () => () => ({ + isFtsEnabled: false, +})); + jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnectors: jest.fn(), useConnects: jest.fn(), diff --git a/frontend/src/components/ConsumerGroups/Details/Details.tsx b/frontend/src/components/ConsumerGroups/Details/Details.tsx index 006882e18..f024df640 100644 --- a/frontend/src/components/ConsumerGroups/Details/Details.tsx +++ b/frontend/src/components/ConsumerGroups/Details/Details.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { useNavigate, useSearchParams } from 'react-router-dom'; +import { useNavigate } from 'react-router-dom'; import useAppParams from 'lib/hooks/useAppParams'; import { clusterConsumerGroupResetRelativePath, @@ -10,14 +10,11 @@ import Search from 'components/common/Search/Search'; import ClusterContext from 'components/contexts/ClusterContext'; import * as Metrics from 'components/common/Metrics'; import { Tag } from 'components/common/Tag/Tag.styled'; -import groupBy from 'lib/functions/groupBy'; -import { Table } from 'components/common/table/Table/Table.styled'; import getTagColor from 'components/common/Tag/getTagColor'; import { Dropdown } from 'components/common/Dropdown'; import { ControlPanelWrapper } from 'components/common/ControlPanel/ControlPanel.styled'; import { Action, ConsumerGroupState, ResourceType } from 'generated-sources'; import { ActionDropdownItem } from 'components/common/ActionComponent'; -import TableHeaderCell from 'components/common/table/TableHeaderCell/TableHeaderCell'; import { useConsumerGroupDetails, useDeleteConsumerGroupMutation, @@ -25,18 +22,18 @@ import { import Tooltip from 'components/common/Tooltip/Tooltip'; import { CONSUMER_GROUP_STATE_TOOLTIPS } from 'lib/constants'; import ResourcePageHeading from 'components/common/ResourcePageHeading/ResourcePageHeading'; +import { exportTableCSV, TableProvider } from 'components/common/NewTable'; +import { Button } from 'components/common/Button/Button'; -import ListItem from './ListItem'; +import { TopicsTable } from './TopicsTable/TopicsTable'; const Details: React.FC = () => { const navigate = useNavigate(); - const [searchParams] = useSearchParams(); - const searchValue = searchParams.get('q') || ''; const { isReadOnly } = React.useContext(ClusterContext); const routeParams = useAppParams(); const { clusterName, consumerGroupID } = routeParams; - const consumerGroup = useConsumerGroupDetails(routeParams); + const { data: consumerGroup } = useConsumerGroupDetails(routeParams); const deleteConsumerGroup = useDeleteConsumerGroupMutation(routeParams); const onDelete = async () => { @@ -48,112 +45,103 @@ const Details: React.FC = () => { navigate(clusterConsumerGroupResetRelativePath); }; - const partitionsByTopic = groupBy( - consumerGroup.data?.partitions || [], - 'topic' - ); - const filteredPartitionsByTopic = Object.keys(partitionsByTopic).filter( - (el) => el.includes(searchValue) - ); - const currentPartitionsByTopic = searchValue.length - ? filteredPartitionsByTopic - : Object.keys(partitionsByTopic); - - const hasAssignedTopics = consumerGroup?.data?.topics !== 0; + const hasAssignedTopics = consumerGroup?.topics !== 0; return ( -
-
- - {!isReadOnly && ( - - - Reset offset - - + {({ table }) => { + const handleExportClick = () => { + exportTableCSV(table, { prefix: 'connector-topics' }); + }; + + return ( + <> +
+ - Delete consumer group - - - )} - -
- - - - - {consumerGroup.data?.state} - - } - content={ - CONSUMER_GROUP_STATE_TOOLTIPS[ - consumerGroup.data?.state || ConsumerGroupState.UNKNOWN - ] - } - placement="bottom-start" - /> - - - {consumerGroup.data?.members} - - - {consumerGroup.data?.topics} - - - {consumerGroup.data?.partitions?.length} - - - {consumerGroup.data?.coordinator?.id} - - - {consumerGroup.data?.consumerLag} - - - - - - - - - - - - - - - {currentPartitionsByTopic.map((key) => ( - - ))} - -
-
+ + + {!isReadOnly && ( + + + Reset offset + + + Delete consumer group + + + )} + +
+ + + + + {consumerGroup?.state} + + } + content={ + CONSUMER_GROUP_STATE_TOOLTIPS[ + consumerGroup?.state || ConsumerGroupState.UNKNOWN + ] + } + placement="bottom-start" + /> + + + {consumerGroup?.members} + + + {consumerGroup?.topics} + + + {consumerGroup?.partitions?.length} + + + {consumerGroup?.coordinator?.id} + + + {consumerGroup?.consumerLag} + + + + + + + + + + ); + }} + ); }; diff --git a/frontend/src/components/ConsumerGroups/Details/ListItem.styled.ts b/frontend/src/components/ConsumerGroups/Details/ListItem.styled.ts deleted file mode 100644 index 358a45e0a..000000000 --- a/frontend/src/components/ConsumerGroups/Details/ListItem.styled.ts +++ /dev/null @@ -1,7 +0,0 @@ -import styled from 'styled-components'; - -export const FlexWrapper = styled.div` - display: flex; - align-items: center; - gap: 8px; -`; diff --git a/frontend/src/components/ConsumerGroups/Details/ListItem.tsx b/frontend/src/components/ConsumerGroups/Details/ListItem.tsx deleted file mode 100644 index 19701b91d..000000000 --- a/frontend/src/components/ConsumerGroups/Details/ListItem.tsx +++ /dev/null @@ -1,85 +0,0 @@ -import React from 'react'; -import { - Action, - ConsumerGroupTopicPartition, - ResourceType, -} from 'generated-sources'; -import { Link } from 'react-router-dom'; -import { ClusterName } from 'lib/interfaces/cluster'; -import { ClusterGroupParam, clusterTopicPath } from 'lib/paths'; -import { useDeleteConsumerGroupOffsetsMutation } from 'lib/hooks/api/consumers'; -import useAppParams from 'lib/hooks/useAppParams'; -import { Dropdown } from 'components/common/Dropdown'; -import { ActionDropdownItem } from 'components/common/ActionComponent'; -import MessageToggleIcon from 'components/common/Icons/MessageToggleIcon'; -import IconButtonWrapper from 'components/common/Icons/IconButtonWrapper'; -import { TableKeyLink } from 'components/common/table/Table/TableKeyLink.styled'; - -import TopicContents from './TopicContents/TopicContents'; -import { FlexWrapper } from './ListItem.styled'; - -interface Props { - clusterName: ClusterName; - name: string; - consumers: ConsumerGroupTopicPartition[]; -} - -const ListItem: React.FC = ({ clusterName, name, consumers }) => { - const [isOpen, setIsOpen] = React.useState(false); - const consumerProps = useAppParams(); - const deleteOffsetMutation = - useDeleteConsumerGroupOffsetsMutation(consumerProps); - - const getTotalconsumerLag = () => { - if (consumers.every((consumer) => consumer?.consumerLag === null)) { - return 'N/A'; - } - let count = 0; - consumers.forEach((consumer) => { - count += consumer?.consumerLag || 0; - }); - return count; - }; - - const deleteOffsetHandler = (topicName?: string) => { - if (topicName === undefined) return; - deleteOffsetMutation.mutateAsync(topicName); - }; - - return ( - <> - - - - setIsOpen(!isOpen)} aria-hidden> - - - - {name} - - - - {getTotalconsumerLag()} - - - deleteOffsetHandler(name)} - danger - confirm="Are you sure you want to delete offsets from the topic?" - permission={{ - resource: ResourceType.CONSUMER, - action: Action.RESET_OFFSETS, - value: consumerProps.consumerGroupID, - }} - > - Delete offsets - - - - - {isOpen && } - - ); -}; - -export default ListItem; diff --git a/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContent.styled.ts b/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContent.styled.ts deleted file mode 100644 index af36dfbee..000000000 --- a/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContent.styled.ts +++ /dev/null @@ -1,18 +0,0 @@ -import styled, { css } from 'styled-components'; - -export const TopicContentWrapper = styled.tr` - background-color: ${({ theme }) => theme.default.backgroundColor}; - & > td { - padding: 16px !important; - background-color: ${({ theme }) => - theme.consumerTopicContent.td.backgroundColor}; - } -`; - -export const ContentBox = styled.div( - ({ theme }) => css` - background-color: ${theme.default.backgroundColor}; - padding: 20px; - border-radius: 8px; - ` -); diff --git a/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContents.tsx b/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContents.tsx index 8686c26ff..a6f3ebc47 100644 --- a/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContents.tsx +++ b/frontend/src/components/ConsumerGroups/Details/TopicContents/TopicContents.tsx @@ -3,8 +3,6 @@ import TableHeaderCell from 'components/common/table/TableHeaderCell/TableHeader import { ConsumerGroupTopicPartition, SortOrder } from 'generated-sources'; import React from 'react'; -import { ContentBox, TopicContentWrapper } from './TopicContent.styled'; - interface Props { consumers: ConsumerGroupTopicPartition[]; } @@ -129,40 +127,34 @@ const TopicContents: React.FC = ({ consumers }) => { }, [orderBy, sortOrder, consumers]); return ( - - - - - - - {TABLE_HEADERS_MAP.map((header) => ( - - ))} - - - - {sortedConsumers.map((consumer) => ( - - - - - - - - - ))} - -
{consumer.partition}{consumer.consumerId}{consumer.host}{consumer.consumerLag}{consumer.currentOffset}{consumer.endOffset}
- - - + + + + {TABLE_HEADERS_MAP.map((header) => ( + + ))} + + + + {sortedConsumers.map((consumer) => ( + + + + + + + + + ))} + +
{consumer.partition}{consumer.consumerId}{consumer.host}{consumer.consumerLag}{consumer.currentOffset}{consumer.endOffset}
); }; diff --git a/frontend/src/components/ConsumerGroups/Details/TopicsTable/TopicsTable.tsx b/frontend/src/components/ConsumerGroups/Details/TopicsTable/TopicsTable.tsx new file mode 100644 index 000000000..64f768e5c --- /dev/null +++ b/frontend/src/components/ConsumerGroups/Details/TopicsTable/TopicsTable.tsx @@ -0,0 +1,35 @@ +import React from 'react'; +import Table from 'components/common/NewTable'; +import { ConsumerGroupTopicPartition } from 'generated-sources'; +import { useSearchParams } from 'react-router-dom'; +import TopicContents from 'components/ConsumerGroups/Details/TopicContents/TopicContents'; + +import { + getConsumerGroupTopicsTableColumns, + getConsumerGroupTopicsTableData, +} from './lib/utils'; + +type TopicsTableProps = { + partitions: ConsumerGroupTopicPartition[]; +}; + +export const TopicsTable = ({ partitions }: TopicsTableProps) => { + const [searchParams] = useSearchParams(); + const searchQuery = searchParams.get('q') || ''; + + const columns = getConsumerGroupTopicsTableColumns(); + const tableData = getConsumerGroupTopicsTableData({ + partitions, + searchQuery, + }); + + return ( + true} + columns={columns} + data={tableData} + emptyMessage="No topics" + renderSubComponent={() => } + /> + ); +}; diff --git a/frontend/src/components/ConsumerGroups/Details/TopicsTable/cells/cells.tsx b/frontend/src/components/ConsumerGroups/Details/TopicsTable/cells/cells.tsx new file mode 100644 index 000000000..cd46f2f95 --- /dev/null +++ b/frontend/src/components/ConsumerGroups/Details/TopicsTable/cells/cells.tsx @@ -0,0 +1,70 @@ +import React from 'react'; +import { CellContext } from '@tanstack/react-table'; +import { Link } from 'react-router-dom'; +import { ClusterGroupParam, clusterTopicPath } from 'lib/paths'; +import { TableKeyLink } from 'components/common/table/Table/TableKeyLink.styled'; +import useAppParams from 'lib/hooks/useAppParams'; +import { ConsumerGroupTopicsTableRow } from 'components/ConsumerGroups/Details/TopicsTable/lib/types'; +import { ActionDropdownItem } from 'components/common/ActionComponent'; +import { Action, ResourceType } from 'generated-sources'; +import { Dropdown } from 'components/common/Dropdown'; +import { useDeleteConsumerGroupOffsetsMutation } from 'lib/hooks/api/consumers'; + +type TopicNameProps = CellContext< + ConsumerGroupTopicsTableRow, + ConsumerGroupTopicsTableRow['topicName'] +>; + +export const TopicName = ({ getValue }: TopicNameProps) => { + const routeParams = useAppParams(); + const { clusterName } = routeParams; + const topicName = getValue(); + + return ( + + {topicName} + + ); +}; + +type ConsumerLagProps = CellContext< + ConsumerGroupTopicsTableRow, + ConsumerGroupTopicsTableRow['consumerLag'] +>; + +export const ConsumerLag = ({ getValue }: ConsumerLagProps) => getValue(); + +type ActionsProps = CellContext< + ConsumerGroupTopicsTableRow, + ConsumerGroupTopicsTableRow['topicName'] +>; + +export const Actions = ({ getValue }: ActionsProps) => { + const routeParams = useAppParams(); + const topicName = getValue(); + + const deleteOffsetMutation = + useDeleteConsumerGroupOffsetsMutation(routeParams); + + const deleteOffsetHandler = () => { + if (topicName === undefined) return; + deleteOffsetMutation.mutateAsync(topicName); + }; + + return ( + + + Delete offsets + + + ); +}; diff --git a/frontend/src/components/ConsumerGroups/Details/TopicsTable/lib/types.ts b/frontend/src/components/ConsumerGroups/Details/TopicsTable/lib/types.ts new file mode 100644 index 000000000..a6228a750 --- /dev/null +++ b/frontend/src/components/ConsumerGroups/Details/TopicsTable/lib/types.ts @@ -0,0 +1,6 @@ +import { Topic } from 'generated-sources'; + +export type ConsumerGroupTopicsTableRow = { + topicName: Topic['name']; + consumerLag: string | number; +}; diff --git a/frontend/src/components/ConsumerGroups/Details/TopicsTable/lib/utils.ts b/frontend/src/components/ConsumerGroups/Details/TopicsTable/lib/utils.ts new file mode 100644 index 000000000..6e5df177c --- /dev/null +++ b/frontend/src/components/ConsumerGroups/Details/TopicsTable/lib/utils.ts @@ -0,0 +1,61 @@ +import { ConsumerGroupTopicPartition } from 'generated-sources'; +import { createColumnHelper } from '@tanstack/react-table'; +import { NA } from 'components/Brokers/BrokersList/lib'; +import * as Cell from 'components/ConsumerGroups/Details/TopicsTable/cells/cells'; + +import { ConsumerGroupTopicsTableRow } from './types'; + +const getConsumerLagByTopic = (partitions: ConsumerGroupTopicPartition[]) => + partitions.reduce>( + (acc, p) => ({ + ...acc, + [p.topic]: [...(acc[p.topic] ?? []), p.consumerLag ?? 0], + }), + {} + ); + +const calculateConsumerLag = (lags: number[]) => { + const nonNullLags = lags.filter((x) => x != null); + return nonNullLags.length === 0 ? NA : nonNullLags.reduce((a, v) => a + v, 0); +}; + +export const getConsumerGroupTopicsTableData = ({ + partitions = [], + searchQuery, +}: { + partitions: ConsumerGroupTopicPartition[]; + searchQuery: string; +}): ConsumerGroupTopicsTableRow[] => { + if (partitions.length === 0) return []; + + const grouped = getConsumerLagByTopic(partitions); + return Object.entries(grouped) + .filter(([topic]) => topic.includes(searchQuery)) + .map(([topic, lags]) => ({ + topicName: topic, + consumerLag: calculateConsumerLag(lags), + })); +}; + +export const getConsumerGroupTopicsTableColumns = () => { + const columnHelper = createColumnHelper(); + + return [ + columnHelper.accessor('topicName', { + header: 'Topic', + cell: Cell.TopicName, + size: 800, + }), + columnHelper.accessor('consumerLag', { + header: 'Consumer lag', + cell: Cell.ConsumerLag, + size: 350, + }), + columnHelper.accessor('topicName', { + id: 'actions', + header: undefined, + cell: Cell.Actions, + size: 10, + }), + ]; +}; diff --git a/frontend/src/components/ConsumerGroups/List.tsx b/frontend/src/components/ConsumerGroups/List.tsx index ad66e718c..bf382c382 100644 --- a/frontend/src/components/ConsumerGroups/List.tsx +++ b/frontend/src/components/ConsumerGroups/List.tsx @@ -19,6 +19,8 @@ import ResourcePageHeading from 'components/common/ResourcePageHeading/ResourceP import { useLocalStoragePersister } from 'components/common/NewTable/ColumnResizer/lib'; import useFts from 'components/common/Fts/useFts'; import Fts from 'components/common/Fts/Fts'; +import { DownloadCsvButton } from 'components/common/DownloadCsvButton/DownloadCsvButton'; +import { consumerGroupsApiClient } from 'lib/api'; const List = () => { const { clusterName } = useAppParams(); @@ -26,16 +28,20 @@ const List = () => { const navigate = useNavigate(); const { isFtsEnabled } = useFts('consumer_groups'); - const consumerGroups = useConsumerGroups({ + const params = { clusterName, orderBy: (searchParams.get('sortBy') as ConsumerGroupOrdering) || undefined, sortOrder: (searchParams.get('sortDirection')?.toUpperCase() as SortOrder) || undefined, - page: Number(searchParams.get('page') || 1), - perPage: Number(searchParams.get('perPage') || PER_PAGE), search: searchParams.get('q') || '', fts: isFtsEnabled, + }; + + const consumerGroups = useConsumerGroups({ + ...params, + page: Number(searchParams.get('page') || 1), + perPage: Number(searchParams.get('perPage') || PER_PAGE), }); const columns = React.useMemo[]>( @@ -53,6 +59,9 @@ const List = () => { /> ), size: 600, + meta: { + csvFn: (row) => row.groupId, + }, }, { id: ConsumerGroupOrdering.MEMBERS, @@ -80,6 +89,9 @@ const List = () => { accessorKey: 'coordinator.id', enableSorting: false, size: 104, + meta: { + csvFn: (row) => String(row.coordinator?.id) || '-', + }, }, { id: ConsumerGroupOrdering.STATE, @@ -97,6 +109,9 @@ const List = () => { ); }, size: 124, + meta: { + csvFn: (row) => String(row.state), + }, }, ], [] @@ -104,9 +119,18 @@ const List = () => { const columnSizingPersister = useLocalStoragePersister('Consumers'); + const fetchCsv = async () => { + return consumerGroupsApiClient.getConsumerGroupsCsv(params); + }; + return ( <> - + + + { + const { clusterName } = useAppParams<{ clusterName: ClusterName }>(); const { isReadOnly } = React.useContext(ClusterContext); const [searchParams, setSearchParams] = useSearchParams(); - - useFts('topics'); + const { isFtsEnabled } = useFts('topics'); // Set the search params to the url based on the localStorage value React.useEffect(() => { @@ -48,22 +58,44 @@ const ListPage: React.FC = () => { setSearchParams(searchParams); }; + const params: GetTopicsRequest = { + clusterName, + showInternal: !searchParams.has('hideInternal'), + search: searchParams.get('q') || undefined, + orderBy: (searchParams.get('sortBy') as TopicColumnsToSort) || undefined, + sortOrder: + (searchParams.get('sortDirection')?.toUpperCase() as SortOrder) || + undefined, + fts: isFtsEnabled, + }; + + const fetchCsv = async () => { + return topicsApiClient.getTopicsCsv(params); + }; + return ( <> - {!isReadOnly && ( - - Add a Topic - - )} + <> + {!isReadOnly && ( + + Add a Topic + + )} + + + { }> - + ); diff --git a/frontend/src/components/Topics/List/TopicTable.tsx b/frontend/src/components/Topics/List/TopicTable.tsx index 1ce85b5e6..a4245a862 100644 --- a/frontend/src/components/Topics/List/TopicTable.tsx +++ b/frontend/src/components/Topics/List/TopicTable.tsx @@ -1,37 +1,26 @@ import React from 'react'; -import { SortOrder, Topic, TopicColumnsToSort } from 'generated-sources'; +import { GetTopicsRequest, Topic, TopicColumnsToSort } from 'generated-sources'; import { ColumnDef } from '@tanstack/react-table'; import Table, { SizeCell } from 'components/common/NewTable'; -import useAppParams from 'lib/hooks/useAppParams'; -import { ClusterName } from 'lib/interfaces/cluster'; import { useSearchParams } from 'react-router-dom'; import ClusterContext from 'components/contexts/ClusterContext'; import { useTopics } from 'lib/hooks/api/topics'; import { PER_PAGE } from 'lib/constants'; import { useLocalStoragePersister } from 'components/common/NewTable/ColumnResizer/lib'; -import useFts from 'components/common/Fts/useFts'; +import { formatBytes } from 'components/common/BytesFormatted/utils'; import { TopicTitleCell } from './TopicTitleCell'; import ActionsCell from './ActionsCell'; import BatchActionsbar from './BatchActionsBar'; -const TopicTable: React.FC = () => { - const { clusterName } = useAppParams<{ clusterName: ClusterName }>(); +const TopicTable: React.FC<{ params: GetTopicsRequest }> = ({ params }) => { const [searchParams] = useSearchParams(); const { isReadOnly } = React.useContext(ClusterContext); - const { isFtsEnabled } = useFts('topics'); const { data } = useTopics({ - clusterName, - fts: isFtsEnabled, + ...params, page: Number(searchParams.get('page') || 1), perPage: Number(searchParams.get('perPage') || PER_PAGE), - search: searchParams.get('q') || undefined, - showInternal: !searchParams.has('hideInternal'), - orderBy: (searchParams.get('sortBy') as TopicColumnsToSort) || undefined, - sortOrder: - (searchParams.get('sortDirection')?.toUpperCase() as SortOrder) || - undefined, }); const topics = data?.topics || []; @@ -93,6 +82,9 @@ const TopicTable: React.FC = () => { accessorKey: 'segmentSize', size: 100, cell: SizeCell, + meta: { + csvFn: (row: Topic) => formatBytes(row.segmentSize, 0), + }, }, { id: 'actions', diff --git a/frontend/src/components/Topics/List/__tests__/TopicTable.spec.tsx b/frontend/src/components/Topics/List/__tests__/TopicTable.spec.tsx index de4ad3ebd..82d5d464d 100644 --- a/frontend/src/components/Topics/List/__tests__/TopicTable.spec.tsx +++ b/frontend/src/components/Topics/List/__tests__/TopicTable.spec.tsx @@ -64,7 +64,7 @@ describe('TopicTable Components', () => { }} > - + , { diff --git a/frontend/src/components/common/BytesFormatted/BytesFormatted.tsx b/frontend/src/components/common/BytesFormatted/BytesFormatted.tsx index 5bfc5a167..b2f3cf6cf 100644 --- a/frontend/src/components/common/BytesFormatted/BytesFormatted.tsx +++ b/frontend/src/components/common/BytesFormatted/BytesFormatted.tsx @@ -1,5 +1,6 @@ import React from 'react'; +import { formatBytes } from './utils'; import { NoWrap } from './BytesFormatted.styled'; interface Props { @@ -7,23 +8,11 @@ interface Props { precision?: number; } -export const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; - const BytesFormatted: React.FC = ({ value, precision = 0 }) => { - const formattedValue = React.useMemo((): string => { - try { - const bytes = typeof value === 'string' ? parseInt(value, 10) : value; - if (Number.isNaN(bytes) || (bytes && bytes < 0)) return `-Bytes`; - if (!bytes || bytes < 1024) return `${Math.ceil(bytes || 0)} ${sizes[0]}`; - const pow = Math.floor(Math.log2(bytes) / 10); - const multiplier = 10 ** (precision < 0 ? 0 : precision); - return `${Math.round((bytes * multiplier) / 1024 ** pow) / multiplier} ${ - sizes[pow] - }`; - } catch (e) { - return `-Bytes`; - } - }, [precision, value]); + const formattedValue = React.useMemo( + () => formatBytes(value, precision), + [precision, value] + ); return {formattedValue}; }; diff --git a/frontend/src/components/common/BytesFormatted/__tests__/BytesFormatted.spec.tsx b/frontend/src/components/common/BytesFormatted/__tests__/BytesFormatted.spec.tsx index e97767fa2..6cd94dd99 100644 --- a/frontend/src/components/common/BytesFormatted/__tests__/BytesFormatted.spec.tsx +++ b/frontend/src/components/common/BytesFormatted/__tests__/BytesFormatted.spec.tsx @@ -1,8 +1,7 @@ import React from 'react'; -import BytesFormatted, { - sizes, -} from 'components/common/BytesFormatted/BytesFormatted'; +import BytesFormatted from 'components/common/BytesFormatted/BytesFormatted'; import { render, screen } from '@testing-library/react'; +import { sizes } from 'components/common/BytesFormatted/utils'; describe('BytesFormatted', () => { it('renders Bytes correctly', () => { diff --git a/frontend/src/components/common/BytesFormatted/utils.ts b/frontend/src/components/common/BytesFormatted/utils.ts new file mode 100644 index 000000000..04c1c24b9 --- /dev/null +++ b/frontend/src/components/common/BytesFormatted/utils.ts @@ -0,0 +1,19 @@ +export const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + +export const formatBytes = ( + value: string | number | undefined, + precision: number = 0 +): string => { + try { + const bytes = typeof value === 'string' ? parseInt(value, 10) : value; + if (Number.isNaN(bytes) || (bytes && bytes < 0)) return '-Bytes'; + if (!bytes || bytes < 1024) return `${Math.ceil(bytes || 0)} Bytes`; + + const pow = Math.floor(Math.log2(bytes) / 10); + const multiplier = 10 ** (precision < 0 ? 0 : precision); + + return `${Math.round((bytes * multiplier) / 1024 ** pow) / multiplier} ${sizes[pow]}`; + } catch (e) { + return '-Bytes'; + } +}; diff --git a/frontend/src/components/common/DownloadCsvButton/DownloadCsvButton.tsx b/frontend/src/components/common/DownloadCsvButton/DownloadCsvButton.tsx new file mode 100644 index 000000000..8c9bf8183 --- /dev/null +++ b/frontend/src/components/common/DownloadCsvButton/DownloadCsvButton.tsx @@ -0,0 +1,51 @@ +import React, { useState } from 'react'; +import { Button } from 'components/common/Button/Button'; + +type CsvFetcher = () => Promise; + +interface DownloadCsvButtonProps { + fetchCsv: CsvFetcher; + filePrefix: string; +} + +export function DownloadCsvButton({ + fetchCsv, + filePrefix, +}: DownloadCsvButtonProps) { + const [isDownloading, setIsDownloading] = useState(false); + + const handleDownload = async () => { + setIsDownloading(true); + try { + const csvString = await fetchCsv(); + + const blob = new Blob([csvString], { type: 'text/csv' }); + const url = URL.createObjectURL(blob); + + const a = document.createElement('a'); + a.href = url; + + const dateStr = new Date().toISOString().slice(0, 10); + a.download = `${filePrefix}-${dateStr}.csv`; + + document.body.appendChild(a); + a.click(); + + document.body.removeChild(a); + URL.revokeObjectURL(url); + } finally { + setIsDownloading(false); + } + }; + + return ( + + ); +} diff --git a/frontend/src/components/common/NewTable/Provider/context.ts b/frontend/src/components/common/NewTable/Provider/context.ts new file mode 100644 index 000000000..41870eb00 --- /dev/null +++ b/frontend/src/components/common/NewTable/Provider/context.ts @@ -0,0 +1,16 @@ +import { createContext, useContext, useMemo } from 'react'; +import type { Table } from '@tanstack/react-table'; + +export type TableContextValue = { + table: Table | null; + setTable: (t: Table) => void; +}; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const TableContext = createContext | null>(null); + +export const useTableInstance = () => { + const ctx = useContext | null>(TableContext); + + return useMemo(() => ctx, [ctx]); +}; diff --git a/frontend/src/components/common/NewTable/Provider/index.ts b/frontend/src/components/common/NewTable/Provider/index.ts new file mode 100644 index 000000000..5d6c54b24 --- /dev/null +++ b/frontend/src/components/common/NewTable/Provider/index.ts @@ -0,0 +1,2 @@ +export { TableProvider } from './provider'; +export { useTableInstance } from './context'; diff --git a/frontend/src/components/common/NewTable/Provider/provider.tsx b/frontend/src/components/common/NewTable/Provider/provider.tsx new file mode 100644 index 000000000..1c284b189 --- /dev/null +++ b/frontend/src/components/common/NewTable/Provider/provider.tsx @@ -0,0 +1,20 @@ +import React, { useMemo, useState } from 'react'; +import type { Table } from '@tanstack/react-table'; + +import { TableContext, TableContextValue } from './context'; + +type TableProviderProps = { + children: React.ReactNode | ((ctx: TableContextValue) => React.ReactNode); +}; + +export function TableProvider({ children }: TableProviderProps) { + const [table, setTable] = useState | null>(null); + + const value = useMemo(() => ({ table, setTable }), [table, setTable]); + + return ( + + {typeof children === 'function' ? children(value) : children} + + ); +} diff --git a/frontend/src/components/common/NewTable/Table.tsx b/frontend/src/components/common/NewTable/Table.tsx index 665f141c6..f859c0f88 100644 --- a/frontend/src/components/common/NewTable/Table.tsx +++ b/frontend/src/components/common/NewTable/Table.tsx @@ -32,6 +32,7 @@ import SelectRowCell from './SelectRowCell'; import SelectRowHeader from './SelectRowHeader'; import ColumnFilter, { type Persister } from './ColumnFilter'; import { ColumnSizingPersister } from './ColumnResizer/lib/persister/types'; +import { useTableInstance } from './Provider'; export interface TableProps { data: TData[]; @@ -170,6 +171,8 @@ function Table({ const [searchParams, setSearchParams] = useSearchParams(); const location = useLocation(); + const ctx = useTableInstance(); + const [rowSelection, setRowSelection] = React.useState({}); const onSortingChange = React.useCallback( @@ -265,14 +268,20 @@ function Table({ }, }); + useEffect(() => { + ctx?.setTable(table); + }, [table]); + const columnSizeVars = React.useMemo(() => { const headers = table.getFlatHeaders(); const colSizes: { [key: string]: number } = {}; for (let i = 0; i < headers.length; i += 1) { const header = headers[i]!; + colSizes[`--header-${header.id}-size`] = header.getSize(); colSizes[`--col-${header.column.id}-size`] = header.column.getSize(); } + return colSizes; }, [table.getState().columnSizingInfo, table.getState().columnSizing]); diff --git a/frontend/src/components/common/NewTable/__test__/Table.spec.tsx b/frontend/src/components/common/NewTable/__test__/Table.spec.tsx index 73d044c00..8f9bed485 100644 --- a/frontend/src/components/common/NewTable/__test__/Table.spec.tsx +++ b/frontend/src/components/common/NewTable/__test__/Table.spec.tsx @@ -7,6 +7,7 @@ import Table, { LinkCell, TagCell, } from 'components/common/NewTable'; +import { TableProvider } from 'components/common/NewTable/Provider'; import { screen } from '@testing-library/dom'; import { ColumnDef, Row } from '@tanstack/react-table'; import userEvent from '@testing-library/user-event'; @@ -101,12 +102,14 @@ interface Props extends TableProps { const renderComponent = (props: Partial = {}) => { render( -
+ +
+ , { initialEntries: [props.path || ''] } ); diff --git a/frontend/src/components/common/NewTable/index.ts b/frontend/src/components/common/NewTable/index.ts index 4584db2a5..c8ce6a5c0 100644 --- a/frontend/src/components/common/NewTable/index.ts +++ b/frontend/src/components/common/NewTable/index.ts @@ -4,6 +4,9 @@ import SizeCell from './SizeCell'; import LinkCell from './LinkCell'; import TagCell from './TagCell'; +export { TableProvider, useTableInstance } from './Provider'; +export { exportTableCSV } from './utils/exportTableCSV'; + export type { TableProps }; export { TimestampCell, SizeCell, LinkCell, TagCell }; diff --git a/frontend/src/components/common/NewTable/utils/exportTableCSV.ts b/frontend/src/components/common/NewTable/utils/exportTableCSV.ts new file mode 100644 index 000000000..4786a3bba --- /dev/null +++ b/frontend/src/components/common/NewTable/utils/exportTableCSV.ts @@ -0,0 +1,118 @@ +import type { RowData, Table } from '@tanstack/react-table'; +import innerText from 'react-innertext'; + +const escapeCsv = (value: string) => { + if (value.includes(',') || value.includes('"') || value.includes('\n')) { + return `"${value.replace(/"/g, '""')}"`; + } + return value; +}; + +export type ExportCsvOptions = { + filename?: string; + prefix?: string; + includeDate?: boolean; + dateFormat?: (d: Date) => string; +}; + +export const exportTableCSV = ( + table: Table | null | undefined, + options: ExportCsvOptions = {} +) => { + if (!table) return; + + const { + filename, + prefix = 'table_data', + includeDate = true, + dateFormat = (d: Date) => d.toISOString().slice(0, 10), + } = options; + + const rowsToExport = table.getSelectedRowModel().rows.length + ? table.getSelectedRowModel().rows + : table.getRowModel().rows; + + if (!rowsToExport.length) return; + + const headersColumns = table.getAllColumns().filter((col) => { + const header = col.columnDef.meta?.csv ?? col.columnDef.header; + const hasAccessorKey = + 'accessorKey' in col.columnDef && col.columnDef.accessorKey; + return header && hasAccessorKey; + }); + + const headers = headersColumns.map( + (col) => col.columnDef.meta?.csv ?? col.columnDef.header ?? col.id + ); + + const body = rowsToExport.map((row) => + headersColumns.map((col) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const original = row.original as Record; + + if (col.columnDef.meta?.csvFn) { + return escapeCsv(String(col.columnDef.meta.csvFn(row.original))); + } + + if (col.columnDef.cell && typeof col.columnDef.cell === 'function') { + try { + const accessorKey = + 'accessorKey' in col.columnDef + ? (col.columnDef.accessorKey as string) + : undefined; + + const cellValue = col.columnDef.cell({ + getValue: () => (accessorKey ? original[accessorKey] : undefined), + row, + column: col, + table, + cell: row.getAllCells().find((c) => c.column.id === col.id)!, + renderValue: () => + accessorKey ? original[accessorKey] : undefined, + }); + + if ( + cellValue && + typeof cellValue === 'object' && + 'props' in cellValue + ) { + return escapeCsv(innerText(cellValue) || ''); + } + + return escapeCsv(String(cellValue ?? '')); + } catch (error) { + // eslint-disable-next-line no-console + console.warn('CSV export: cell renderer failed', error); + } + } + + if ('accessorKey' in col.columnDef && col.columnDef.accessorKey) { + const accessorKey = col.columnDef.accessorKey as string; + const value = original[accessorKey]; + if (value === null || value === undefined) { + return ''; + } + + if (typeof value === 'object') { + return escapeCsv(JSON.stringify(value)); + } + return escapeCsv(String(value)); + } + + return ''; + }) + ); + + const csv = [headers.join(','), ...body.map((r) => r.join(','))].join('\n'); + + const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' }); + const datePart = includeDate ? `_${dateFormat(new Date())}` : ''; + const file = filename ?? `${prefix}${datePart}.csv`; + + const link = document.createElement('a'); + link.href = window.URL.createObjectURL(blob); + link.setAttribute('download', file); + document.body.appendChild(link); + link.click(); + link.remove(); +}; diff --git a/frontend/src/components/common/ResourcePageHeading/ResourcePageHeading.tsx b/frontend/src/components/common/ResourcePageHeading/ResourcePageHeading.tsx index b3cf9f163..9b5d2619e 100644 --- a/frontend/src/components/common/ResourcePageHeading/ResourcePageHeading.tsx +++ b/frontend/src/components/common/ResourcePageHeading/ResourcePageHeading.tsx @@ -7,6 +7,7 @@ type ResourcePageHeadingProps = ComponentProps; const ResourcePageHeading: FC = (props) => { const { clusterName } = useAppParams<{ clusterName: ClusterName }>(); + return ; }; diff --git a/frontend/src/tanstack.d.ts b/frontend/src/tanstack.d.ts index b50d25efb..df607d41e 100644 --- a/frontend/src/tanstack.d.ts +++ b/frontend/src/tanstack.d.ts @@ -6,6 +6,8 @@ declare module '@tanstack/react-table' { interface ColumnMeta { filterVariant?: 'multi-select' | 'text'; width?: string; + csv?: string; + csvFn?: (row: TData) => string; } interface FilterFns { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6f6eb55a8..0c7fc6dab 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -135,3 +135,5 @@ snappy = {module = 'org.xerial.snappy:snappy-java', version = '1.1.10.7'} lucene = {module = 'org.apache.lucene:lucene-core', version.ref = 'lucene'} lucene-queryparser = {module = 'org.apache.lucene:lucene-queryparser', version.ref = 'lucene'} lucene-analysis-common = {module = 'org.apache.lucene:lucene-analysis-common', version.ref = 'lucene'} + +fastcsv = {module = 'de.siegmar:fastcsv', version = '4.1.0'}