From 2a9484955da5a9a7f83e089f6c30ed770e04b4c7 Mon Sep 17 00:00:00 2001 From: slonka Date: Wed, 7 Apr 2021 11:48:08 +0200 Subject: [PATCH 01/46] Use jcp version with delta xds --- build.gradle | 2 +- .../envoycontrol/GroupCacheStatusInfo.java | 13 +- .../servicemesh/envoycontrol/SimpleCache.java | 321 +++++++++++++++++- .../envoycontrol/groups/GroupChangeWatcher.kt | 23 ++ .../groups/NodeMetadataValidator.kt | 12 + .../CompositeDiscoveryServerCallbacks.kt | 16 + .../LoggingDiscoveryServerCallbacks.kt | 31 ++ .../MetricsDiscoveryServerCallbacks.kt | 14 + .../envoycontrol/snapshot/GlobalSnapshot.kt | 10 +- .../envoycontrol/v2/SimpleCacheTest.java | 10 +- .../SimpleCacheWithMissingEndpointsTest.java | 8 +- .../envoycontrol/v3/SimpleCacheTest.java | 3 +- .../SimpleCacheWithMissingEndpointsTest.java | 8 +- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 9 +- .../snapshot/SnapshotUpdaterTest.kt | 15 + .../filters/RBACFilterFactoryTest.kt | 19 +- 16 files changed, 460 insertions(+), 54 deletions(-) diff --git a/build.gradle b/build.gradle index 174ec0d3a..de202b4a2 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ allprojects { project.ext.versions = [ kotlin : '1.3.72', - java_controlplane : '0.1.27', + java_controlplane : '0.1.29-delta-xds-slonka-SNAPSHOT', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', diff --git a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java index de714b084..3c10fc8c3 100644 --- a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java +++ b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java @@ -8,7 +8,6 @@ /** * {@code GroupCacheStatusInfo} provides an implementation of {@link StatusInfo} for a group of {@link CacheStatusInfo}. - * This class is copy of {@link io.envoyproxy.controlplane.cache.GroupCacheStatusInfo} */ @ThreadSafe class GroupCacheStatusInfo implements StatusInfo { @@ -26,6 +25,11 @@ public long lastWatchRequestTime() { return statuses.stream().mapToLong(CacheStatusInfo::lastWatchRequestTime).max().orElse(0); } + @Override + public long lastDeltaWatchRequestTime() { + return statuses.stream().mapToLong(CacheStatusInfo::lastDeltaWatchRequestTime).max().orElse(0); + } + /** * {@inheritDoc} */ @@ -41,4 +45,9 @@ public T nodeGroup() { public int numWatches() { return statuses.stream().mapToInt(CacheStatusInfo::numWatches).sum(); } -} \ No newline at end of file + + @Override + public int numDeltaWatches() { + return statuses.stream().mapToInt(CacheStatusInfo::numDeltaWatches).sum(); + } +} diff --git a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java index 09a1c59b1..f266336c5 100644 --- a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java +++ b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java @@ -2,16 +2,21 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.protobuf.Message; import io.envoyproxy.controlplane.cache.CacheStatusInfo; +import io.envoyproxy.controlplane.cache.DeltaResponse; +import io.envoyproxy.controlplane.cache.DeltaWatch; +import io.envoyproxy.controlplane.cache.DeltaXdsRequest; import io.envoyproxy.controlplane.cache.NodeGroup; import io.envoyproxy.controlplane.cache.Resources; import io.envoyproxy.controlplane.cache.Response; import io.envoyproxy.controlplane.cache.Snapshot; import io.envoyproxy.controlplane.cache.SnapshotCache; import io.envoyproxy.controlplane.cache.StatusInfo; +import io.envoyproxy.controlplane.cache.VersionedResource; import io.envoyproxy.controlplane.cache.Watch; import io.envoyproxy.controlplane.cache.WatchCancelledException; import io.envoyproxy.controlplane.cache.XdsRequest; @@ -23,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -34,6 +40,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.function.Function; import static io.envoyproxy.controlplane.cache.Resources.RESOURCE_TYPES_IN_ORDER; @@ -199,7 +206,148 @@ public Watch createWatch( /** * {@inheritDoc} - * @return + */ + @Override + public DeltaWatch createDeltaWatch( + DeltaXdsRequest request, + String requesterVersion, + Map resourceVersions, + Set pendingResources, + boolean isWildcard, + Consumer responseConsumer, + boolean hasClusterChanged) { + + Resources.ResourceType requestResourceType = request.getResourceType(); + Preconditions.checkNotNull(requestResourceType, "unsupported type URL %s", + request.getTypeUrl()); + T group; + if (request.v3Request() != null) { + group = groups.hash(request.v3Request().getNode()); + } else { + group = groups.hash(request.v2Request().getNode()); + } + + // even though we're modifying, we take a readLock to allow multiple watches to be created in parallel since it + // doesn't conflict + readLock.lock(); + try { + CacheStatusInfo status = statuses.computeIfAbsent(group, g -> new ConcurrentHashMap<>()) + .computeIfAbsent(requestResourceType, s -> new CacheStatusInfo<>(group)); + status.setLastWatchRequestTime(System.currentTimeMillis()); + + U snapshot = snapshots.get(group); + String version = snapshot == null ? "" : snapshot.version(requestResourceType, Collections.emptyList()); + + DeltaWatch watch = new DeltaWatch(request, + ImmutableMap.copyOf(resourceVersions), + ImmutableSet.copyOf(pendingResources), + requesterVersion, + isWildcard, + responseConsumer); + + // If no snapshot, leave an open watch. + + if (snapshot == null) { + long watchId = setDeltaWatch(status, watch); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("open watch {} for {}[{}] from node {} for version {}", + watchId, + request.getTypeUrl(), + String.join(", ", watch.trackedResources().keySet()), + group, + requesterVersion); + } + + return watch; + } + + // If the requested version is up-to-date or missing a response, leave an open watch. + if (version.equals(requesterVersion)) { + // If the request is not wildcard, we have pending resources and we have them, we should respond immediately. + if (!isWildcard && watch.pendingResources().size() != 0) { + // If any of the pending resources are in the snapshot respond immediately. If not we'll fall back to + // version comparisons. + Map> resources = snapshot.versionedResources(request.getResourceType()); + Map> requestedResources = watch.pendingResources() + .stream() + .filter(resources::containsKey) + .collect(Collectors.toMap(Function.identity(), resources::get)); + ResponseState responseState = respondDelta(watch, + requestedResources, + Collections.emptyList(), + version, + group); + if (responseState.equals(ResponseState.RESPONDED) || responseState.equals(ResponseState.CANCELLED)) { + return watch; + } + } else if (hasClusterChanged && requestResourceType.equals(Resources.ResourceType.ENDPOINT)) { + Map> snapshotResources = snapshot.versionedResources(request.getResourceType()); + List removedResources = findRemovedResources(watch, + snapshotResources); + Map> changedResources = findChangedResources(watch, snapshotResources); + ResponseState responseState = respondDelta( + watch, + changedResources, + removedResources, + version, + group); + if (responseState.equals(ResponseState.RESPONDED) || responseState.equals(ResponseState.CANCELLED)) { + return watch; + } + } + + long watchId = setDeltaWatch(status, watch); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("open watch {} for {}[{}] from node {} for version {}", + watchId, + request.getTypeUrl(), + String.join(", ", watch.trackedResources().keySet()), + group, + requesterVersion); + } + + return watch; + } + + // Otherwise, version is different, the watch may be responded immediately + Map> snapshotResources = snapshot.versionedResources(request.getResourceType()); + List removedResources = findRemovedResources(watch, + snapshotResources); + Map> changedResources = findChangedResources(watch, snapshotResources); + ResponseState responseState = respondDelta(watch, + changedResources, + removedResources, + version, + group); + if (responseState.equals(ResponseState.RESPONDED) || responseState.equals(ResponseState.CANCELLED)) { + return watch; + } + + long watchId = setDeltaWatch(status, watch); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("did not respond immediately, leaving open watch {} for {}[{}] from node {} for version {}", + watchId, + request.getTypeUrl(), + String.join(", ", watch.trackedResources().keySet()), + group, + requesterVersion); + } + + return watch; + } finally { + readLock.unlock(); + } + } + + private long setDeltaWatch(CacheStatusInfo status, DeltaWatch watch) { + long watchId = watchCount.incrementAndGet(); + status.setDeltaWatch(watchId, watch); + watch.setStop(() -> status.removeDeltaWatch(watchId)); + return watchId; + } + + /** + * {@inheritDoc} */ @Override public U getSnapshot(T group) { @@ -230,10 +378,11 @@ public Collection groups() { public void setSnapshot(T group, U snapshot) { // we take a writeLock to prevent watches from being created while we update the snapshot ConcurrentMap> status; + U previousSnapshot; writeLock.lock(); try { // Update the existing snapshot entry. - snapshots.put(group, snapshot); + previousSnapshot = snapshots.put(group, snapshot); status = statuses.get(group); } finally { writeLock.unlock(); @@ -243,8 +392,9 @@ public void setSnapshot(T group, U snapshot) { return; } - // Responses should be in specific order and TYPE_URLS has a list of resources in the right order. - respondWithSpecificOrder(group, snapshot, status); + // Responses should be in specific order and typeUrls has a list of resources in the right + // order. + respondWithSpecificOrder(group, previousSnapshot, snapshot, status); } /** @@ -267,7 +417,9 @@ public StatusInfo statusInfo(T group) { } @VisibleForTesting - protected void respondWithSpecificOrder(T group, U snapshot, ConcurrentMap> statusMap) { + protected void respondWithSpecificOrder(T group, + U previousSnapshot, U snapshot, + ConcurrentMap> statusMap) { for (Resources.ResourceType resourceType : RESOURCE_TYPES_IN_ORDER) { CacheStatusInfo status = statusMap.get(resourceType); if (status == null) continue; // todo: why this happens? @@ -294,23 +446,77 @@ protected void respondWithSpecificOrder(T group, U snapshot, ConcurrentMap> previousResources = previousSnapshot == null + ? Collections.emptyMap() + : previousSnapshot.versionedResources(resourceType); + Map> snapshotResources = snapshot.versionedResources(resourceType); + + Map> snapshotChangedResources = snapshotResources.entrySet() + .stream() + .filter(entry -> { + VersionedResource versionedResource = previousResources.get(entry.getKey()); + return versionedResource == null || !versionedResource + .version().equals(entry.getValue().version()); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Set snapshotRemovedResources = previousResources.keySet() + .stream() + .filter(s -> !snapshotResources.containsKey(s)) + .collect(Collectors.toSet()); + + status.deltaWatchesRemoveIf((id, watch) -> { + String version = snapshot.version(watch.request().getResourceType(), Collections.emptyList()); + + if (!watch.version().equals(version)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("responding to open watch {}[{}] with new version {}", + id, + String.join(", ", watch.trackedResources().keySet()), + version); + } + + List removedResources = snapshotRemovedResources.stream() + .filter(s -> watch.trackedResources().get(s) != null) + .collect(Collectors.toList()); + + Map> changedResources = findChangedResources(watch, snapshotChangedResources); + + ResponseState responseState = respondDelta(watch, + changedResources, + removedResources, + version, + group); + // Discard the watch if it was responded or cancelled. + // A new watch will be created for future snapshots once envoy ACKs the response. + return ResponseState.RESPONDED.equals(responseState) || ResponseState.CANCELLED.equals(responseState); + } + + // Do not discard the watch. The request version is the same as the snapshot version, so we wait to respond. + return false; + }); } } - private Response createResponse(XdsRequest request, Map resources, String version) { + private Response createResponse(XdsRequest request, Map> resources, + String version) { Collection filtered = request.getResourceNamesList().isEmpty() - ? resources.values() - : request.getResourceNamesList().stream() - .map(resources::get) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + ? resources.values().stream() + .map(VersionedResource::resource) + .collect(Collectors.toList()) + : request.getResourceNamesList().stream() + .map(resources::get) + .filter(Objects::nonNull) + .map(VersionedResource::resource) + .collect(Collectors.toList()); return Response.create(request, filtered, version); } private boolean respond(Watch watch, U snapshot, T group) { - Map snapshotResources = snapshot.resources(watch.request().getResourceType()); - Map snapshotForMissingResources = Collections.emptyMap(); + Map> snapshotResources = snapshot.versionedResources(watch.request().getResourceType()); + Map> snapshotForMissingResources = Collections.emptyMap(); if (!watch.request().getResourceNamesList().isEmpty() && watch.ads()) { Collection missingNames = watch.request().getResourceNamesList().stream() @@ -340,7 +546,7 @@ private boolean respond(Watch watch, U snapshot, T group) { for (String missingName : missingNames) { snapshotForMissingResources.put( missingName, - ClusterLoadAssignment.newBuilder().setClusterName(missingName).build() + VersionedResource.create(ClusterLoadAssignment.newBuilder().setClusterName(missingName).build()) ); } } else { @@ -366,7 +572,7 @@ private boolean respond(Watch watch, U snapshot, T group) { version); Response response; if (!snapshotForMissingResources.isEmpty()) { - snapshotForMissingResources.putAll((Map) snapshotResources); + snapshotForMissingResources.putAll(snapshotResources); response = createResponse( watch.request(), snapshotForMissingResources, @@ -392,4 +598,89 @@ private boolean respond(Watch watch, U snapshot, T group) { return false; } + + private List findRemovedResources(DeltaWatch watch, Map> snapshotResources) { + // remove resources for which client has a tracked version but do not exist in snapshot + return watch.trackedResources().keySet() + .stream() + .filter(s -> !snapshotResources.containsKey(s)) + .collect(Collectors.toList()); + } + + private Map> findChangedResources(DeltaWatch watch, + Map> snapshotResources) { + return snapshotResources.entrySet() + .stream() + .filter(entry -> { + if (watch.pendingResources().contains(entry.getKey())) { + return true; + } + String resourceVersion = watch.trackedResources().get(entry.getKey()); + if (resourceVersion == null) { + // resource is not tracked, should respond it only if watch is wildcard + return watch.isWildcard(); + } + return !entry.getValue().version().equals(resourceVersion); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private ResponseState respondDeltaTracked(DeltaWatch watch, + Map> snapshotResources, + List removedResources, + String version, + T group) { + + Map> resources = snapshotResources.entrySet() + .stream() + .filter(entry -> { + if (watch.pendingResources().contains(entry.getKey())) { + return true; + } + String resourceVersion = watch.trackedResources().get(entry.getKey()); + if (resourceVersion == null) { + // resource is not tracked, should respond it only if watch is wildcard + return watch.isWildcard(); + } + return !entry.getValue().version().equals(resourceVersion); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return respondDelta(watch, resources, removedResources, version, group); + } + + private ResponseState respondDelta(DeltaWatch watch, + Map> resources, + List removedResources, + String version, + T group) { + if (resources.isEmpty() && removedResources.isEmpty()) { + return ResponseState.UNRESPONDED; + } + + DeltaResponse response = DeltaResponse.create( + watch.request(), + resources, + removedResources, + version); + + try { + watch.respond(response); + return ResponseState.RESPONDED; + } catch (WatchCancelledException e) { + LOGGER.error( + "failed to respond for {} from node {} with version {} because watch was already cancelled", + watch.request().getTypeUrl(), + group, + version); + } + + return ResponseState.CANCELLED; + } + + private enum ResponseState { + RESPONDED, + UNRESPONDED, + CANCELLED + } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt index 12ba07f11..fac68a39a 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt @@ -1,6 +1,9 @@ package pl.allegro.tech.servicemesh.envoycontrol.groups import io.envoyproxy.controlplane.cache.ConfigWatcher +import io.envoyproxy.controlplane.cache.DeltaResponse +import io.envoyproxy.controlplane.cache.DeltaWatch +import io.envoyproxy.controlplane.cache.DeltaXdsRequest import io.envoyproxy.controlplane.cache.Response import io.envoyproxy.controlplane.cache.Watch import io.envoyproxy.controlplane.cache.XdsRequest @@ -57,6 +60,26 @@ internal class GroupChangeWatcher( return watch } + override fun createDeltaWatch( + request: DeltaXdsRequest?, + requesterVersion: String?, + resourceVersions: MutableMap?, + pendingResources: MutableSet?, + isWildcard: Boolean, + responseConsumer: Consumer?, + hasClusterChanged: Boolean + ): DeltaWatch { + return cache.createDeltaWatch( + request, + requesterVersion, + resourceVersions, + pendingResources, + isWildcard, + responseConsumer, + hasClusterChanged + ) + } + private fun emitNewGroupsEvent(difference: List) { groupChangeEmitter?.next(difference) } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt index 3ef88140f..5b8c75efa 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt @@ -1,6 +1,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.groups import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks +import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest import io.envoyproxy.envoy.api.v2.DiscoveryResponse import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.ADS import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.XDS @@ -57,6 +58,17 @@ class NodeMetadataValidator( request?.node?.let { validateV3Metadata(it) } } + override fun onV2StreamDeltaRequest(streamId: Long, request: DeltaDiscoveryRequest?) { + request?.node?.let { validateV2Metadata(it) } + } + + override fun onV3StreamDeltaRequest( + streamId: Long, + request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest? + ) { + request?.node?.let { validateV3Metadata(it) } + } + override fun onV2StreamRequest(streamId: Long, request: DiscoveryRequestV2?) { request?.node?.let { validateV2Metadata(it) } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt index 683f2beb9..ea7f3cdc2 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt @@ -2,6 +2,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.server.callbacks import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks import io.envoyproxy.controlplane.server.exception.RequestException +import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest import io.micrometer.core.instrument.MeterRegistry import pl.allegro.tech.servicemesh.envoycontrol.logger import io.envoyproxy.envoy.api.v2.DiscoveryRequest as v2DiscoveryRequest @@ -48,6 +49,21 @@ class CompositeDiscoveryServerCallbacks( } } + override fun onV2StreamDeltaRequest(streamId: Long, request: DeltaDiscoveryRequest?) { + runCallbacks { + it.onV2StreamDeltaRequest(streamId, request) + } + } + + override fun onV3StreamDeltaRequest( + streamId: Long, + request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest? + ) { + runCallbacks { + it.onV3StreamDeltaRequest(streamId, request) + } + } + override fun onStreamResponse( streamId: Long, request: v2DiscoveryRequest?, diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt index 98b53044d..331d2f4f1 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt @@ -4,8 +4,10 @@ import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks import org.slf4j.LoggerFactory import io.envoyproxy.envoy.api.v2.DiscoveryRequest as v2DiscoveryRequest import io.envoyproxy.envoy.api.v2.DiscoveryResponse as v2DiscoveryResponse +import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest as v2DeltaDiscoveryRequest import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as v3DiscoveryRequest import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse as v3DiscoveryResponse +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest as v3DeltaDiscoveryRequest class LoggingDiscoveryServerCallbacks( private val logFullRequest: Boolean, @@ -21,6 +23,17 @@ class LoggingDiscoveryServerCallbacks( logger.debug("onV3StreamRequest streamId: {} request: {}", streamId, requestData(request)) } + override fun onV2StreamDeltaRequest(streamId: Long, request: v2DeltaDiscoveryRequest?) { + logger.debug("onV2StreamDeltaRequest streamId: {} request: {}", streamId, requestData(request)) + } + + override fun onV3StreamDeltaRequest( + streamId: Long, + request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest? + ) { + logger.debug("onV3StreamDeltaRequest streamId: {} request: {}", streamId, requestData(request)) + } + override fun onStreamCloseWithError(streamId: Long, typeUrl: String?, error: Throwable?) { logger.debug("onStreamCloseWithError streamId: {}, typeUrl: {}", streamId, typeUrl, error) } @@ -59,6 +72,24 @@ class LoggingDiscoveryServerCallbacks( ) } + private fun requestData(request: v3DeltaDiscoveryRequest?): String { + return if (logFullRequest) { + "$request" + } else { + "id: ${request?.node?.id}, cluster: ${request?.node?.cluster}, " + + "type: ${request?.typeUrl}, responseNonce: ${request?.responseNonce}" + } + } + + private fun requestData(request: v2DeltaDiscoveryRequest?): String { + return if (logFullRequest) { + "$request" + } else { + "id: ${request?.node?.id}, cluster: ${request?.node?.cluster}, " + + "type: ${request?.typeUrl}, responseNonce: ${request?.responseNonce}" + } + } + private fun requestData(request: v2DiscoveryRequest?): String { return if (logFullRequest) { "$request" diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt index afd51e30a..0bee84530 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt @@ -2,6 +2,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.server.callbacks import io.envoyproxy.controlplane.cache.Resources import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks +import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest import io.envoyproxy.envoy.api.v2.DiscoveryRequest as V2DiscoveryRequest import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as V3DiscoveryRequest import io.micrometer.core.instrument.MeterRegistry @@ -59,11 +60,24 @@ class MetricsDiscoveryServerCallbacks(private val meterRegistry: MeterRegistry) .increment() } + override fun onV3StreamDeltaRequest( + streamId: Long, + request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest + ) { + meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}.delta") + .increment() + } + override fun onV2StreamRequest(streamId: Long, request: V2DiscoveryRequest) { meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}") .increment() } + override fun onV2StreamDeltaRequest(streamId: Long, request: DeltaDiscoveryRequest) { + meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}.delta") + .increment() + } + override fun onStreamCloseWithError(streamId: Long, typeUrl: String, error: Throwable) { connections.decrementAndGet() connectionsByType(typeUrl).decrementAndGet() diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt index 8ebac1ccf..5b736fa78 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt @@ -24,12 +24,12 @@ internal fun globalSnapshot( v2Clusters: List, v2SecuredClusters: List ): GlobalSnapshot { - val clusters = SnapshotResources.create(clusters, "") - val securedClusters = SnapshotResources.create(securedClusters, "") - val v2Clusters = SnapshotResources.create(v2Clusters, "") - val v2SecuredClusters = SnapshotResources.create(v2SecuredClusters, "") + val clusters = SnapshotResources.create(clusters, "") + val securedClusters = SnapshotResources.create(securedClusters, "") + val v2Clusters = SnapshotResources.create(v2Clusters, "") + val v2SecuredClusters = SnapshotResources.create(v2SecuredClusters, "") val allServicesNames = getClustersForAllServicesGroups(clusters.resources(), properties) - val endpoints = SnapshotResources.create(endpoints, "") + val endpoints = SnapshotResources.create(endpoints, "") return GlobalSnapshot( clusters = clusters, securedClusters = securedClusters, diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java index 2a9885649..9f2ebc49e 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java @@ -6,12 +6,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.NodeGroup; -import io.envoyproxy.controlplane.cache.Resources; -import io.envoyproxy.controlplane.cache.Response; -import io.envoyproxy.controlplane.cache.StatusInfo; -import io.envoyproxy.controlplane.cache.Watch; -import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.cache.*; import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; @@ -523,7 +518,8 @@ protected static void assertThatWatchReceivesSnapshot(WatchAndTracker watchAndTr assertThat(response).isNotNull(); assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); assertThat(response.resources().toArray(new Message[0])) - .containsExactlyElementsOf(snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values()); + .containsExactlyElementsOf(snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values() + .stream().map(VersionedResource::resource).collect(Collectors.toList())); } protected static class ResponseTracker implements Consumer { diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java index 9463db3b6..bd7281a49 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java @@ -2,10 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.Resources; -import io.envoyproxy.controlplane.cache.Response; -import io.envoyproxy.controlplane.cache.Watch; -import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.cache.*; import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; @@ -17,6 +14,7 @@ import org.junit.jupiter.api.Test; import java.util.Collections; +import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -75,7 +73,7 @@ private static void assertThatWatchReceivesSnapshotWithMissingResources(WatchAnd assertThat(response).isNotNull(); assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); Message[] responseValues = response.resources().toArray(new Message[0]); - Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().toArray(new Message[0]); + Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().stream().map(VersionedResource::resource).collect(Collectors.toList()).toArray(new Message[0]); assertThat(responseValues.length).isEqualTo(2); assertThat(responseValues.length).isEqualTo(snapshotValues.length); diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java index 6f85b7128..ffb5cf02a 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java @@ -515,7 +515,8 @@ protected static void assertThatWatchReceivesSnapshot(WatchAndTracker watchAndTr assertThat(response).isNotNull(); assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); assertThat(response.resources().toArray(new Message[0])) - .containsExactlyElementsOf(snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values()); + .containsExactlyElementsOf(snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values() + .stream().map(VersionedResource::resource).collect(Collectors.toList())); } protected static class ResponseTracker implements Consumer { diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java index 2b82ec6fe..ac16b947a 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java @@ -2,10 +2,7 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.Resources; -import io.envoyproxy.controlplane.cache.Response; -import io.envoyproxy.controlplane.cache.Watch; -import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.cache.*; import io.envoyproxy.controlplane.cache.v3.Snapshot; import io.envoyproxy.envoy.config.core.v3.Node; import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; @@ -17,6 +14,7 @@ import org.junit.jupiter.api.Test; import java.util.Collections; +import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -75,7 +73,7 @@ private static void assertThatWatchReceivesSnapshotWithMissingResources(WatchAnd assertThat(response).isNotNull(); assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); Message[] responseValues = response.resources().toArray(new Message[0]); - Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().toArray(new Message[0]); + Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().stream().map(VersionedResource::resource).collect(Collectors.toList()).toArray(new Message[0]); assertThat(responseValues.length).isEqualTo(2); assertThat(responseValues.length).isEqualTo(snapshotValues.length); diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 3aaec10d2..fddf0c7dd 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -2,6 +2,7 @@ package pl.allegro.tech.servicemesh.envoycontrol import com.google.protobuf.util.Durations import io.envoyproxy.controlplane.cache.SnapshotResources +import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource import io.envoyproxy.envoy.config.core.v3.ConfigSource @@ -168,11 +169,11 @@ class EnvoySnapshotFactoryTest { private fun createGlobalSnapshot(cluster: Cluster?): GlobalSnapshot { return GlobalSnapshot( - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), emptySet(), - SnapshotResources.create(emptyList(), "v1"), emptyMap(), + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), emptySet(), + SnapshotResources.create(emptyList(), "v1"), emptyMap(), SnapshotResources.create(listOf(cluster), "v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3") + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3") ) } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt index 072dbb0ab..12cf9a87e 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt @@ -1,6 +1,9 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot import com.google.protobuf.util.Durations +import io.envoyproxy.controlplane.cache.DeltaResponse +import io.envoyproxy.controlplane.cache.DeltaWatch +import io.envoyproxy.controlplane.cache.DeltaXdsRequest import io.envoyproxy.controlplane.cache.Resources import io.envoyproxy.controlplane.cache.Response import io.envoyproxy.controlplane.cache.SnapshotCache @@ -629,6 +632,18 @@ class SnapshotUpdaterTest { .isTrue() } } + + override fun createDeltaWatch( + request: DeltaXdsRequest?, + requesterVersion: String?, + resourceVersions: MutableMap?, + pendingResources: MutableSet?, + isWildcard: Boolean, + responseConsumer: Consumer?, + hasClusterChanged: Boolean + ): DeltaWatch { + throw UnsupportedOperationException("not used in testing") + } } private fun hasSnapshot(cache: SnapshotCache, group: Group): Snapshot { diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt index d25532705..b9355b203 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt @@ -3,6 +3,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.fil import com.google.protobuf.Any import com.google.protobuf.util.JsonFormat import io.envoyproxy.controlplane.cache.SnapshotResources +import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.core.v3.Address import io.envoyproxy.envoy.config.core.v3.SocketAddress import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment @@ -92,13 +93,13 @@ internal class RBACFilterFactoryTest { ) val snapshot = GlobalSnapshot( - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), ""), setOf(), - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), ""), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), "") + SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "") ) val clusterLoadAssignment = ClusterLoadAssignment.newBuilder() @@ -116,13 +117,13 @@ internal class RBACFilterFactoryTest { ).build() val snapshotForSourceIpAuth = GlobalSnapshot( - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), ""), setOf(), SnapshotResources.create(listOf(clusterLoadAssignment), ""), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), "") + SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "") ) @Test From 6927a2c824566c4162b3a30d4ec761b76730e71b Mon Sep 17 00:00:00 2001 From: slonka Date: Thu, 8 Apr 2021 09:54:37 +0200 Subject: [PATCH 02/46] Switch JCP version to hash byte array --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index de202b4a2..c6a3652f1 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ allprojects { project.ext.versions = [ kotlin : '1.3.72', - java_controlplane : '0.1.29-delta-xds-slonka-SNAPSHOT', + java_controlplane : '0.1.29-delta-xds-slonka-hash-bytes-SNAPSHOT', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', From a40d466dede15e05342953cc63c1dba6d6ebf95b Mon Sep 17 00:00:00 2001 From: slonka Date: Thu, 8 Apr 2021 11:13:27 +0200 Subject: [PATCH 03/46] Convert snapshots to strings to do the comparision, otherwise some type checking fails --- .../v2/SimpleCacheWithMissingEndpointsTest.java | 6 +++--- .../v3/SimpleCacheWithMissingEndpointsTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java index bd7281a49..875d369ee 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java @@ -73,11 +73,11 @@ private static void assertThatWatchReceivesSnapshotWithMissingResources(WatchAnd assertThat(response).isNotNull(); assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); Message[] responseValues = response.resources().toArray(new Message[0]); - Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().stream().map(VersionedResource::resource).collect(Collectors.toList()).toArray(new Message[0]); + Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().stream().map(VersionedResource::resource).toArray(Message[]::new); assertThat(responseValues.length).isEqualTo(2); assertThat(responseValues.length).isEqualTo(snapshotValues.length); - assertThat(responseValues[0]).isEqualToComparingFieldByFieldRecursively(snapshotValues[0]); - assertThat(responseValues[1]).isEqualToComparingFieldByFieldRecursively(snapshotValues[1]); + assertThat(responseValues[0].toString()).isEqualTo(snapshotValues[0].toString()); + assertThat(responseValues[1].toString()).isEqualTo(snapshotValues[1].toString()); } } diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java index ac16b947a..e78ad708f 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java @@ -73,11 +73,11 @@ private static void assertThatWatchReceivesSnapshotWithMissingResources(WatchAnd assertThat(response).isNotNull(); assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); Message[] responseValues = response.resources().toArray(new Message[0]); - Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().stream().map(VersionedResource::resource).collect(Collectors.toList()).toArray(new Message[0]); + Message[] snapshotValues = snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values().stream().map(VersionedResource::resource).toArray(Message[]::new); assertThat(responseValues.length).isEqualTo(2); assertThat(responseValues.length).isEqualTo(snapshotValues.length); - assertThat(responseValues[0]).isEqualToComparingFieldByFieldRecursively(snapshotValues[0]); - assertThat(responseValues[1]).isEqualToComparingFieldByFieldRecursively(snapshotValues[1]); + assertThat(responseValues[0].toString()).isEqualTo(snapshotValues[0].toString()); + assertThat(responseValues[1].toString()).isEqualTo(snapshotValues[1].toString()); } } From 36877972b8948a069355cb30ff18d8f87a604906 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 12 Apr 2021 10:18:10 +0200 Subject: [PATCH 04/46] tmp --- .../envoycontrol/groups/GroupChangeWatcher.kt | 10 +++- .../server/CachedProtoResourcesSerializer.kt | 50 +++++++++++++------ .../src/main/resources/application-local.yaml | 4 +- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt index fac68a39a..e95c897b5 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/GroupChangeWatcher.kt @@ -69,7 +69,9 @@ internal class GroupChangeWatcher( responseConsumer: Consumer?, hasClusterChanged: Boolean ): DeltaWatch { - return cache.createDeltaWatch( + val oldGroups = cache.groups() + + val watch = cache.createDeltaWatch( request, requesterVersion, resourceVersions, @@ -78,6 +80,12 @@ internal class GroupChangeWatcher( responseConsumer, hasClusterChanged ) + val groups = cache.groups() + metrics.setCacheGroupsCount(groups.size) + if (oldGroups != groups) { + emitNewGroupsEvent(groups - oldGroups) + } + return watch } private fun emitNewGroupsEvent(difference: List) { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt index d84e4f652..d17a4b4a3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt @@ -5,15 +5,14 @@ import com.google.common.cache.CacheBuilder import com.google.protobuf.Any import com.google.protobuf.Message import io.envoyproxy.controlplane.cache.Resources -import io.micrometer.core.instrument.MeterRegistry -import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics -import pl.allegro.tech.servicemesh.envoycontrol.utils.noopTimer -import java.util.function.Supplier - import io.envoyproxy.controlplane.cache.Resources.ApiVersion.V2 import io.envoyproxy.controlplane.cache.Resources.ApiVersion.V3 import io.envoyproxy.controlplane.server.serializer.DefaultProtoResourcesSerializer +import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.Timer +import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics +import pl.allegro.tech.servicemesh.envoycontrol.utils.noopTimer +import java.util.function.Supplier internal class CachedProtoResourcesSerializer( private val meterRegistry: MeterRegistry, @@ -33,7 +32,12 @@ internal class CachedProtoResourcesSerializer( private val v2Timer = createTimer(reportMetrics, meterRegistry, "protobuf-cache-v2.serialize.time") private val v3Timer = createTimer(reportMetrics, meterRegistry, "protobuf-cache-v3.serialize.time") - private fun createCache(cacheName: String): Cache, MutableCollection> { + private val deltaCacheV2: Cache = createCache("protobuf-delta-cache-v2") + private val deltaCacheV3: Cache = createCache("protobuf-delta-cache-v3") + private val deltaV2Timer = createTimer(reportMetrics, meterRegistry, "protobuf-delta-cache-v2.serialize.time") + private val deltaV3Timer = createTimer(reportMetrics, meterRegistry, "protobuf-delta-cache-v3.serialize.time") + + private fun createCache(cacheName: String): Cache { return if (reportMetrics) { GuavaCacheMetrics .monitor( @@ -41,13 +45,13 @@ internal class CachedProtoResourcesSerializer( CacheBuilder.newBuilder() .recordStats() .weakValues() - .build, MutableCollection>(), + .build(), cacheName ) } else { CacheBuilder.newBuilder() .weakValues() - .build, MutableCollection>() + .build() } } @@ -55,11 +59,12 @@ internal class CachedProtoResourcesSerializer( resources: MutableCollection, apiVersion: Resources.ApiVersion ): MutableCollection { - return if (apiVersion == V2) { - v2Timer.record(Supplier { getResources(resources, apiVersion) }) - } else { - v3Timer.record(Supplier { getResources(resources, apiVersion) }) + val timer = when (apiVersion) { + V2 -> v2Timer + V3 -> v3Timer } + + return timer.record(Supplier { getResources(resources, apiVersion) }) } private fun getResources( @@ -78,8 +83,21 @@ internal class CachedProtoResourcesSerializer( } } - @Suppress("NotImplementedDeclaration") - override fun serialize(resource: Message?, apiVersion: Resources.ApiVersion?): Any { - throw NotImplementedError("Serializing single messages is not supported") - } + // override fun serialize(resource: Message, apiVersion: Resources.ApiVersion): Any { + // val cache = when (apiVersion) { + // V2 -> deltaCacheV2 + // V3 -> deltaCacheV3 + // } + // + // val timer = when (apiVersion) { + // V2 -> deltaV2Timer + // V3 -> deltaV3Timer + // } + // + // return timer.record(Supplier { + // cache.get(resource) { + // super.maybeRewriteTypeUrl(Any.pack(resource), apiVersion) + // } + // }) + // } } diff --git a/envoy-control-runner/src/main/resources/application-local.yaml b/envoy-control-runner/src/main/resources/application-local.yaml index f89a8ada0..f23249ada 100644 --- a/envoy-control-runner/src/main/resources/application-local.yaml +++ b/envoy-control-runner/src/main/resources/application-local.yaml @@ -2,8 +2,8 @@ envoy-control: source: consul: host: localhost - port: 18500 + port: 18509 chaos: username: "user" - password: "pass" \ No newline at end of file + password: "pass" From abb187f2d5a41e1135210d44050a067ce3754876 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 12 Apr 2021 12:54:48 +0200 Subject: [PATCH 05/46] Since using delta mode, cache single messages not whole collections --- .../server/CachedProtoResourcesSerializer.kt | 75 +++++++------------ 1 file changed, 25 insertions(+), 50 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt index d17a4b4a3..251cd3138 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt @@ -19,24 +19,11 @@ internal class CachedProtoResourcesSerializer( private val reportMetrics: Boolean ) : DefaultProtoResourcesSerializer() { - private fun createTimer(reportMetrics: Boolean, meterRegistry: MeterRegistry, timerName: String): Timer { - return if (reportMetrics) { - meterRegistry.timer(timerName) - } else { - noopTimer - } - } - - private val cacheV2: Cache, MutableCollection> = createCache("protobuf-cache-v2") - private val cacheV3: Cache, MutableCollection> = createCache("protobuf-cache-v3") + private val cacheV2: Cache = createCache("protobuf-cache-v2") + private val cacheV3: Cache = createCache("protobuf-cache-v3") private val v2Timer = createTimer(reportMetrics, meterRegistry, "protobuf-cache-v2.serialize.time") private val v3Timer = createTimer(reportMetrics, meterRegistry, "protobuf-cache-v3.serialize.time") - private val deltaCacheV2: Cache = createCache("protobuf-delta-cache-v2") - private val deltaCacheV3: Cache = createCache("protobuf-delta-cache-v3") - private val deltaV2Timer = createTimer(reportMetrics, meterRegistry, "protobuf-delta-cache-v2.serialize.time") - private val deltaV3Timer = createTimer(reportMetrics, meterRegistry, "protobuf-delta-cache-v3.serialize.time") - private fun createCache(cacheName: String): Cache { return if (reportMetrics) { GuavaCacheMetrics @@ -55,49 +42,37 @@ internal class CachedProtoResourcesSerializer( } } - override fun serialize( - resources: MutableCollection, - apiVersion: Resources.ApiVersion - ): MutableCollection { - val timer = when (apiVersion) { - V2 -> v2Timer - V3 -> v3Timer + private fun createTimer(reportMetrics: Boolean, meterRegistry: MeterRegistry, timerName: String): Timer { + return if (reportMetrics) { + meterRegistry.timer(timerName) + } else { + noopTimer } - - return timer.record(Supplier { getResources(resources, apiVersion) }) } - private fun getResources( - resources: MutableCollection, - apiVersion: Resources.ApiVersion - ): MutableCollection { + + /** + * {@inheritDoc} + */ + override fun serialize(resource: Message, apiVersion: Resources.ApiVersion): Any { val cache = when (apiVersion) { V2 -> cacheV2 V3 -> cacheV3 } - return cache.get(resources) { - resources.asSequence() - .map { super.maybeRewriteTypeUrl(Any.pack(it), apiVersion) } - .toMutableList() + val timer = when (apiVersion) { + V2 -> v2Timer + V3 -> v3Timer } - } - // override fun serialize(resource: Message, apiVersion: Resources.ApiVersion): Any { - // val cache = when (apiVersion) { - // V2 -> deltaCacheV2 - // V3 -> deltaCacheV3 - // } - // - // val timer = when (apiVersion) { - // V2 -> deltaV2Timer - // V3 -> deltaV3Timer - // } - // - // return timer.record(Supplier { - // cache.get(resource) { - // super.maybeRewriteTypeUrl(Any.pack(resource), apiVersion) - // } - // }) - // } + return timer.record(Supplier { + cache.get(resource) { + super.maybeRewriteTypeUrl( + Any.pack( + resource + ), apiVersion + ) + } + }) + } } From af3089d6363647dae365efbcce96c8a37bfcf69c Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 12 Apr 2021 13:51:01 +0200 Subject: [PATCH 06/46] Make ktlint pass --- .../envoycontrol/server/CachedProtoResourcesSerializer.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt index 251cd3138..aa702a9bf 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt @@ -50,7 +50,6 @@ internal class CachedProtoResourcesSerializer( } } - /** * {@inheritDoc} */ From a716b2e3863d1121dfcf0f11a2d061f9975ce13d Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 19 Apr 2021 09:16:53 +0200 Subject: [PATCH 07/46] Do not call resources on every loop invocation. Enable cache metrics --- .../tech/servicemesh/envoycontrol/server/ServerProperties.kt | 2 +- .../servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt | 3 ++- .../snapshot/resource/listeners/filters/RBACFilterFactory.kt | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt index f669f495b..29d88a0a3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt @@ -20,7 +20,7 @@ class ServerProperties { parallelPoolSize = 1 } var snapshotCleanup = SnapshotCleanupProperties() - var reportProtobufCacheMetrics = false + var reportProtobufCacheMetrics = true var logFullRequest = false var logFullResponse = false } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 674c04b22..9ec0f1206 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -217,8 +217,9 @@ class EnvoySnapshotFactory( globalSnapshot: GlobalSnapshot, egressRouteSpecifications: Collection ): List { + val resources = globalSnapshot.endpoints.resources() return egressRouteSpecifications - .mapNotNull { globalSnapshot.endpoints.resources().get(it.clusterName) } + .mapNotNull { resources.get(it.clusterName) } } private fun newSnapshotForGroup( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt index 9c7bb2500..b10fb361c 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt @@ -289,7 +289,8 @@ class RBACFilterFactory( selectorMatching: SelectorMatching?, snapshot: GlobalSnapshot ): List { - val clusterLoadAssignment = snapshot.endpoints.resources()[client.name] + val resources = snapshot.endpoints.resources() + val clusterLoadAssignment = resources[client.name] val sourceIpPrincipal = mapEndpointsToExactPrincipals(clusterLoadAssignment) return if (sourceIpPrincipal == null) { From 587add5df297de6488b924f88d88acc56a3488cf Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 19 Apr 2021 12:35:14 +0200 Subject: [PATCH 08/46] Hold resuling resources in the global snapshot instead of calling it --- .../snapshot/EnvoySnapshotFactory.kt | 2 +- .../envoycontrol/snapshot/GlobalSnapshot.kt | 22 +++++++++---------- .../resource/clusters/EnvoyClustersFactory.kt | 8 +++---- .../listeners/filters/RBACFilterFactory.kt | 2 +- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 12 +++++----- .../snapshot/SnapshotUpdaterTest.kt | 18 +++++++-------- .../filters/RBACFilterFactoryTest.kt | 20 ++++++++--------- .../snapshot/debug/SnapshotDebugInfo.kt | 4 ++-- 8 files changed, 44 insertions(+), 44 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 9ec0f1206..ccfeea890 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -217,7 +217,7 @@ class EnvoySnapshotFactory( globalSnapshot: GlobalSnapshot, egressRouteSpecifications: Collection ): List { - val resources = globalSnapshot.endpoints.resources() + val resources = globalSnapshot.endpoints return egressRouteSpecifications .mapNotNull { resources.get(it.clusterName) } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt index 5b736fa78..133df0d1c 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt @@ -5,13 +5,13 @@ import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment data class GlobalSnapshot( - val clusters: SnapshotResources, + val clusters: Map, val allServicesNames: Set, - val endpoints: SnapshotResources, + val endpoints: Map, val clusterConfigurations: Map, - val securedClusters: SnapshotResources, - val v2Clusters: SnapshotResources, - val v2SecuredClusters: SnapshotResources + val securedClusters: Map, + val v2Clusters: Map, + val v2SecuredClusters: Map ) @Suppress("LongParameterList") @@ -24,12 +24,12 @@ internal fun globalSnapshot( v2Clusters: List, v2SecuredClusters: List ): GlobalSnapshot { - val clusters = SnapshotResources.create(clusters, "") - val securedClusters = SnapshotResources.create(securedClusters, "") - val v2Clusters = SnapshotResources.create(v2Clusters, "") - val v2SecuredClusters = SnapshotResources.create(v2SecuredClusters, "") - val allServicesNames = getClustersForAllServicesGroups(clusters.resources(), properties) - val endpoints = SnapshotResources.create(endpoints, "") + val clusters = SnapshotResources.create(clusters, "").resources() + val securedClusters = SnapshotResources.create(securedClusters, "").resources() + val v2Clusters = SnapshotResources.create(v2Clusters, "").resources() + val v2SecuredClusters = SnapshotResources.create(v2SecuredClusters, "").resources() + val allServicesNames = getClustersForAllServicesGroups(clusters, properties) + val endpoints = SnapshotResources.create(endpoints, "").resources() return GlobalSnapshot( clusters = clusters, securedClusters = securedClusters, diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index f8894bba3..a89c53393 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -108,15 +108,15 @@ class EnvoyClustersFactory( private fun getEdsClustersForGroup(group: Group, globalSnapshot: GlobalSnapshot): List { val clusters: Map = if (enableTlsForGroup(group)) { if (group.version == ResourceVersion.V3) { - globalSnapshot.securedClusters.resources() + globalSnapshot.securedClusters } else { - globalSnapshot.v2SecuredClusters.resources() + globalSnapshot.v2SecuredClusters } } else { if (group.version == ResourceVersion.V3) { - globalSnapshot.clusters.resources() + globalSnapshot.clusters } else { - globalSnapshot.v2Clusters.resources() + globalSnapshot.v2Clusters } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt index b10fb361c..41a139172 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactory.kt @@ -289,7 +289,7 @@ class RBACFilterFactory( selectorMatching: SelectorMatching?, snapshot: GlobalSnapshot ): List { - val resources = snapshot.endpoints.resources() + val resources = snapshot.endpoints val clusterLoadAssignment = resources[client.name] val sourceIpPrincipal = mapEndpointsToExactPrincipals(clusterLoadAssignment) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index fddf0c7dd..90896c681 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -2,12 +2,12 @@ package pl.allegro.tech.servicemesh.envoycontrol import com.google.protobuf.util.Durations import io.envoyproxy.controlplane.cache.SnapshotResources -import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource import io.envoyproxy.envoy.config.core.v3.ConfigSource import io.envoyproxy.envoy.config.core.v3.Metadata import io.envoyproxy.envoy.config.listener.v3.Listener +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.simple.SimpleMeterRegistry import org.assertj.core.api.Assertions.assertThat @@ -169,11 +169,11 @@ class EnvoySnapshotFactoryTest { private fun createGlobalSnapshot(cluster: Cluster?): GlobalSnapshot { return GlobalSnapshot( - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), emptySet(), - SnapshotResources.create(emptyList(), "v1"), emptyMap(), - SnapshotResources.create(listOf(cluster), "v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3") + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources(), emptySet(), + SnapshotResources.create(emptyList(), "v1").resources(), emptyMap(), + SnapshotResources.create(listOf(cluster), "v3").resources(), + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources(), + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources() ) } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt index 12cf9a87e..7def0d18e 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdaterTest.kt @@ -696,7 +696,7 @@ class SnapshotUpdaterTest { ) private fun GlobalSnapshot.hasHttp2Cluster(clusterName: String): GlobalSnapshot { - val cluster = this.clusters.resources()[clusterName] + val cluster = this.clusters[clusterName] assertThat(cluster).isNotNull assertThat(cluster!!.hasHttp2ProtocolOptions()).isTrue() return this @@ -706,7 +706,7 @@ class SnapshotUpdaterTest { clusterName: String, zones: Set ): GlobalSnapshot { - val endpoints = this.endpoints.resources()[clusterName] + val endpoints = this.endpoints[clusterName] assertThat(endpoints).isNotNull assertThat(endpoints!!.endpointsList.map { it.locality.zone }.toSet()).containsAll(zones) assertThat(endpoints.endpointsList.flatMap { it.lbEndpointsList }).isEmpty() @@ -714,7 +714,7 @@ class SnapshotUpdaterTest { } private fun GlobalSnapshot.hasAnEndpoint(clusterName: String, ip: String, port: Int): GlobalSnapshot { - val endpoints = this.endpoints.resources()[clusterName] + val endpoints = this.endpoints[clusterName] assertThat(endpoints).isNotNull assertThat(endpoints!!.endpointsList.flatMap { it.lbEndpointsList }) .anyMatch { it.endpoint.address.socketAddress.let { it.address == ip && it.portValue == port } } @@ -722,20 +722,20 @@ class SnapshotUpdaterTest { } private fun GlobalSnapshot.hasTheSameClusters(other: GlobalSnapshot): GlobalSnapshot { - val clusters = this.clusters.resources() - assertThat(clusters).isEqualTo(other.clusters.resources()) + val clusters = this.clusters + assertThat(clusters).isEqualTo(other.clusters) return this } private fun GlobalSnapshot.hasTheSameEndpoints(other: GlobalSnapshot): GlobalSnapshot { - val endpoints = this.endpoints.resources() - assertThat(endpoints).isEqualTo(other.endpoints.resources()) + val endpoints = this.endpoints + assertThat(endpoints).isEqualTo(other.endpoints) return this } private fun GlobalSnapshot.hasTheSameSecuredClusters(other: GlobalSnapshot): GlobalSnapshot { - val securedClusters = this.securedClusters.resources() - assertThat(securedClusters).isEqualTo(other.securedClusters.resources()) + val securedClusters = this.securedClusters + assertThat(securedClusters).isEqualTo(other.securedClusters) return this } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt index b9355b203..1a76bfcbf 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/RBACFilterFactoryTest.kt @@ -93,13 +93,13 @@ internal class RBACFilterFactoryTest { ) val snapshot = GlobalSnapshot( - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "").resources(), setOf(), - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "").resources(), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), "") + SnapshotResources.create(listOf(), "").resources(), + SnapshotResources.create(listOf(), "").resources(), + SnapshotResources.create(listOf(), "").resources() ) val clusterLoadAssignment = ClusterLoadAssignment.newBuilder() @@ -117,13 +117,13 @@ internal class RBACFilterFactoryTest { ).build() val snapshotForSourceIpAuth = GlobalSnapshot( - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "").resources(), setOf(), - SnapshotResources.create(listOf(clusterLoadAssignment), ""), + SnapshotResources.create(listOf(clusterLoadAssignment), "").resources(), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), "") + SnapshotResources.create(listOf(), "").resources(), + SnapshotResources.create(listOf(), "").resources(), + SnapshotResources.create(listOf(), "").resources() ) @Test diff --git a/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugInfo.kt b/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugInfo.kt index c554d8c32..0ca659b94 100644 --- a/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugInfo.kt +++ b/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugInfo.kt @@ -55,8 +55,8 @@ data class SnapshotDebugInfo( constructor(globalSnapshot: GlobalSnapshot) : this( snapshot = Snapshot( - clusters = globalSnapshot.clusters.resources(), - endpoints = globalSnapshot.endpoints.resources() + clusters = globalSnapshot.clusters, + endpoints = globalSnapshot.endpoints ), versions = Versions() ) From 30ef1db422fe7a51eda5071940a8f8c8d9d80867 Mon Sep 17 00:00:00 2001 From: slonka Date: Tue, 18 May 2021 08:43:29 +0200 Subject: [PATCH 09/46] Revert application-local port --- envoy-control-runner/src/main/resources/application-local.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envoy-control-runner/src/main/resources/application-local.yaml b/envoy-control-runner/src/main/resources/application-local.yaml index f23249ada..abfb4498b 100644 --- a/envoy-control-runner/src/main/resources/application-local.yaml +++ b/envoy-control-runner/src/main/resources/application-local.yaml @@ -2,7 +2,7 @@ envoy-control: source: consul: host: localhost - port: 18509 + port: 18500 chaos: username: "user" From dbac67236895fd8d872276a5a117b0d821eca567 Mon Sep 17 00:00:00 2001 From: slonka Date: Tue, 18 May 2021 10:26:22 +0200 Subject: [PATCH 10/46] Revert cache metrics --- .../envoycontrol/server/CachedProtoResourcesSerializer.kt | 3 --- .../tech/servicemesh/envoycontrol/server/ServerProperties.kt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt index aa702a9bf..eeb0a434d 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt @@ -50,9 +50,6 @@ internal class CachedProtoResourcesSerializer( } } - /** - * {@inheritDoc} - */ override fun serialize(resource: Message, apiVersion: Resources.ApiVersion): Any { val cache = when (apiVersion) { V2 -> cacheV2 diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt index 29d88a0a3..f669f495b 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt @@ -20,7 +20,7 @@ class ServerProperties { parallelPoolSize = 1 } var snapshotCleanup = SnapshotCleanupProperties() - var reportProtobufCacheMetrics = true + var reportProtobufCacheMetrics = false var logFullRequest = false var logFullResponse = false } From 4c47b8ac51dad84bce50011f992a79d0f3bf490b Mon Sep 17 00:00:00 2001 From: slonka Date: Tue, 18 May 2021 11:32:27 +0200 Subject: [PATCH 11/46] Add delta test for metered connections and requests --- .../envoycontrol/EnvoyControlSmokeTest.kt | 29 +++++ .../MetricsDiscoveryServerCallbacksTest.kt | 114 +++++++++++++++-- .../config/EnvoyControlTestConfiguration.kt | 1 + .../resources/envoy/config_delta_ads.yaml | 118 ++++++++++++++++++ 4 files changed, 253 insertions(+), 9 deletions(-) create mode 100644 envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt index 1a8c26797..39ce4d796 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt @@ -100,6 +100,35 @@ class AdsEnvoyControlSmokeTest : EnvoyControlSmokeTest { override fun envoy() = envoy } +class DeltaAdsEnvoyControlSmokeTest : EnvoyControlSmokeTest { + companion object { + + @JvmField + @RegisterExtension + val consul = ConsulExtension() + + @JvmField + @RegisterExtension + val envoyControl = EnvoyControlExtension(consul) + + @JvmField + @RegisterExtension + val service = EchoServiceExtension() + + @JvmField + @RegisterExtension + val envoy = EnvoyExtension(envoyControl, service, config = Ads) + } + + override fun consul() = consul + + override fun envoyControl() = envoyControl + + override fun service() = service + + override fun envoy() = envoy +} + class XdsEnvoyControlSmokeTest : EnvoyControlSmokeTest { companion object { diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/MetricsDiscoveryServerCallbacksTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/MetricsDiscoveryServerCallbacksTest.kt index 6f7b70d88..beffcee16 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/MetricsDiscoveryServerCallbacksTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/MetricsDiscoveryServerCallbacksTest.kt @@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted import pl.allegro.tech.servicemesh.envoycontrol.config.Ads +import pl.allegro.tech.servicemesh.envoycontrol.config.DeltaAds import pl.allegro.tech.servicemesh.envoycontrol.config.Xds import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension @@ -57,6 +58,23 @@ class XdsMetricsDiscoveryServerCallbacksTest : MetricsDiscoveryServerCallbacksTe ADS to 0, UNKNOWN to 0 ) + + override fun expectedGrpcRequestsCounterValues() = mapOf( + CDS.name.toLowerCase() to isGreaterThanZero(), + EDS.name.toLowerCase() to isGreaterThanZero(), + LDS.name.toLowerCase() to isGreaterThanZero(), + RDS.name.toLowerCase() to isGreaterThanZero(), + SDS.name.toLowerCase() to isNull(), + ADS.name.toLowerCase() to isNull(), + UNKNOWN.name.toLowerCase() to isNull(), + "${CDS.name.toLowerCase()}.delta" to isNull(), + "${EDS.name.toLowerCase()}.delta" to isNull(), + "${LDS.name.toLowerCase()}.delta" to isNull(), + "${RDS.name.toLowerCase()}.delta" to isNull(), + "${SDS.name.toLowerCase()}.delta" to isNull(), + "${ADS.name.toLowerCase()}.delta" to isNull(), + "${UNKNOWN.name.toLowerCase()}.delta" to isNull() + ) } class AdsMetricsDiscoveryServerCallbackTest : MetricsDiscoveryServerCallbacksTest { @@ -96,6 +114,79 @@ class AdsMetricsDiscoveryServerCallbackTest : MetricsDiscoveryServerCallbacksTes ADS to 1, // all info is exchanged on one stream UNKNOWN to 0 ) + + override fun expectedGrpcRequestsCounterValues() = mapOf( + CDS.name.toLowerCase() to isGreaterThanZero(), + EDS.name.toLowerCase() to isGreaterThanZero(), + LDS.name.toLowerCase() to isGreaterThanZero(), + RDS.name.toLowerCase() to isGreaterThanZero(), + SDS.name.toLowerCase() to isNull(), + ADS.name.toLowerCase() to isNull(), + UNKNOWN.name.toLowerCase() to isNull(), + "${CDS.name.toLowerCase()}.delta" to isNull(), + "${EDS.name.toLowerCase()}.delta" to isNull(), + "${LDS.name.toLowerCase()}.delta" to isNull(), + "${RDS.name.toLowerCase()}.delta" to isNull(), + "${SDS.name.toLowerCase()}.delta" to isNull(), + "${ADS.name.toLowerCase()}.delta" to isNull(), + "${UNKNOWN.name.toLowerCase()}.delta" to isNull() + ) +} + +class DeltaAdsMetricsDiscoveryServerCallbackTest : MetricsDiscoveryServerCallbacksTest { + companion object { + + @JvmField + @RegisterExtension + val consul = ConsulExtension() + + @JvmField + @RegisterExtension + val envoyControl = EnvoyControlExtension(consul) + + @JvmField + @RegisterExtension + val service = EchoServiceExtension() + + @JvmField + @RegisterExtension + val envoy = EnvoyExtension(envoyControl, service, config = DeltaAds) + } + + override fun consul() = consul + + override fun envoyControl() = envoyControl + + override fun service() = service + + override fun envoy() = envoy + + override fun expectedGrpcConnectionsGaugeValues() = mapOf( + CDS to 0, + EDS to 0, + LDS to 0, + RDS to 0, + SDS to 0, + ADS to 1, // all info is exchanged on one stream + UNKNOWN to 0 + ) + + override fun expectedGrpcRequestsCounterValues() = mapOf( + CDS.name.toLowerCase() to isNull(), + EDS.name.toLowerCase() to isNull(), + LDS.name.toLowerCase() to isNull(), + RDS.name.toLowerCase() to isNull(), + SDS.name.toLowerCase() to isNull(), + ADS.name.toLowerCase() to isNull(), + UNKNOWN.name.toLowerCase() to isNull(), + "${CDS.name.toLowerCase()}.delta" to isGreaterThanZero(), + "${EDS.name.toLowerCase()}.delta" to isGreaterThanZero(), + "${LDS.name.toLowerCase()}.delta" to isGreaterThanZero(), + "${RDS.name.toLowerCase()}.delta" to isGreaterThanZero(), + "${SDS.name.toLowerCase()}.delta" to isNull(), + "${ADS.name.toLowerCase()}.delta" to isNull(), + "${UNKNOWN.name.toLowerCase()}.delta" to isNull() + ) } interface MetricsDiscoveryServerCallbacksTest { @@ -108,6 +199,16 @@ interface MetricsDiscoveryServerCallbacksTest { fun envoy(): EnvoyExtension + fun expectedGrpcConnectionsGaugeValues(): Map + + fun expectedGrpcRequestsCounterValues(): Map Boolean> + + fun MeterRegistry.counterValue(name: String) = this.find(name).counter()?.count()?.toInt() + + fun isGreaterThanZero() = { x: Int? -> x!! > 0 } + + fun isNull() = { x: Int? -> x == null } + @Test fun `should measure gRPC connections`() { // given @@ -124,8 +225,6 @@ interface MetricsDiscoveryServerCallbacksTest { } } - fun expectedGrpcConnectionsGaugeValues(): Map - @Test fun `should measure gRPC requests`() { // given @@ -134,14 +233,11 @@ interface MetricsDiscoveryServerCallbacksTest { // expect untilAsserted { - listOf(CDS, EDS, LDS, RDS).forEach { - assertThat(meterRegistry.counterValue("grpc.requests.${it.name.toLowerCase()}")).isGreaterThan(0) - } - listOf(SDS, ADS, UNKNOWN).forEach { - assertThat(meterRegistry.counterValue("grpc.requests.${it.name.toLowerCase()}")).isNull() + expectedGrpcRequestsCounterValues().forEach { (type, condition) -> + val counterValue = meterRegistry.counterValue("grpc.requests.$type") + println("$type $counterValue") + assertThat(counterValue).satisfies { condition(it) } } } } - - fun MeterRegistry.counterValue(name: String) = this.find(name).counter()?.count()?.toInt() } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index 4f0d27a61..e442be828 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -38,6 +38,7 @@ val AdsCustomHealthCheck = EnvoyConfig("envoy/config_ads_custom_health_check.yam val AdsDynamicForwardProxy = EnvoyConfig("envoy/config_ads_dynamic_forward_proxy.yaml") val FaultyConfig = EnvoyConfig("envoy/bad_config.yaml") val Ads = EnvoyConfig("envoy/config_ads.yaml") +val DeltaAds = EnvoyConfig("envoy/config_delta_ads.yaml") val AdsV2 = EnvoyConfig("envoy/config_ads_v2.yaml") val Echo1EnvoyAuthConfig = EnvoyConfig("envoy/config_auth.yaml") val Echo2EnvoyAuthConfig = Echo1EnvoyAuthConfig.copy( diff --git a/envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml b/envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml new file mode 100644 index 000000000..62f1b971d --- /dev/null +++ b/envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml @@ -0,0 +1,118 @@ +admin: + access_log_path: /dev/null + address: + socket_address: { address: 0.0.0.0, port_value: 10000 } +dynamic_resources: + lds_config: + resource_api_version: V3 + ads: {} + cds_config: + resource_api_version: V3 + ads: {} + ads_config: + transport_api_version: V3 + api_type: DELTA_GRPC + grpc_services: + envoy_grpc: + cluster_name: envoy-control-xds +node: + cluster: test-cluster + id: test-id + metadata: + service_name: "echo2" + ads: true + ingress_host: "0.0.0.0" + ingress_port: 5001 + egress_host: "0.0.0.0" + egress_port: 5000 + use_remote_address: true + generate_request_id: true + preserve_external_request_id: true + access_log_enabled: false + add_upstream_external_address_header: true + resources_dir: "/etc/envoy/extra" + proxy_settings: + incoming: + endpoints: + - path: "/endpoint" + clients: ["authorizedClient"] + - path: "/secured_endpoint" + clients: ["echo"] + outgoing: + dependencies: + - service: "service-1" + - service: "service-2" + - service: "service-3" + - service: "service-4" + - service: "service-5" + - service: "echo" + timeoutPolicy: + requestTimeout: "15s" + - service: "consul" + timeoutPolicy: + requestTimeout: "15s" + - service: "proxy1" + - service: "proxy2" + - service: "service-redirect" + handleInternalRedirect: true + - service: "host-rewrite-service" + rewriteHostHeader: true + - domain: "https://my.example.com" + - domain: "https://bad.host.example.com" + - domain: "https://www.example.com" + - domain: "https://www.example-redirect.com" + handleInternalRedirect: true + +static_resources: + clusters: + - connect_timeout: 1s + load_assignment: + cluster_name: envoy-control-xds + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: HOST_IP + port_value: HOST_PORT + - endpoint: + address: + socket_address: + address: HOST_IP + port_value: HOST2_PORT + http2_protocol_options: {} + name: envoy-control-xds + - name: envoy-original-destination + type: ORIGINAL_DST + lb_policy: CLUSTER_PROVIDED + original_dst_lb_config: + use_http_header: true + connect_timeout: + seconds: 1 + http_protocol_options: + allow_absolute_url: true + - name: local_service + type: STATIC + load_assignment: + cluster_name: local_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: LOCAL_SERVICE_IP + port_value: 5678 + connect_timeout: 1s + - name: this_admin + type: STATIC + load_assignment: + cluster_name: this_admin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 10000 + connect_timeout: + seconds: 1 From 7757905440e560da739aff660655a412c7c632f0 Mon Sep 17 00:00:00 2001 From: slonka Date: Tue, 18 May 2021 11:33:55 +0200 Subject: [PATCH 12/46] Add delta requests in metrics docs --- docs/deployment/observability.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/deployment/observability.md b/docs/deployment/observability.md index a9567485e..ebce6bb30 100644 --- a/docs/deployment/observability.md +++ b/docs/deployment/observability.md @@ -51,14 +51,20 @@ Metric | Description #### xDS requests -Metric | Description ---------------------------| -------------------------------------------------------- -**grpc.requests.cds** | Counter of received gRPC CDS requests -**grpc.requests.eds** | Counter of received gRPC EDS requests -**grpc.requests.lds** | Counter of received gRPC LDS requests -**grpc.requests.rds** | Counter of received gRPC RDS requests -**grpc.requests.sds** | Counter of received gRPC SDS requests -**grpc.requests.unknown** | Counter of received gRPC requests for unknown resource +Metric | Description +------------------------------- | -------------------------------------------------------- +**grpc.requests.cds** | Counter of received gRPC CDS requests +**grpc.requests.eds** | Counter of received gRPC EDS requests +**grpc.requests.lds** | Counter of received gRPC LDS requests +**grpc.requests.rds** | Counter of received gRPC RDS requests +**grpc.requests.sds** | Counter of received gRPC SDS requests +**grpc.requests.unknown** | Counter of received gRPC requests for unknown resource +**grpc.requests.cds.delta** | Counter of received gRPC delta CDS requests +**grpc.requests.eds.delta** | Counter of received gRPC delta EDS requests +**grpc.requests.lds.delta** | Counter of received gRPC delta LDS requests +**grpc.requests.rds.delta** | Counter of received gRPC delta RDS requests +**grpc.requests.sds.delta** | Counter of received gRPC delta SDS requests +**grpc.requests.unknown.delta** | Counter of received gRPC delta requests for unknown resource #### Snapshot From b5aa6bd279bc570130638358d6839cfc40387339 Mon Sep 17 00:00:00 2001 From: slonka Date: Tue, 18 May 2021 12:00:17 +0200 Subject: [PATCH 13/46] Reused ADS config instead of making another one --- .../config/EnvoyControlTestConfiguration.kt | 8 +- .../resources/envoy/config_delta_ads.yaml | 118 ------------------ 2 files changed, 7 insertions(+), 119 deletions(-) delete mode 100644 envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index e442be828..d47e5f7f9 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -38,7 +38,13 @@ val AdsCustomHealthCheck = EnvoyConfig("envoy/config_ads_custom_health_check.yam val AdsDynamicForwardProxy = EnvoyConfig("envoy/config_ads_dynamic_forward_proxy.yaml") val FaultyConfig = EnvoyConfig("envoy/bad_config.yaml") val Ads = EnvoyConfig("envoy/config_ads.yaml") -val DeltaAds = EnvoyConfig("envoy/config_delta_ads.yaml") +val DeltaAds = Ads.copy( + configOverride = """ + dynamic_resources: + ads_config: + api_type: DELTA_GRPC +""".trimIndent() +) val AdsV2 = EnvoyConfig("envoy/config_ads_v2.yaml") val Echo1EnvoyAuthConfig = EnvoyConfig("envoy/config_auth.yaml") val Echo2EnvoyAuthConfig = Echo1EnvoyAuthConfig.copy( diff --git a/envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml b/envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml deleted file mode 100644 index 62f1b971d..000000000 --- a/envoy-control-tests/src/main/resources/envoy/config_delta_ads.yaml +++ /dev/null @@ -1,118 +0,0 @@ -admin: - access_log_path: /dev/null - address: - socket_address: { address: 0.0.0.0, port_value: 10000 } -dynamic_resources: - lds_config: - resource_api_version: V3 - ads: {} - cds_config: - resource_api_version: V3 - ads: {} - ads_config: - transport_api_version: V3 - api_type: DELTA_GRPC - grpc_services: - envoy_grpc: - cluster_name: envoy-control-xds -node: - cluster: test-cluster - id: test-id - metadata: - service_name: "echo2" - ads: true - ingress_host: "0.0.0.0" - ingress_port: 5001 - egress_host: "0.0.0.0" - egress_port: 5000 - use_remote_address: true - generate_request_id: true - preserve_external_request_id: true - access_log_enabled: false - add_upstream_external_address_header: true - resources_dir: "/etc/envoy/extra" - proxy_settings: - incoming: - endpoints: - - path: "/endpoint" - clients: ["authorizedClient"] - - path: "/secured_endpoint" - clients: ["echo"] - outgoing: - dependencies: - - service: "service-1" - - service: "service-2" - - service: "service-3" - - service: "service-4" - - service: "service-5" - - service: "echo" - timeoutPolicy: - requestTimeout: "15s" - - service: "consul" - timeoutPolicy: - requestTimeout: "15s" - - service: "proxy1" - - service: "proxy2" - - service: "service-redirect" - handleInternalRedirect: true - - service: "host-rewrite-service" - rewriteHostHeader: true - - domain: "https://my.example.com" - - domain: "https://bad.host.example.com" - - domain: "https://www.example.com" - - domain: "https://www.example-redirect.com" - handleInternalRedirect: true - -static_resources: - clusters: - - connect_timeout: 1s - load_assignment: - cluster_name: envoy-control-xds - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: HOST_IP - port_value: HOST_PORT - - endpoint: - address: - socket_address: - address: HOST_IP - port_value: HOST2_PORT - http2_protocol_options: {} - name: envoy-control-xds - - name: envoy-original-destination - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - original_dst_lb_config: - use_http_header: true - connect_timeout: - seconds: 1 - http_protocol_options: - allow_absolute_url: true - - name: local_service - type: STATIC - load_assignment: - cluster_name: local_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: LOCAL_SERVICE_IP - port_value: 5678 - connect_timeout: 1s - - name: this_admin - type: STATIC - load_assignment: - cluster_name: this_admin - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 10000 - connect_timeout: - seconds: 1 From 07624434f069aa390e1146586609f0b0c16b90ef Mon Sep 17 00:00:00 2001 From: slonka Date: Wed, 19 May 2021 10:27:23 +0200 Subject: [PATCH 14/46] Replace star imports with single line imports --- .../envoycontrol/v2/SimpleCacheTest.java | 20 ++++++++++++------- .../SimpleCacheWithMissingEndpointsTest.java | 11 +++++----- .../envoycontrol/v3/SimpleCacheTest.java | 15 +++++++++++--- .../SimpleCacheWithMissingEndpointsTest.java | 11 +++++----- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java index 9f2ebc49e..f883820bb 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java @@ -1,12 +1,15 @@ package pl.allegro.tech.servicemesh.envoycontrol.v2; -import static io.envoyproxy.controlplane.cache.Resources.V2.CLUSTER_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.V2.ROUTE_TYPE_URL; -import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.*; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.StatusInfo; +import io.envoyproxy.controlplane.cache.VersionedResource; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; @@ -15,8 +18,8 @@ import io.envoyproxy.envoy.api.v2.RouteConfiguration; import io.envoyproxy.envoy.api.v2.auth.Secret; import io.envoyproxy.envoy.api.v2.core.Node; -import java.util.Arrays; -import java.util.Collection; +import org.junit.Test; + import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -25,7 +28,10 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import java.util.stream.Collectors; -import org.junit.Test; + +import static io.envoyproxy.controlplane.cache.Resources.V2.CLUSTER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.ROUTE_TYPE_URL; +import static org.assertj.core.api.Assertions.assertThat; /** * This class is copy of {@link io.envoyproxy.controlplane.cache.v2.SimpleCacheTest} diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java index 875d369ee..38acfeb61 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheWithMissingEndpointsTest.java @@ -2,21 +2,22 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.*; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.VersionedResource; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import io.envoyproxy.envoy.api.v2.core.Node; - -import static org.assertj.core.api.Assertions.assertThat; - import org.junit.Ignore; import org.junit.jupiter.api.Test; import java.util.Collections; -import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static org.assertj.core.api.Assertions.assertThat; public class SimpleCacheWithMissingEndpointsTest extends SimpleCacheTest { diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java index ffb5cf02a..a94aafc4d 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java @@ -3,7 +3,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.*; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.StatusInfo; +import io.envoyproxy.controlplane.cache.VersionedResource; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.cache.v3.Snapshot; import io.envoyproxy.envoy.config.cluster.v3.Cluster; import io.envoyproxy.envoy.config.core.v3.Node; @@ -13,9 +19,12 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; import org.junit.Test; -import pl.allegro.tech.servicemesh.envoycontrol.v3.SimpleCache; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import java.util.stream.Collectors; diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java index e78ad708f..77d9ee749 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheWithMissingEndpointsTest.java @@ -2,21 +2,22 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.*; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.VersionedResource; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.cache.v3.Snapshot; import io.envoyproxy.envoy.config.core.v3.Node; import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; - -import static org.assertj.core.api.Assertions.assertThat; - import org.junit.Ignore; import org.junit.jupiter.api.Test; import java.util.Collections; -import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static org.assertj.core.api.Assertions.assertThat; public class SimpleCacheWithMissingEndpointsTest extends SimpleCacheTest { From 1f93c5ed5622a706e64b1c04ef0bb1a6275d7426 Mon Sep 17 00:00:00 2001 From: slonka Date: Wed, 19 May 2021 11:16:47 +0200 Subject: [PATCH 15/46] Fix imports --- .../groups/NodeMetadataValidator.kt | 3 +- .../CompositeDiscoveryServerCallbacks.kt | 23 ++++++----- .../LoggingDiscoveryServerCallbacks.kt | 40 +++++++++---------- .../MetricsDiscoveryServerCallbacks.kt | 11 ++--- .../snapshot/EnvoySnapshotFactory.kt | 4 +- 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt index 5b8c75efa..d9d752d57 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt @@ -12,6 +12,7 @@ import io.envoyproxy.envoy.api.v2.DiscoveryRequest as DiscoveryRequestV2 import io.envoyproxy.envoy.api.v2.core.Node as NodeV2 import io.envoyproxy.envoy.config.core.v3.Node as NodeV3 import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as DiscoveryRequestV3 +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest as DeltaDiscoveryRequestV3 class V2NotSupportedException : NodeMetadataValidationException( "Blocked service from receiving updates. V2 resources are not supported by server." @@ -64,7 +65,7 @@ class NodeMetadataValidator( override fun onV3StreamDeltaRequest( streamId: Long, - request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest? + request: DeltaDiscoveryRequestV3? ) { request?.node?.let { validateV3Metadata(it) } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt index ea7f3cdc2..f1ddc53eb 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/CompositeDiscoveryServerCallbacks.kt @@ -5,10 +5,11 @@ import io.envoyproxy.controlplane.server.exception.RequestException import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest import io.micrometer.core.instrument.MeterRegistry import pl.allegro.tech.servicemesh.envoycontrol.logger -import io.envoyproxy.envoy.api.v2.DiscoveryRequest as v2DiscoveryRequest -import io.envoyproxy.envoy.api.v2.DiscoveryResponse as v2DiscoveryResponse -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as v3DiscoveryRequest -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse as v3DiscoveryResponse +import io.envoyproxy.envoy.api.v2.DiscoveryRequest as DiscoveryRequestV2 +import io.envoyproxy.envoy.api.v2.DiscoveryResponse as DiscoveryResponseV2 +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest as DeltaDiscoveryRequestV3 +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as DiscoveryRequestV3 +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse as DiscoveryResponseV3 class CompositeException(exceptions: List) : RuntimeException("Composite exception: " + exceptions.map { it.message }.joinToString(",", "[", "]")) @@ -37,13 +38,13 @@ class CompositeDiscoveryServerCallbacks( } } - override fun onV2StreamRequest(streamId: Long, request: v2DiscoveryRequest?) { + override fun onV2StreamRequest(streamId: Long, request: DiscoveryRequestV2?) { runCallbacks { it.onV2StreamRequest(streamId, request) } } - override fun onV3StreamRequest(streamId: Long, request: v3DiscoveryRequest?) { + override fun onV3StreamRequest(streamId: Long, request: DiscoveryRequestV3?) { runCallbacks { it.onV3StreamRequest(streamId, request) } @@ -57,7 +58,7 @@ class CompositeDiscoveryServerCallbacks( override fun onV3StreamDeltaRequest( streamId: Long, - request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest? + request: DeltaDiscoveryRequestV3? ) { runCallbacks { it.onV3StreamDeltaRequest(streamId, request) @@ -66,8 +67,8 @@ class CompositeDiscoveryServerCallbacks( override fun onStreamResponse( streamId: Long, - request: v2DiscoveryRequest?, - response: v2DiscoveryResponse? + request: DiscoveryRequestV2?, + response: DiscoveryResponseV2? ) { runCallbacks { it.onStreamResponse(streamId, request, response) @@ -76,8 +77,8 @@ class CompositeDiscoveryServerCallbacks( override fun onV3StreamResponse( streamId: Long, - request: v3DiscoveryRequest?, - response: v3DiscoveryResponse? + request: DiscoveryRequestV3?, + response: DiscoveryResponseV3? ) { runCallbacks { it.onV3StreamResponse(streamId, request, response) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt index 331d2f4f1..3d88beab4 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/LoggingDiscoveryServerCallbacks.kt @@ -2,12 +2,12 @@ package pl.allegro.tech.servicemesh.envoycontrol.server.callbacks import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks import org.slf4j.LoggerFactory -import io.envoyproxy.envoy.api.v2.DiscoveryRequest as v2DiscoveryRequest -import io.envoyproxy.envoy.api.v2.DiscoveryResponse as v2DiscoveryResponse -import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest as v2DeltaDiscoveryRequest -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as v3DiscoveryRequest -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse as v3DiscoveryResponse -import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest as v3DeltaDiscoveryRequest +import io.envoyproxy.envoy.api.v2.DiscoveryRequest as DiscoveryRequestV2 +import io.envoyproxy.envoy.api.v2.DiscoveryResponse as DiscoveryResponseV2 +import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest as DeltaDiscoveryRequestV2 +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest as DeltaDiscoveryRequestV3 +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as DiscoveryRequestV3 +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse as DiscoveryResponseV3 class LoggingDiscoveryServerCallbacks( private val logFullRequest: Boolean, @@ -19,17 +19,17 @@ class LoggingDiscoveryServerCallbacks( logger.debug("onStreamClose streamId: {} typeUrl: {}", streamId, typeUrl) } - override fun onV3StreamRequest(streamId: Long, request: v3DiscoveryRequest?) { + override fun onV3StreamRequest(streamId: Long, request: DiscoveryRequestV3?) { logger.debug("onV3StreamRequest streamId: {} request: {}", streamId, requestData(request)) } - override fun onV2StreamDeltaRequest(streamId: Long, request: v2DeltaDiscoveryRequest?) { + override fun onV2StreamDeltaRequest(streamId: Long, request: DeltaDiscoveryRequestV2?) { logger.debug("onV2StreamDeltaRequest streamId: {} request: {}", streamId, requestData(request)) } override fun onV3StreamDeltaRequest( streamId: Long, - request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest? + request: DeltaDiscoveryRequestV3? ) { logger.debug("onV3StreamDeltaRequest streamId: {} request: {}", streamId, requestData(request)) } @@ -42,14 +42,14 @@ class LoggingDiscoveryServerCallbacks( logger.debug("onStreamOpen streamId: {}, typeUrl: {}", streamId, typeUrl) } - override fun onV2StreamRequest(streamId: Long, request: v2DiscoveryRequest?) { + override fun onV2StreamRequest(streamId: Long, request: DiscoveryRequestV2?) { logger.debug("onV2StreamRequest streamId: {} request: {}", streamId, requestData(request)) } override fun onStreamResponse( streamId: Long, - request: v2DiscoveryRequest?, - response: v2DiscoveryResponse? + request: DiscoveryRequestV2?, + response: DiscoveryResponseV2? ) { logger.debug( "onStreamResponseV2 streamId: {}, request: {}, response: {}", @@ -61,8 +61,8 @@ class LoggingDiscoveryServerCallbacks( override fun onV3StreamResponse( streamId: Long, - request: v3DiscoveryRequest?, - response: v3DiscoveryResponse? + request: DiscoveryRequestV3?, + response: DiscoveryResponseV3? ) { logger.debug( "onStreamResponseV3 streamId: {}, request: {}, response: {}", @@ -72,7 +72,7 @@ class LoggingDiscoveryServerCallbacks( ) } - private fun requestData(request: v3DeltaDiscoveryRequest?): String { + private fun requestData(request: DeltaDiscoveryRequestV3?): String { return if (logFullRequest) { "$request" } else { @@ -81,7 +81,7 @@ class LoggingDiscoveryServerCallbacks( } } - private fun requestData(request: v2DeltaDiscoveryRequest?): String { + private fun requestData(request: DeltaDiscoveryRequestV2?): String { return if (logFullRequest) { "$request" } else { @@ -90,7 +90,7 @@ class LoggingDiscoveryServerCallbacks( } } - private fun requestData(request: v2DiscoveryRequest?): String { + private fun requestData(request: DiscoveryRequestV2?): String { return if (logFullRequest) { "$request" } else { @@ -99,7 +99,7 @@ class LoggingDiscoveryServerCallbacks( } } - private fun responseData(response: v2DiscoveryResponse?): String { + private fun responseData(response: DiscoveryResponseV2?): String { return if (logFullResponse) { "$response" } else { @@ -107,7 +107,7 @@ class LoggingDiscoveryServerCallbacks( } } - private fun requestData(request: v3DiscoveryRequest?): String { + private fun requestData(request: DiscoveryRequestV3?): String { return if (logFullRequest) { "$request" } else { @@ -116,7 +116,7 @@ class LoggingDiscoveryServerCallbacks( } } - private fun responseData(response: v3DiscoveryResponse?): String { + private fun responseData(response: DiscoveryResponseV3?): String { return if (logFullResponse) { "$response" } else { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt index 0bee84530..ac3ba6a00 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt @@ -3,8 +3,9 @@ package pl.allegro.tech.servicemesh.envoycontrol.server.callbacks import io.envoyproxy.controlplane.cache.Resources import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks import io.envoyproxy.envoy.api.v2.DeltaDiscoveryRequest -import io.envoyproxy.envoy.api.v2.DiscoveryRequest as V2DiscoveryRequest -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as V3DiscoveryRequest +import io.envoyproxy.envoy.api.v2.DiscoveryRequest as DiscoveryRequestV2 +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as DiscoveryRequestV3 +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest as DeltaDiscoveryRequestV3 import io.micrometer.core.instrument.MeterRegistry import java.util.concurrent.atomic.AtomicInteger @@ -55,20 +56,20 @@ class MetricsDiscoveryServerCallbacks(private val meterRegistry: MeterRegistry) connectionsByType(typeUrl).decrementAndGet() } - override fun onV3StreamRequest(streamId: Long, request: V3DiscoveryRequest) { + override fun onV3StreamRequest(streamId: Long, request: DiscoveryRequestV3) { meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}") .increment() } override fun onV3StreamDeltaRequest( streamId: Long, - request: io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest + request: DeltaDiscoveryRequestV3 ) { meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}.delta") .increment() } - override fun onV2StreamRequest(streamId: Long, request: V2DiscoveryRequest) { + override fun onV2StreamRequest(streamId: Long, request: DiscoveryRequestV2) { meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}") .increment() } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index ccfeea890..08f12a7f6 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -217,9 +217,9 @@ class EnvoySnapshotFactory( globalSnapshot: GlobalSnapshot, egressRouteSpecifications: Collection ): List { - val resources = globalSnapshot.endpoints + val endpoints = globalSnapshot.endpoints return egressRouteSpecifications - .mapNotNull { resources.get(it.clusterName) } + .mapNotNull { endpoints.get(it.clusterName) } } private fun newSnapshotForGroup( From b104288a126f22359a4a01ebe575366901f7a7cf Mon Sep 17 00:00:00 2001 From: slonka Date: Thu, 20 May 2021 08:22:16 +0200 Subject: [PATCH 16/46] Use DeltaAds as random --- .../envoycontrol/config/EnvoyControlTestConfiguration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index d47e5f7f9..b49d51187 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -61,7 +61,7 @@ val AdsWithDisabledEndpointPermissions = EnvoyConfig("envoy/config_ads_disabled_ val AdsWithStaticListeners = EnvoyConfig("envoy/config_ads_static_listeners.yaml") val AdsWithNoDependencies = EnvoyConfig("envoy/config_ads_no_dependencies.yaml") val Xds = EnvoyConfig("envoy/config_xds.yaml") -val RandomConfigFile = if (Random.nextBoolean()) Ads else Xds +val RandomConfigFile = DeltaAds @Deprecated("use extension approach instead, e.g. RetryPolicyTest") abstract class EnvoyControlTestConfiguration : BaseEnvoyTest() { From 93847e71f5bf3f31bb8b4a0de33c017461e86700 Mon Sep 17 00:00:00 2001 From: slonka Date: Thu, 20 May 2021 08:22:38 +0200 Subject: [PATCH 17/46] Add DeltaAds to random config list --- .../envoycontrol/config/EnvoyControlTestConfiguration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index b49d51187..7a01c0c00 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -61,7 +61,7 @@ val AdsWithDisabledEndpointPermissions = EnvoyConfig("envoy/config_ads_disabled_ val AdsWithStaticListeners = EnvoyConfig("envoy/config_ads_static_listeners.yaml") val AdsWithNoDependencies = EnvoyConfig("envoy/config_ads_no_dependencies.yaml") val Xds = EnvoyConfig("envoy/config_xds.yaml") -val RandomConfigFile = DeltaAds +val RandomConfigFile = listOf(Ads, Xds, DeltaAds).random() @Deprecated("use extension approach instead, e.g. RetryPolicyTest") abstract class EnvoyControlTestConfiguration : BaseEnvoyTest() { From 559589753267316cfd214fe1ff4245d42d7c482e Mon Sep 17 00:00:00 2001 From: slonka Date: Thu, 20 May 2021 08:27:04 +0200 Subject: [PATCH 18/46] Fix ktlint --- .../envoycontrol/config/EnvoyControlTestConfiguration.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index 7a01c0c00..7f07bfa66 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -23,7 +23,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoContainer import pl.allegro.tech.servicemesh.envoycontrol.logger import java.time.Duration import java.util.concurrent.TimeUnit -import kotlin.random.Random data class EnvoyConfig( val filePath: String, @@ -61,7 +60,7 @@ val AdsWithDisabledEndpointPermissions = EnvoyConfig("envoy/config_ads_disabled_ val AdsWithStaticListeners = EnvoyConfig("envoy/config_ads_static_listeners.yaml") val AdsWithNoDependencies = EnvoyConfig("envoy/config_ads_no_dependencies.yaml") val Xds = EnvoyConfig("envoy/config_xds.yaml") -val RandomConfigFile = listOf(Ads, Xds, DeltaAds).random() +val RandomConfigFile = DeltaAds @Deprecated("use extension approach instead, e.g. RetryPolicyTest") abstract class EnvoyControlTestConfiguration : BaseEnvoyTest() { From 600d7f3ac83eb7209186fbf72bd86ed3ce563b93 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 10:50:21 +0200 Subject: [PATCH 19/46] Only send RDS when LDS or RDS changed --- .../snapshot/EnvoySnapshotFactory.kt | 2 +- .../envoycontrol/snapshot/SnapshotsVersions.kt | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 08f12a7f6..1a99fe09b 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -252,7 +252,7 @@ class EnvoySnapshotFactory( // which is present only in services (not domains) so it could be implemented differently. val endpoints = getServicesEndpointsForGroup(globalSnapshot, egressRouteSpecification) - val version = snapshotsVersions.version(group, clusters, endpoints, listeners) + val version = snapshotsVersions.version(group, clusters, endpoints, listeners, routes) return createSnapshot( clusters = clusters, diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt index 8889a16d7..d1d9c37d5 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt @@ -3,6 +3,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment import io.envoyproxy.envoy.config.listener.v3.Listener +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotsVersions.Companion.newVersion import java.util.UUID @@ -31,7 +32,8 @@ class SnapshotsVersions { group: Group, clusters: List, endpoints: List, - listeners: List = listOf() + listeners: List = listOf(), + routes: List = listOf() ): Version { val versionsWithData = versions.compute(group) { _, previous -> val version = when (previous) { @@ -44,15 +46,16 @@ class SnapshotsVersions { else -> { val clustersChanged = previous.clusters != clusters val listenersChanged = previous.listeners != listeners + val routesChanged = previous.routes != routes Version( clusters = selectClusters(previous, clusters, clustersChanged), endpoints = selectEndpoints(previous, endpoints, clustersChanged), listeners = selectListeners(previous, listenersChanged), - routes = selectRoutes(previous, listenersChanged, clustersChanged) + routes = selectRoutes(previous, listenersChanged, routesChanged) ) } } - VersionsWithData(version, clusters, endpoints, listeners) + VersionsWithData(version, clusters, endpoints, listeners, routes) } return versionsWithData!!.version } @@ -60,9 +63,9 @@ class SnapshotsVersions { private fun selectRoutes( previous: VersionsWithData, listenersChanged: Boolean, - clustersChanged: Boolean + routesChanged: Boolean ): RoutesVersion { - return if (listenersChanged || clustersChanged) RoutesVersion(newVersion()) else previous.version.routes + return if (listenersChanged || routesChanged) RoutesVersion(newVersion()) else previous.version.routes } private fun selectListeners(previous: VersionsWithData, hasChanged: Boolean): ListenersVersion { @@ -101,7 +104,8 @@ class SnapshotsVersions { val version: Version, val clusters: List, val endpoints: List, - val listeners: List + val listeners: List, + val routes: List ) data class Version( From 85da78255feac4ded3fa3de26f486ffb0a25da4a Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 10:55:40 +0200 Subject: [PATCH 20/46] Use delta xds everywhere --- envoy-control-tests/src/main/resources/envoy/bad_config.yaml | 2 +- envoy-control-tests/src/main/resources/envoy/config_ads.yaml | 2 +- .../src/main/resources/envoy/config_ads_all_dependencies.yaml | 2 +- .../main/resources/envoy/config_ads_custom_health_check.yaml | 2 +- .../envoy/config_ads_disabled_endpoint_permissions.yaml | 2 +- .../main/resources/envoy/config_ads_dynamic_forward_proxy.yaml | 2 +- .../src/main/resources/envoy/config_ads_no_dependencies.yaml | 2 +- .../src/main/resources/envoy/config_ads_static_listeners.yaml | 2 +- envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml | 2 +- envoy-control-tests/src/main/resources/envoy/config_auth.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/envoy-control-tests/src/main/resources/envoy/bad_config.yaml b/envoy-control-tests/src/main/resources/envoy/bad_config.yaml index bbebcfee6..637aa2f92 100644 --- a/envoy-control-tests/src/main/resources/envoy/bad_config.yaml +++ b/envoy-control-tests/src/main/resources/envoy/bad_config.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads.yaml index 26c4d0a95..62f1b971d 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml index b6a5788a6..2131947b1 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml index fd13368c4..644e6b299 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml index ddee4bdbc..de43436ee 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml index 21062f4cf..2d0ec19bf 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml index 3f34c2520..6fce98c47 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml index 6e35391a7..a86ddb65d 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml index 5ea7254ee..624811028 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml @@ -8,7 +8,7 @@ dynamic_resources: cds_config: ads: {} ads_config: - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_auth.yaml b/envoy-control-tests/src/main/resources/envoy/config_auth.yaml index b6c8c5ff8..650cf739b 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_auth.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_auth.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: GRPC + api_type: DELTA_GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds From b9637cb198cbc096ff71d764d2c1f95083ce9b48 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 12:39:00 +0200 Subject: [PATCH 21/46] Revert "Only send RDS when LDS or RDS changed" This reverts commit 600d7f3ac83eb7209186fbf72bd86ed3ce563b93. --- .../snapshot/EnvoySnapshotFactory.kt | 2 +- .../envoycontrol/snapshot/SnapshotsVersions.kt | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 1a99fe09b..08f12a7f6 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -252,7 +252,7 @@ class EnvoySnapshotFactory( // which is present only in services (not domains) so it could be implemented differently. val endpoints = getServicesEndpointsForGroup(globalSnapshot, egressRouteSpecification) - val version = snapshotsVersions.version(group, clusters, endpoints, listeners, routes) + val version = snapshotsVersions.version(group, clusters, endpoints, listeners) return createSnapshot( clusters = clusters, diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt index d1d9c37d5..8889a16d7 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt @@ -3,7 +3,6 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment import io.envoyproxy.envoy.config.listener.v3.Listener -import io.envoyproxy.envoy.config.route.v3.RouteConfiguration import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotsVersions.Companion.newVersion import java.util.UUID @@ -32,8 +31,7 @@ class SnapshotsVersions { group: Group, clusters: List, endpoints: List, - listeners: List = listOf(), - routes: List = listOf() + listeners: List = listOf() ): Version { val versionsWithData = versions.compute(group) { _, previous -> val version = when (previous) { @@ -46,16 +44,15 @@ class SnapshotsVersions { else -> { val clustersChanged = previous.clusters != clusters val listenersChanged = previous.listeners != listeners - val routesChanged = previous.routes != routes Version( clusters = selectClusters(previous, clusters, clustersChanged), endpoints = selectEndpoints(previous, endpoints, clustersChanged), listeners = selectListeners(previous, listenersChanged), - routes = selectRoutes(previous, listenersChanged, routesChanged) + routes = selectRoutes(previous, listenersChanged, clustersChanged) ) } } - VersionsWithData(version, clusters, endpoints, listeners, routes) + VersionsWithData(version, clusters, endpoints, listeners) } return versionsWithData!!.version } @@ -63,9 +60,9 @@ class SnapshotsVersions { private fun selectRoutes( previous: VersionsWithData, listenersChanged: Boolean, - routesChanged: Boolean + clustersChanged: Boolean ): RoutesVersion { - return if (listenersChanged || routesChanged) RoutesVersion(newVersion()) else previous.version.routes + return if (listenersChanged || clustersChanged) RoutesVersion(newVersion()) else previous.version.routes } private fun selectListeners(previous: VersionsWithData, hasChanged: Boolean): ListenersVersion { @@ -104,8 +101,7 @@ class SnapshotsVersions { val version: Version, val clusters: List, val endpoints: List, - val listeners: List, - val routes: List + val listeners: List ) data class Version( From b71a282f424fe9018ef6cddb55564759a0f0400a Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 12:40:03 +0200 Subject: [PATCH 22/46] Revert to random Ads Xds Delta --- .../envoycontrol/config/EnvoyControlTestConfiguration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index 7f07bfa66..0916431cf 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -60,7 +60,7 @@ val AdsWithDisabledEndpointPermissions = EnvoyConfig("envoy/config_ads_disabled_ val AdsWithStaticListeners = EnvoyConfig("envoy/config_ads_static_listeners.yaml") val AdsWithNoDependencies = EnvoyConfig("envoy/config_ads_no_dependencies.yaml") val Xds = EnvoyConfig("envoy/config_xds.yaml") -val RandomConfigFile = DeltaAds +val RandomConfigFile = listOf(Ads, Xds, DeltaAds).random() @Deprecated("use extension approach instead, e.g. RetryPolicyTest") abstract class EnvoyControlTestConfiguration : BaseEnvoyTest() { From 7ab3d835511b8009ffdf3e1f592b9727215839b8 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 13:07:07 +0200 Subject: [PATCH 23/46] Revert "Revert "Only send RDS when LDS or RDS changed"" This reverts commit b9637cb198cbc096ff71d764d2c1f95083ce9b48. --- .../snapshot/EnvoySnapshotFactory.kt | 2 +- .../envoycontrol/snapshot/SnapshotsVersions.kt | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 08f12a7f6..1a99fe09b 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -252,7 +252,7 @@ class EnvoySnapshotFactory( // which is present only in services (not domains) so it could be implemented differently. val endpoints = getServicesEndpointsForGroup(globalSnapshot, egressRouteSpecification) - val version = snapshotsVersions.version(group, clusters, endpoints, listeners) + val version = snapshotsVersions.version(group, clusters, endpoints, listeners, routes) return createSnapshot( clusters = clusters, diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt index 8889a16d7..d1d9c37d5 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotsVersions.kt @@ -3,6 +3,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot import io.envoyproxy.envoy.config.cluster.v3.Cluster import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment import io.envoyproxy.envoy.config.listener.v3.Listener +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotsVersions.Companion.newVersion import java.util.UUID @@ -31,7 +32,8 @@ class SnapshotsVersions { group: Group, clusters: List, endpoints: List, - listeners: List = listOf() + listeners: List = listOf(), + routes: List = listOf() ): Version { val versionsWithData = versions.compute(group) { _, previous -> val version = when (previous) { @@ -44,15 +46,16 @@ class SnapshotsVersions { else -> { val clustersChanged = previous.clusters != clusters val listenersChanged = previous.listeners != listeners + val routesChanged = previous.routes != routes Version( clusters = selectClusters(previous, clusters, clustersChanged), endpoints = selectEndpoints(previous, endpoints, clustersChanged), listeners = selectListeners(previous, listenersChanged), - routes = selectRoutes(previous, listenersChanged, clustersChanged) + routes = selectRoutes(previous, listenersChanged, routesChanged) ) } } - VersionsWithData(version, clusters, endpoints, listeners) + VersionsWithData(version, clusters, endpoints, listeners, routes) } return versionsWithData!!.version } @@ -60,9 +63,9 @@ class SnapshotsVersions { private fun selectRoutes( previous: VersionsWithData, listenersChanged: Boolean, - clustersChanged: Boolean + routesChanged: Boolean ): RoutesVersion { - return if (listenersChanged || clustersChanged) RoutesVersion(newVersion()) else previous.version.routes + return if (listenersChanged || routesChanged) RoutesVersion(newVersion()) else previous.version.routes } private fun selectListeners(previous: VersionsWithData, hasChanged: Boolean): ListenersVersion { @@ -101,7 +104,8 @@ class SnapshotsVersions { val version: Version, val clusters: List, val endpoints: List, - val listeners: List + val listeners: List, + val routes: List ) data class Version( From a310d747942481f3e4d8043ebe13d8d0b67b06e3 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 13:07:16 +0200 Subject: [PATCH 24/46] Revert "Use delta xds everywhere" This reverts commit 85da78255feac4ded3fa3de26f486ffb0a25da4a. --- envoy-control-tests/src/main/resources/envoy/bad_config.yaml | 2 +- envoy-control-tests/src/main/resources/envoy/config_ads.yaml | 2 +- .../src/main/resources/envoy/config_ads_all_dependencies.yaml | 2 +- .../main/resources/envoy/config_ads_custom_health_check.yaml | 2 +- .../envoy/config_ads_disabled_endpoint_permissions.yaml | 2 +- .../main/resources/envoy/config_ads_dynamic_forward_proxy.yaml | 2 +- .../src/main/resources/envoy/config_ads_no_dependencies.yaml | 2 +- .../src/main/resources/envoy/config_ads_static_listeners.yaml | 2 +- envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml | 2 +- envoy-control-tests/src/main/resources/envoy/config_auth.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/envoy-control-tests/src/main/resources/envoy/bad_config.yaml b/envoy-control-tests/src/main/resources/envoy/bad_config.yaml index 637aa2f92..bbebcfee6 100644 --- a/envoy-control-tests/src/main/resources/envoy/bad_config.yaml +++ b/envoy-control-tests/src/main/resources/envoy/bad_config.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads.yaml index 62f1b971d..26c4d0a95 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml index 2131947b1..b6a5788a6 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_all_dependencies.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml index 644e6b299..fd13368c4 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_custom_health_check.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml index de43436ee..ddee4bdbc 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_disabled_endpoint_permissions.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml index 2d0ec19bf..21062f4cf 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_dynamic_forward_proxy.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml index 6fce98c47..3f34c2520 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_no_dependencies.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml index a86ddb65d..6e35391a7 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_static_listeners.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml index 624811028..5ea7254ee 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml @@ -8,7 +8,7 @@ dynamic_resources: cds_config: ads: {} ads_config: - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds diff --git a/envoy-control-tests/src/main/resources/envoy/config_auth.yaml b/envoy-control-tests/src/main/resources/envoy/config_auth.yaml index 650cf739b..b6c8c5ff8 100644 --- a/envoy-control-tests/src/main/resources/envoy/config_auth.yaml +++ b/envoy-control-tests/src/main/resources/envoy/config_auth.yaml @@ -11,7 +11,7 @@ dynamic_resources: ads: {} ads_config: transport_api_version: V3 - api_type: DELTA_GRPC + api_type: GRPC grpc_services: envoy_grpc: cluster_name: envoy-control-xds From f0a5804dcb0c7ed7d3b478fada3aa8859d0f0533 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 15:02:36 +0200 Subject: [PATCH 25/46] Make tests for DeltaAds, Ads with * in dependencies and random --- .../envoycontrol/SnapshotDebugTest.kt | 158 +++++++++++++----- .../config/EnvoyControlTestConfiguration.kt | 9 + 2 files changed, 123 insertions(+), 44 deletions(-) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/SnapshotDebugTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/SnapshotDebugTest.kt index a21194891..2bfd0463b 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/SnapshotDebugTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/SnapshotDebugTest.kt @@ -4,15 +4,75 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted +import pl.allegro.tech.servicemesh.envoycontrol.config.AdsAllDependencies +import pl.allegro.tech.servicemesh.envoycontrol.config.DeltaAdsAllDependencies import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension -open class SnapshotDebugTest { +open class WildcardSnapshotDebugTest : SnapshotDebugTest { + companion object { + @JvmField + @RegisterExtension + val consul = ConsulExtension() + + @JvmField + @RegisterExtension + val envoyControl = EnvoyControlExtension(consul, properties = mapOf( + "envoy-control.envoy.snapshot.outgoing-permissions.services-allowed-to-use-wildcard" to setOf("echo2", "test-service") + )) + @JvmField + @RegisterExtension + val service = EchoServiceExtension() + + @JvmField + @RegisterExtension + val envoy = EnvoyExtension(envoyControl, service, config = AdsAllDependencies) + } + + override fun consul() = consul + + override fun envoyControl() = envoyControl + + override fun envoy() = envoy + + override fun service() = service +} + +open class DeltaWildcardSnapshotDebugTest : SnapshotDebugTest { companion object { + @JvmField + @RegisterExtension + val consul = ConsulExtension() + + @JvmField + @RegisterExtension + val envoyControl = EnvoyControlExtension(consul, properties = mapOf( + "envoy-control.envoy.snapshot.outgoing-permissions.services-allowed-to-use-wildcard" to setOf("echo2", "test-service") + )) + @JvmField + @RegisterExtension + val service = EchoServiceExtension() + + @JvmField + @RegisterExtension + val envoy = EnvoyExtension(envoyControl, service, config = DeltaAdsAllDependencies) + } + + override fun consul() = consul + + override fun envoyControl() = envoyControl + + override fun envoy() = envoy + + override fun service() = service +} + +open class RandomSnapshotDebugTest : SnapshotDebugTest { + companion object { @JvmField @RegisterExtension val consul = ConsulExtension() @@ -30,8 +90,19 @@ open class SnapshotDebugTest { val envoy = EnvoyExtension(envoyControl, service) } + override fun consul() = consul + + override fun envoyControl() = envoyControl + + override fun envoy() = envoy + + override fun service() = service +} + +interface SnapshotDebugTest { + @Test - open fun `should return snapshot debug info containing snapshot versions`() { + fun `should return snapshot debug info containing snapshot versions`() { // given consul().server.operations.registerService(service(), name = "echo") val nodeMetadata = envoy().container.admin().nodeInfo() @@ -53,7 +124,7 @@ open class SnapshotDebugTest { } @Test - open fun `should return snapshot debug info containing snapshot contents`() { + fun `should return snapshot debug info containing snapshot contents`() { // given consul().server.operations.registerService(service(), name = "echo") val nodeMetadata = envoy().container.admin().nodeInfo() @@ -70,49 +141,51 @@ open class SnapshotDebugTest { } } - private val missingNodeJson = """{ - "metadata": { - "service_name": "service-mesh-service-first", - "identity": "", - "service_version": "0.1.16-SKYHELIX-839-eds-version-metric-SNAPSHOT", - "proxy_settings": { - "incoming": { - "endpoints": null, - "healthCheck": null, - "roles": null, - "timeoutPolicy": null - }, - "outgoing": { - "dependencies": [ - { - "handleInternalRedirect": null, - "timeoutPolicy": null, - "endpoints": [], - "domain": null, - "service": "*" - } - ] - } - }, - "ads": true - }, - "locality": { - "zone": "dev-dc4" - } + fun missingNodeJson(): String { + return """{ + "metadata": { + "service_name": "service-mesh-service-first", + "identity": "", + "service_version": "0.1.16-SKYHELIX-839-eds-version-metric-SNAPSHOT", + "proxy_settings": { + "incoming": { + "endpoints": null, + "healthCheck": null, + "roles": null, + "timeoutPolicy": null + }, + "outgoing": { + "dependencies": [ + { + "handleInternalRedirect": null, + "timeoutPolicy": null, + "endpoints": [], + "domain": null, + "service": "*" + } + ] + } + }, + "ads": true + }, + "locality": { + "zone": "dev-dc4" + } + } + """.trim() } -""".trim() @Test - open fun `should inform about missing snapshot when given node does not exist`() { + fun `should inform about missing snapshot when given node does not exist`() { // when - val snapshot = envoyControl().app.getSnapshot(missingNodeJson) + val snapshot = envoyControl().app.getSnapshot(missingNodeJson()) // then assertThat(snapshot.found).isFalse() } @Test - open fun `should return global snapshot debug info from xds`() { + fun `should return global snapshot debug info from xds`() { untilAsserted { // when val snapshot = envoyControl().app.getGlobalSnapshot(xds = true) @@ -125,7 +198,7 @@ open class SnapshotDebugTest { } @Test - open fun `should return global snapshot debug info from ads`() { + fun `should return global snapshot debug info from ads`() { untilAsserted { // when val snapshotXdsNull = envoyControl().app.getGlobalSnapshot(xds = null) @@ -145,11 +218,8 @@ open class SnapshotDebugTest { } } - open fun consul() = consul - - open fun envoyControl() = envoyControl - - open fun envoy() = envoy - - open fun service() = service + fun consul(): ConsulExtension + fun envoyControl(): EnvoyControlExtension + fun envoy(): EnvoyExtension + fun service(): EchoServiceExtension } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index 0916431cf..a04123344 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -44,6 +44,15 @@ val DeltaAds = Ads.copy( api_type: DELTA_GRPC """.trimIndent() ) + +val DeltaAdsAllDependencies = AdsAllDependencies.copy( + configOverride = """ + dynamic_resources: + ads_config: + api_type: DELTA_GRPC +""".trimIndent() +) + val AdsV2 = EnvoyConfig("envoy/config_ads_v2.yaml") val Echo1EnvoyAuthConfig = EnvoyConfig("envoy/config_auth.yaml") val Echo2EnvoyAuthConfig = Echo1EnvoyAuthConfig.copy( From 87d6f5d5e2089ba5b003c42199d59e8a1819c4f4 Mon Sep 17 00:00:00 2001 From: slonka Date: Mon, 24 May 2021 16:03:02 +0200 Subject: [PATCH 26/46] Kick ci From d25a291041eba8bfa98eec43c52c07e97fb0db10 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Wed, 28 Jul 2021 11:09:20 +0200 Subject: [PATCH 27/46] Upgrade Envoy version to 1.8.2 --- ...ndV3SmokeTest.kt => EnvoyControlV3SmokeTest.kt} | 14 ++------------ .../envoycontrol/LocalReplyMappingTest.kt | 6 +++--- .../envoycontrol/config/envoy/EnvoyContainer.kt | 2 +- tools/envoy/Dockerfile | 2 +- 4 files changed, 7 insertions(+), 17 deletions(-) rename envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/{EnvoyControlV2AndV3SmokeTest.kt => EnvoyControlV3SmokeTest.kt} (79%) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV2AndV3SmokeTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt similarity index 79% rename from envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV2AndV3SmokeTest.kt rename to envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt index e6ee6708e..59559b9b3 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV2AndV3SmokeTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt @@ -13,7 +13,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension -class EnvoyControlV2AndV3SmokeTest { +class EnvoyControlV3SmokeTest { companion object { @@ -24,21 +24,13 @@ class EnvoyControlV2AndV3SmokeTest { @JvmField @RegisterExtension val envoyControl = EnvoyControlExtension(consul, mapOf( - "envoy-control.envoy.snapshot.support-v2-configuration" to true + "envoy-control.envoy.snapshot.support-v2-configuration" to false )) - @JvmField - @RegisterExtension - val serviceEnvoyV2 = EchoServiceExtension() - @JvmField @RegisterExtension val serviceEnvoyV3 = EchoServiceExtension() - @JvmField - @RegisterExtension - val envoyV2 = EnvoyExtension(envoyControl, serviceEnvoyV2, AdsV2, apiVersion = 2) - @JvmField @RegisterExtension val envoyV3 = EnvoyExtension(envoyControl, serviceEnvoyV3, Ads) @@ -48,11 +40,9 @@ class EnvoyControlV2AndV3SmokeTest { fun `should create a server listening on a port`() { untilAsserted { // when - val ingressRootEnvoyV2 = envoyV2.ingressOperations.callLocalService("") val ingressRootEnvoyV3 = envoyV3.ingressOperations.callLocalService("") // then - assertThat(ingressRootEnvoyV2).isFrom(serviceEnvoyV2).isOk() assertThat(ingressRootEnvoyV3).isFrom(serviceEnvoyV3).isOk() } } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/LocalReplyMappingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/LocalReplyMappingTest.kt index 166db58c3..f79b9d172 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/LocalReplyMappingTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/LocalReplyMappingTest.kt @@ -41,7 +41,7 @@ class LocalReplyMappingTest { "path":"%REQ(:path)%" }""", "$localReplyPrefix.matchers[1].response-flag-matcher" to listOf( - "NR" + "NC" ), "$localReplyPrefix.matchers[1].status-code-to-return" to 522, "$localReplyPrefix.matchers[1].body-to-return" to "my-custom no route body", @@ -94,7 +94,7 @@ class LocalReplyMappingTest { assertThat( response.body()?.string() - ).contains("Request to service: service-2 responseFlags:NR body: my-custom no route body") + ).contains("Request to service: service-2 responseFlags:NC body: my-custom no route body") assertThat(response.code()).isEqualTo(522) } } @@ -108,7 +108,7 @@ class LocalReplyMappingTest { assertThat( response.body()?.string() - ).contains("""{"destination":{"serviceTag":null,"path":"/api","serviceName":"service-2"},"path":"/api","body":"","responseFlags":"NR"}""") + ).contains("""{"destination":{"serviceTag":null,"path":"/api","serviceName":"service-2"},"path":"/api","body":"","responseFlags":"NC"}""") assertThat(response.code()).isEqualTo(510) } } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyContainer.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyContainer.kt index c526ed8cd..82902ee0b 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyContainer.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyContainer.kt @@ -35,7 +35,7 @@ class EnvoyContainer( const val ENVOY_UID_ENV_NAME = "ENVOY_UID" const val EGRESS_LISTENER_CONTAINER_PORT = 5000 const val INGRESS_LISTENER_CONTAINER_PORT = 5001 - const val DEFAULT_IMAGE = "envoyproxy/envoy:v1.17.1" + const val DEFAULT_IMAGE = "envoyproxy/envoy:v1.18.2" private const val ADMIN_PORT = 10000 } diff --git a/tools/envoy/Dockerfile b/tools/envoy/Dockerfile index dca6d9a42..062642ca5 100644 --- a/tools/envoy/Dockerfile +++ b/tools/envoy/Dockerfile @@ -1,4 +1,4 @@ -FROM allegro/envoy-dev:v1.16.1-dev-lua-segfault-fix-1-16-0-backport-20201118-df9dc819 +FROM envoyproxy/envoy:v1.18.2 ENV PORT=9999:9999 ENV PORT=80:80 From 273abfbb6621b27babddd27c75109d75c8339f5a Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Wed, 28 Jul 2021 11:53:43 +0200 Subject: [PATCH 28/46] fixed ktlint --- build.gradle | 2 +- .../envoycontrol/EnvoyControlV3SmokeTest.kt | 1 - .../config/EnvoyControlTestConfiguration.kt | 1 - .../main/resources/envoy/config_ads_v2.yaml | 115 ------------------ 4 files changed, 1 insertion(+), 118 deletions(-) delete mode 100644 envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml diff --git a/build.gradle b/build.gradle index 174ec0d3a..7025e9d62 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ allprojects { project.ext.versions = [ kotlin : '1.3.72', - java_controlplane : '0.1.27', + java_controlplane : '0.1.28', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt index 59559b9b3..773cd0621 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt @@ -7,7 +7,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.assertions.isFrom import pl.allegro.tech.servicemesh.envoycontrol.assertions.isOk import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted import pl.allegro.tech.servicemesh.envoycontrol.config.Ads -import pl.allegro.tech.servicemesh.envoycontrol.config.AdsV2 import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index bf50ce8f2..ae9525709 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -38,7 +38,6 @@ val AdsCustomHealthCheck = EnvoyConfig("envoy/config_ads_custom_health_check.yam val AdsDynamicForwardProxy = EnvoyConfig("envoy/config_ads_dynamic_forward_proxy.yaml") val FaultyConfig = EnvoyConfig("envoy/bad_config.yaml") val Ads = EnvoyConfig("envoy/config_ads.yaml") -val AdsV2 = EnvoyConfig("envoy/config_ads_v2.yaml") val Echo1EnvoyAuthConfig = EnvoyConfig("envoy/config_auth.yaml") val Echo2EnvoyAuthConfig = Echo1EnvoyAuthConfig.copy( serviceName = "echo2", diff --git a/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml b/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml deleted file mode 100644 index 5ea7254ee..000000000 --- a/envoy-control-tests/src/main/resources/envoy/config_ads_v2.yaml +++ /dev/null @@ -1,115 +0,0 @@ -admin: - access_log_path: /dev/null - address: - socket_address: { address: 0.0.0.0, port_value: 10000 } -dynamic_resources: - lds_config: - ads: {} - cds_config: - ads: {} - ads_config: - api_type: GRPC - grpc_services: - envoy_grpc: - cluster_name: envoy-control-xds -node: - cluster: test-cluster - id: test-id - metadata: - service_name: "echo2" - ads: true - ingress_host: "0.0.0.0" - ingress_port: 5001 - egress_host: "0.0.0.0" - egress_port: 5000 - use_remote_address: true - generate_request_id: true - preserve_external_request_id: true - access_log_enabled: false - add_upstream_external_address_header: true - resources_dir: "/etc/envoy/extra" - proxy_settings: - incoming: - endpoints: - - path: "/endpoint" - clients: ["authorizedClient"] - - path: "/secured_endpoint" - clients: ["echo"] - outgoing: - dependencies: - - service: "service-1" - - service: "service-2" - - service: "service-3" - - service: "service-4" - - service: "service-5" - - service: "echo" - timeoutPolicy: - requestTimeout: "15s" - - service: "consul" - timeoutPolicy: - requestTimeout: "15s" - - service: "proxy1" - - service: "proxy2" - - service: "service-redirect" - handleInternalRedirect: true - - service: "host-rewrite-service" - rewriteHostHeader: true - - domain: "https://my.example.com" - - domain: "https://bad.host.example.com" - - domain: "https://www.example.com" - - domain: "https://www.example-redirect.com" - handleInternalRedirect: true - -static_resources: - clusters: - - connect_timeout: 1s - load_assignment: - cluster_name: envoy-control-xds - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: HOST_IP - port_value: HOST_PORT - - endpoint: - address: - socket_address: - address: HOST_IP - port_value: HOST2_PORT - http2_protocol_options: {} - name: envoy-control-xds - - name: envoy-original-destination - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - original_dst_lb_config: - use_http_header: true - connect_timeout: - seconds: 1 - http_protocol_options: - allow_absolute_url: true - - name: local_service - type: STATIC - load_assignment: - cluster_name: local_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: LOCAL_SERVICE_IP - port_value: 5678 - connect_timeout: 1s - - name: this_admin - type: STATIC - load_assignment: - cluster_name: this_admin - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 10000 - connect_timeout: - seconds: 1 From 9c07580cdad299ea9473e8e3ca7ebbb065877de7 Mon Sep 17 00:00:00 2001 From: Marcin Skalski Date: Mon, 11 Oct 2021 08:50:33 +0200 Subject: [PATCH 29/46] removed deprecated api v2 usage (#264) removed deprecated api v2 usage --- docs/features/access_log_filter.md | 2 +- docs/features/permissions.md | 4 +- docs/index.md | 2 +- docs/integrations/envoy.md | 6 +- .../servicemesh/envoycontrol/ControlPlane.kt | 30 +-------- .../servicemesh/envoycontrol/groups/Groups.kt | 11 +--- .../envoycontrol/groups/MetadataNodeGroup.kt | 22 ++----- .../groups/NodeMetadataValidator.kt | 9 +-- .../MetricsDiscoveryServerCallbacks.kt | 1 + .../snapshot/EnvoySnapshotFactory.kt | 16 +---- .../envoycontrol/snapshot/GlobalSnapshot.kt | 14 +--- .../snapshot/SnapshotProperties.kt | 1 - .../resource/clusters/EnvoyClustersFactory.kt | 46 +------------ .../listeners/EnvoyListenersFactory.kt | 34 +++------- .../routes/EnvoyEgressRoutesFactory.kt | 18 ++--- .../envoycontrol/v2/SimpleCacheTest.java | 12 ++-- .../envoycontrol/v3/SimpleCacheTest.java | 14 +++- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 4 +- .../groups/MetadataNodeGroupTest.kt | 27 +------- .../envoycontrol/groups/TestNodeFactory.kt | 65 +++++-------------- .../filters/rbac/RBACFilterFactoryJwtTest.kt | 2 - .../filters/rbac/RBACFilterFactoryTest.kt | 4 -- .../snapshot/debug/SnapshotDebugController.kt | 15 ----- .../envoycontrol/EnvoyControlV3SmokeTest.kt | 4 +- 24 files changed, 75 insertions(+), 288 deletions(-) diff --git a/docs/features/access_log_filter.md b/docs/features/access_log_filter.md index 7a55ecbea..f27c10b36 100644 --- a/docs/features/access_log_filter.md +++ b/docs/features/access_log_filter.md @@ -1,6 +1,6 @@ # Access log filter configuration -Using Envoy's [metadata](https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/base.proto#core-metadata) +Using Envoy's [metadata](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto.html#core-metadata) section you can provide additional configuration to the Control Plane. Configuration provided in `metadata.access_log_filter` will be used to set up an access log filter for `Envoy`. diff --git a/docs/features/permissions.md b/docs/features/permissions.md index 0358c2223..0d46207b6 100644 --- a/docs/features/permissions.md +++ b/docs/features/permissions.md @@ -5,7 +5,7 @@ One of the pillars of Service Mesh is security. Envoy Control provides a simple and fine-grained way to restrict traffic between applications. -Using Envoy's [metadata](https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/base.proto#core-metadata) +Using Envoy's [metadata](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/base.proto.html?highlight=metadata#config-core-v3-metadata) section you can provide additional configuration to the Control Plane. The information provided in `metadata.proxy_settings` section is interpreted by Control Plane and it will create a corresponding configuration for `Envoy`. @@ -40,7 +40,7 @@ metadata: In the `incoming` section this configuration defines access to routes: * `/example` - * using a `path` header matcher (more on this in [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/rbac/v2/rbac.proto#config-rbac-v2-permission)) + * using a `path` header matcher (more on this in [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/rbac/v3/rbac.proto.html#config-rbac-v2-permission)) * using methods `GET` and `DELETE` * to clients `service-first` * all other routes diff --git a/docs/index.md b/docs/index.md index c9e0c5412..a2ce5cf9e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,7 +5,7 @@ Data Plane that is platform agnostic. ## Features -* Exposing data from Service Discovery to [Envoy via gRPC xDS v2 API](integrations/envoy.md) +* Exposing data from Service Discovery to [Envoy via gRPC xDS v3 API](integrations/envoy.md) * Scalable [integration with Consul](integrations/consul.md) * [Multi-DC support](features/multi_dc_support.md) * [Permission management](features/permissions.md) diff --git a/docs/integrations/envoy.md b/docs/integrations/envoy.md index b3b78f80c..2d438d50e 100644 --- a/docs/integrations/envoy.md +++ b/docs/integrations/envoy.md @@ -1,7 +1,7 @@ # Integration with Envoy Envoy Control exposes configuration to Envoy via -[v2 xDS API](https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/v2_overview). +[v3 xDS API](https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/overview). The integration is based on [java-control-plane](https://github.com/envoyproxy/java-control-plane) project. ## Sample Envoy configuration @@ -27,7 +27,7 @@ The behavior is changed so that the `503` status code is returned. ## ADS Support By default, the xDS is used instead of -[Aggregated Discovery Service](https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/v2_overview#aggregated-discovery-service) +[Aggregated Discovery Service](https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/xds_api#aggregated-discovery-service) (ADS). To use ADS for given node put the ``` ads: true @@ -43,7 +43,7 @@ for all clusters with properties [described here](../configuration.md#outlier-de ## Retry policy You can configure -[retry policies](https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/route/route.proto#envoy-api-msg-route-retrypolicy) +[retry policies](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-msg-config-route-v3-retrypolicy) for ingress traffic with properties [described here](../configuration.md#retries). ## Metadata diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt index 4d8ffe67c..c192799dc 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt @@ -165,25 +165,7 @@ class ControlPlane private constructor( ) val grpcServerBuilder = grpcServerBuilder() - if (properties.envoy.snapshot.supportV2Configuration) { - val compositeDiscoveryServerCallbacksV2 = listOf( - CompositeDiscoveryServerCallbacks( - meterRegistry, - buildSnapshotCollectingCallback(cache), - loggingDiscoveryServerCallbacks, - meteredConnectionsCallbacks, - NodeMetadataValidator(properties.envoy.snapshot) - ) - ) - grpcServerBuilder.withV2EnvoyServices( - createV2Server( - compositeDiscoveryServerCallbacksV2, - groupChangeWatcher, - cachedProtoResourcesSerializer - ) - ) - } - grpcServerBuilder.withV3EnvoyServices( + grpcServerBuilder.withEnvoyServices( createV3Server( compositeDiscoveryServerCallbacksV3, groupChangeWatcher, @@ -345,15 +327,7 @@ class ControlPlane private constructor( return this } - private fun NettyServerBuilder.withV2EnvoyServices(discoveryServer: V2DiscoveryServer): NettyServerBuilder { - return this.addService(discoveryServer.aggregatedDiscoveryServiceImpl) - .addService(discoveryServer.clusterDiscoveryServiceImpl) - .addService(discoveryServer.endpointDiscoveryServiceImpl) - .addService(discoveryServer.listenerDiscoveryServiceImpl) - .addService(discoveryServer.routeDiscoveryServiceImpl) - } - - private fun NettyServerBuilder.withV3EnvoyServices(discoveryServer: V3DiscoveryServer): NettyServerBuilder { + private fun NettyServerBuilder.withEnvoyServices(discoveryServer: V3DiscoveryServer): NettyServerBuilder { return this.addService(discoveryServer.aggregatedDiscoveryServiceImpl) .addService(discoveryServer.clusterDiscoveryServiceImpl) .addService(discoveryServer.endpointDiscoveryServiceImpl) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/Groups.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/Groups.kt index a1d69f96c..4d5363fd3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/Groups.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/Groups.kt @@ -5,27 +5,20 @@ sealed class Group { abstract val serviceName: String abstract val proxySettings: ProxySettings abstract val listenersConfig: ListenersConfig? - abstract val version: ResourceVersion -} - -enum class ResourceVersion { - V2, V3 } data class ServicesGroup( override val communicationMode: CommunicationMode, override val serviceName: String = "", override val proxySettings: ProxySettings = ProxySettings(), - override val listenersConfig: ListenersConfig? = null, - override val version: ResourceVersion = ResourceVersion.V3 + override val listenersConfig: ListenersConfig? = null ) : Group() data class AllServicesGroup( override val communicationMode: CommunicationMode, override val serviceName: String = "", override val proxySettings: ProxySettings = ProxySettings(), - override val listenersConfig: ListenersConfig? = null, - override val version: ResourceVersion = ResourceVersion.V3 + override val listenersConfig: ListenersConfig? = null ) : Group() data class ListenersConfig( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt index 4ce3fb958..2d9c5588e 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroup.kt @@ -14,9 +14,6 @@ class MetadataNodeGroup( private val logger by logger() override fun hash(node: NodeV2): Group { - if (properties.supportV2Configuration) { - return createV2Group(node) - } throw V2NotSupportedException() } @@ -135,19 +132,10 @@ class MetadataNodeGroup( } private fun createV3Group(node: NodeV3): Group { - val metadata = NodeMetadata(node.metadata, properties) - return createGroup(metadata, node.id, node.metadata, ResourceVersion.V3) - } - - private fun createV2Group(node: NodeV2): Group { - val metadata = NodeMetadata(node.metadata, properties) - return createGroup(metadata, node.id, node.metadata, ResourceVersion.V2) - } - - private fun createGroup(nodeMetadata: NodeMetadata, id: String, metadata: Struct, version: ResourceVersion): Group { + val nodeMetadata = NodeMetadata(node.metadata, properties) val serviceName = serviceName(nodeMetadata) val proxySettings = proxySettings(nodeMetadata) - val listenersConfig = createListenersConfig(id, metadata) + val listenersConfig = createListenersConfig(node.id, node.metadata) return when { hasAllServicesDependencies(nodeMetadata) -> @@ -155,16 +143,14 @@ class MetadataNodeGroup( nodeMetadata.communicationMode, serviceName, proxySettings, - listenersConfig, - version + listenersConfig ) else -> ServicesGroup( nodeMetadata.communicationMode, serviceName, proxySettings, - listenersConfig, - version + listenersConfig ) } } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt index 3ef88140f..65c0918df 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt @@ -77,14 +77,7 @@ class NodeMetadataValidator( } private fun validateV2Metadata(node: NodeV2) { - if (properties.supportV2Configuration) { - // Some validation logic is executed when NodeMetadata is created. - // This may throw NodeMetadataValidationException - val metadata = NodeMetadata(node.metadata, properties) - validateMetadata(metadata) - } else { - throw V2NotSupportedException() - } + throw V2NotSupportedException() } private fun validateMetadata(metadata: NodeMetadata) { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt index afd51e30a..983b70888 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt @@ -17,6 +17,7 @@ class MetricsDiscoveryServerCallbacks(private val meterRegistry: MeterRegistry) companion object { fun fromTypeUrl(typeUrl: String) = when (typeUrl) { + // TODO_deprecate_v2: do we need this mapping still? Resources.V2.CLUSTER_TYPE_URL -> CDS Resources.V2.ENDPOINT_TYPE_URL -> EDS Resources.V2.LISTENER_TYPE_URL -> LDS diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 754412e79..3eb5e55b1 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -46,13 +46,6 @@ class EnvoySnapshotFactory( ) val securedClusters = clustersFactory.getSecuredClusters(clusters) - var v2Clusters = emptyList() - var v2SecuredClusters = emptyList() - if (properties.supportV2Configuration) { - v2Clusters = clustersFactory.mapToV2Clusters(clusters, communicationMode) - v2SecuredClusters = clustersFactory.mapToV2Clusters(securedClusters, communicationMode) - } - val endpoints: List = endpointsFactory.createLoadAssignment( clusters = clusterConfigurations.keys, multiClusterState = servicesStates @@ -63,9 +56,7 @@ class EnvoySnapshotFactory( clusters = clusters, securedClusters = securedClusters, endpoints = endpoints, - properties = properties.outgoingPermissions, - v2Clusters = v2Clusters, - v2SecuredClusters = v2SecuredClusters + properties = properties.outgoingPermissions ) sample.stop(meterRegistry.timer("snapshot-factory.new-snapshot.time")) @@ -226,7 +217,7 @@ class EnvoySnapshotFactory( egressRouteSpecifications: Collection ): List { return egressRouteSpecifications - .mapNotNull { globalSnapshot.endpoints.resources().get(it.clusterName) } + .mapNotNull { globalSnapshot.endpoints.resources()[it.clusterName] } } private fun newSnapshotForGroup( @@ -243,8 +234,7 @@ class EnvoySnapshotFactory( val routes = listOf( egressRoutesFactory.createEgressRouteConfig( group.serviceName, egressRouteSpecification, - group.listenersConfig?.addUpstreamExternalAddressHeader ?: false, - group.version + group.listenersConfig?.addUpstreamExternalAddressHeader ?: false ), ingressRoutesFactory.createSecuredIngressRouteConfig(group.proxySettings) ) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt index 8ebac1ccf..c7097629c 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt @@ -9,9 +9,7 @@ data class GlobalSnapshot( val allServicesNames: Set, val endpoints: SnapshotResources, val clusterConfigurations: Map, - val securedClusters: SnapshotResources, - val v2Clusters: SnapshotResources, - val v2SecuredClusters: SnapshotResources + val securedClusters: SnapshotResources ) @Suppress("LongParameterList") @@ -20,14 +18,10 @@ internal fun globalSnapshot( endpoints: Iterable, properties: OutgoingPermissionsProperties = OutgoingPermissionsProperties(), clusterConfigurations: Map, - securedClusters: List, - v2Clusters: List, - v2SecuredClusters: List + securedClusters: List ): GlobalSnapshot { val clusters = SnapshotResources.create(clusters, "") val securedClusters = SnapshotResources.create(securedClusters, "") - val v2Clusters = SnapshotResources.create(v2Clusters, "") - val v2SecuredClusters = SnapshotResources.create(v2SecuredClusters, "") val allServicesNames = getClustersForAllServicesGroups(clusters.resources(), properties) val endpoints = SnapshotResources.create(endpoints, "") return GlobalSnapshot( @@ -35,9 +29,7 @@ internal fun globalSnapshot( securedClusters = securedClusters, endpoints = endpoints, allServicesNames = allServicesNames, - clusterConfigurations = clusterConfigurations, - v2Clusters = v2Clusters, - v2SecuredClusters = v2SecuredClusters + clusterConfigurations = clusterConfigurations ) } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index 91cdc1a02..75b93b7b4 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -10,7 +10,6 @@ import java.net.URI import java.time.Duration class SnapshotProperties { - var supportV2Configuration = false var routes = RoutesProperties() var localService = LocalServiceProperties() var egress = EgressProperties() diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index 36c6854e7..397d0f3fe 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -37,7 +37,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.ADS import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.XDS import pl.allegro.tech.servicemesh.envoycontrol.groups.Group -import pl.allegro.tech.servicemesh.envoycontrol.groups.ResourceVersion import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesGroup import pl.allegro.tech.servicemesh.envoycontrol.logger import pl.allegro.tech.servicemesh.envoycontrol.snapshot.ClusterConfiguration @@ -104,13 +103,6 @@ class EnvoyClustersFactory( } } - fun mapToV2Clusters( - clusters: List, - communicationMode: CommunicationMode - ): List { - return clusters.map { mapToV2Cluster(it, communicationMode) } - } - fun getClustersForGroup(group: Group, globalSnapshot: GlobalSnapshot): List = getEdsClustersForGroup(group, globalSnapshot) + getStrictDnsClustersForGroup(group) + clustersForJWT @@ -168,17 +160,9 @@ class EnvoyClustersFactory( private fun getEdsClustersForGroup(group: Group, globalSnapshot: GlobalSnapshot): List { val clusters: Map = if (enableTlsForGroup(group)) { - if (group.version == ResourceVersion.V3) { - globalSnapshot.securedClusters.resources() - } else { - globalSnapshot.v2SecuredClusters.resources() - } + globalSnapshot.securedClusters.resources() } else { - if (group.version == ResourceVersion.V3) { - globalSnapshot.clusters.resources() - } else { - globalSnapshot.v2Clusters.resources() - } + globalSnapshot.clusters.resources() } val clustersForGroup = when (group) { @@ -199,32 +183,6 @@ class EnvoyClustersFactory( private fun shouldAddDynamicForwardProxyCluster(group: Group) = group.proxySettings.outgoing.getDomainPatternDependencies().isNotEmpty() - private fun mapToV2Cluster(cluster: Cluster, communicationMode: CommunicationMode): Cluster { - return cluster.let { - val v2Cluster = Cluster.newBuilder(it) - val v2EdsClusterConfig = Cluster.EdsClusterConfig.newBuilder(v2Cluster.edsClusterConfig) - - when (communicationMode) { - ADS -> v2Cluster.setEdsClusterConfig( - v2EdsClusterConfig.setEdsConfig( - ConfigSource.newBuilder(v2EdsClusterConfig.edsConfig).setResourceApiVersion(ApiVersion.V2) - ) - ).build() - XDS -> - v2Cluster.setEdsClusterConfig( - v2EdsClusterConfig.setEdsConfig( - ConfigSource.newBuilder(v2EdsClusterConfig.edsConfig) - .setResourceApiVersion(ApiVersion.V2) - .setApiConfigSource( - ApiConfigSource.newBuilder(v2EdsClusterConfig.edsConfig.apiConfigSource) - .setTransportApiVersion(ApiVersion.V2) - ) - ) - ).build() - } - } - } - private fun enableTlsForGroup(group: Group): Boolean { return group.listenersConfig?.hasStaticSecretsDefined ?: false } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt index 7b1b82b97..51f78ea14 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt @@ -39,7 +39,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.ADS import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.XDS import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.groups.ListenersConfig -import pl.allegro.tech.servicemesh.envoycontrol.groups.ResourceVersion import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.config.LocalReplyConfigFactory @@ -112,7 +111,6 @@ class EnvoyListenersFactory( TLS("tls") } - val defaultApiConfigSourceV2: ApiConfigSource = apiConfigSource(ApiVersion.V2) val defaultApiConfigSourceV3: ApiConfigSource = apiConfigSource(ApiVersion.V3) fun apiConfigSource(apiVersion: ApiVersion): ApiConfigSource { @@ -230,7 +228,7 @@ class EnvoyListenersFactory( ): Filter { val connectionManagerBuilder = HttpConnectionManager.newBuilder() .setStatPrefix("egress_http") - .setRds(egressRds(group.communicationMode, group.version)) + .setRds(egressRds(group.communicationMode)) .setHttpProtocolOptions(egressHttp1ProtocolOptions()) .setPreserveExternalRequestId(listenersConfig.preserveExternalRequestId) .setGenerateRequestId(boolValue(listenersConfig.generateRequestId)) @@ -289,17 +287,15 @@ class EnvoyListenersFactory( .build() } - private fun egressRds(communicationMode: CommunicationMode, version: ResourceVersion): Rds { + private fun egressRds(communicationMode: CommunicationMode): Rds { val configSource = ConfigSource.newBuilder() .setInitialFetchTimeout(egressRdsInitialFetchTimeout) - if (version == ResourceVersion.V3) { - configSource.setResourceApiVersion(ApiVersion.V3) - } + configSource.resourceApiVersion = ApiVersion.V3 when (communicationMode) { - ADS -> configSource.setAds(AggregatedConfigSource.getDefaultInstance()) - XDS -> setXdsConfigSourceVersion(version, configSource) + ADS -> configSource.ads = AggregatedConfigSource.getDefaultInstance() + XDS -> configSource.apiConfigSource = defaultApiConfigSourceV3 } return Rds.newBuilder() @@ -327,7 +323,7 @@ class EnvoyListenersFactory( .setDelayedCloseTimeout(durationInSeconds(0)) .setCommonHttpProtocolOptions(httpProtocolOptions) .setCodecType(HttpConnectionManager.CodecType.AUTO) - .setRds(ingressRds(group.communicationMode, group.version)) + .setRds(ingressRds(group.communicationMode)) .setHttpProtocolOptions(ingressHttp1ProtocolOptions(group.serviceName)) if (listenersConfig.useRemoteAddress) { @@ -348,17 +344,15 @@ class EnvoyListenersFactory( .build() } - private fun ingressRds(communicationMode: CommunicationMode, version: ResourceVersion): Rds { + private fun ingressRds(communicationMode: CommunicationMode): Rds { val configSource = ConfigSource.newBuilder() .setInitialFetchTimeout(ingressRdsInitialFetchTimeout) - if (version == ResourceVersion.V3) { - configSource.setResourceApiVersion(ApiVersion.V3) - } + configSource.resourceApiVersion = ApiVersion.V3 when (communicationMode) { - ADS -> configSource.setAds(AggregatedConfigSource.getDefaultInstance()) - XDS -> setXdsConfigSourceVersion(version, configSource) + ADS -> configSource.ads = AggregatedConfigSource.getDefaultInstance() + XDS -> configSource.apiConfigSource = defaultApiConfigSourceV3 } return Rds.newBuilder() @@ -367,14 +361,6 @@ class EnvoyListenersFactory( .build() } - private fun setXdsConfigSourceVersion(version: ResourceVersion, configSource: ConfigSource.Builder) { - if (version == ResourceVersion.V3) { - configSource.apiConfigSource = defaultApiConfigSourceV3 - } else { - configSource.apiConfigSource = defaultApiConfigSourceV2 - } - } - fun AccessLog.Builder.buildFromSettings(settings: StatusCodeFilterSettings) { this.setFilter( AccessLogFilter.newBuilder().setStatusCodeFilter( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt index 6f5ac1458..cd89cda46 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt @@ -11,7 +11,6 @@ import io.envoyproxy.envoy.config.route.v3.RouteAction import io.envoyproxy.envoy.config.route.v3.RouteConfiguration import io.envoyproxy.envoy.config.route.v3.RouteMatch import io.envoyproxy.envoy.config.route.v3.VirtualHost -import pl.allegro.tech.servicemesh.envoycontrol.groups.ResourceVersion import pl.allegro.tech.servicemesh.envoycontrol.snapshot.RouteSpecification import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties @@ -69,8 +68,7 @@ class EnvoyEgressRoutesFactory( fun createEgressRouteConfig( serviceName: String, routes: Collection, - addUpstreamAddressHeader: Boolean, - resourceVersion: ResourceVersion = ResourceVersion.V3 + addUpstreamAddressHeader: Boolean ): RouteConfiguration { val virtualHosts = routes .filter { it.routeDomains.isNotEmpty() } @@ -86,7 +84,7 @@ class EnvoyEgressRoutesFactory( .build() ) .setRoute( - createRouteAction(routeSpecification, resourceVersion) + createRouteAction(routeSpecification) ).build() ) .build() @@ -120,10 +118,7 @@ class EnvoyEgressRoutesFactory( return routeConfiguration.build() } - private fun createRouteAction( - routeSpecification: RouteSpecification, - resourceVersion: ResourceVersion - ): RouteAction.Builder { + private fun createRouteAction(routeSpecification: RouteSpecification): RouteAction.Builder { val routeAction = RouteAction.newBuilder() .setCluster(routeSpecification.clusterName) @@ -133,12 +128,7 @@ class EnvoyEgressRoutesFactory( } if (routeSpecification.settings.handleInternalRedirect) { - when (resourceVersion) { - ResourceVersion.V2 -> - routeAction.setInternalRedirectAction(RouteAction.InternalRedirectAction.HANDLE_INTERNAL_REDIRECT) - ResourceVersion.V3 -> - routeAction.internalRedirectPolicy = InternalRedirectPolicy.newBuilder().build() - } + routeAction.internalRedirectPolicy = InternalRedirectPolicy.newBuilder().build() } if (properties.egress.hostHeaderRewriting.enabled && routeSpecification.settings.rewriteHostHeader) { diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java index 2a9885649..1a524dbad 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v2/SimpleCacheTest.java @@ -1,7 +1,4 @@ package pl.allegro.tech.servicemesh.envoycontrol.v2; -import static io.envoyproxy.controlplane.cache.Resources.V2.CLUSTER_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.V2.ROUTE_TYPE_URL; -import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; @@ -20,8 +17,8 @@ import io.envoyproxy.envoy.api.v2.RouteConfiguration; import io.envoyproxy.envoy.api.v2.auth.Secret; import io.envoyproxy.envoy.api.v2.core.Node; -import java.util.Arrays; -import java.util.Collection; +import org.junit.Test; + import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -30,7 +27,10 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import java.util.stream.Collectors; -import org.junit.Test; + +import static io.envoyproxy.controlplane.cache.Resources.V2.CLUSTER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.ROUTE_TYPE_URL; +import static org.assertj.core.api.Assertions.assertThat; /** * This class is copy of {@link io.envoyproxy.controlplane.cache.v2.SimpleCacheTest} diff --git a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java index 6f85b7128..2ab038b62 100644 --- a/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java +++ b/envoy-control-core/src/test/java/pl/allegro/tech/servicemesh/envoycontrol/v3/SimpleCacheTest.java @@ -3,7 +3,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.*; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.StatusInfo; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.cache.v3.Snapshot; import io.envoyproxy.envoy.config.cluster.v3.Cluster; import io.envoyproxy.envoy.config.core.v3.Node; @@ -13,9 +18,12 @@ import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; import org.junit.Test; -import pl.allegro.tech.servicemesh.envoycontrol.v3.SimpleCache; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import java.util.stream.Collectors; diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 3aaec10d2..7536d001d 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -170,9 +170,7 @@ class EnvoySnapshotFactoryTest { return GlobalSnapshot( SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), emptySet(), SnapshotResources.create(emptyList(), "v1"), emptyMap(), - SnapshotResources.create(listOf(cluster), "v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3") + SnapshotResources.create(listOf(cluster), "v3") ) } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroupTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroupTest.kt index 458521f53..e59a0aa46 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroupTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/MetadataNodeGroupTest.kt @@ -305,7 +305,7 @@ class MetadataNodeGroupTest { } @Test - fun `should throw exception when V2 node request configuration and support is disabled`() { + fun `should throw exception when V2 node request configuration is send`() { // given val nodeGroup = MetadataNodeGroup(createSnapshotProperties()) val metadata = createMetadataBuilderWithDefaults() @@ -321,27 +321,6 @@ class MetadataNodeGroupTest { } } - @Test - fun `should assign V2 node to group with listed dependencies when support for V2 is enabled`() { - // given - val nodeGroup = MetadataNodeGroup( - createSnapshotProperties(outgoingPermissions = true, supportV2 = true) - ) - val node = nodeV2(serviceDependencies = setOf("a", "b", "c"), ads = false) - - // when - val group = nodeGroup.hash(node) - - // then - assertThat(group).isEqualTo( - ServicesGroup( - proxySettings = ProxySettings().with(serviceDependencies = serviceDependencies("a", "b", "c")), - communicationMode = XDS, - version = ResourceVersion.V2 - ) - ) - } - private fun createMetadataBuilderWithDefaults(): Struct.Builder? { val metadata = NodeV3.newBuilder().metadataBuilder metadata.putFields("ingress_host", Value.newBuilder().setStringValue("127.0.0.1").build()) @@ -354,11 +333,9 @@ class MetadataNodeGroupTest { private fun createSnapshotProperties( allServicesDependenciesValue: String = "*", outgoingPermissions: Boolean = false, - incomingPermissions: Boolean = false, - supportV2: Boolean = false + incomingPermissions: Boolean = false ): SnapshotProperties { val snapshotProperties = SnapshotProperties() - snapshotProperties.supportV2Configuration = supportV2 snapshotProperties.outgoingPermissions.enabled = outgoingPermissions snapshotProperties.outgoingPermissions.allServicesDependencies.identifier = allServicesDependenciesValue snapshotProperties.incomingPermissions.enabled = incomingPermissions diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/TestNodeFactory.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/TestNodeFactory.kt index 04518ca4b..8fc58afcd 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/TestNodeFactory.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/TestNodeFactory.kt @@ -1,4 +1,5 @@ @file:Suppress("MatchingDeclarationName") + package pl.allegro.tech.servicemesh.envoycontrol.groups import com.google.protobuf.ListValue @@ -6,53 +7,8 @@ import com.google.protobuf.NullValue import com.google.protobuf.Struct import com.google.protobuf.Value import com.google.protobuf.util.Durations -import io.envoyproxy.envoy.api.v2.core.Node as NodeV2 import io.envoyproxy.envoy.config.core.v3.Node as NodeV3 -fun nodeV2( - serviceDependencies: Set = emptySet(), - ads: Boolean? = null, - serviceName: String? = null, - incomingSettings: Boolean = false, - clients: List = listOf("client1"), - idleTimeout: String? = null, - responseTimeout: String? = null, - connectionIdleTimeout: String? = null, - healthCheckPath: String? = null, - healthCheckClusterName: String? = null -): NodeV2 { - val meta = NodeV2.newBuilder().metadataBuilder - - serviceName?.let { - meta.putFields("service_name", string(serviceName)) - } - - ads?.let { - meta.putFields("ads", Value.newBuilder().setBoolValue(ads).build()) - } - - if (incomingSettings || serviceDependencies.isNotEmpty()) { - meta.putFields( - "proxy_settings", - proxySettingsProto( - path = "/endpoint", - clients = clients, - serviceDependencies = serviceDependencies, - incomingSettings = incomingSettings, - idleTimeout = idleTimeout, - responseTimeout = responseTimeout, - connectionIdleTimeout = connectionIdleTimeout, - healthCheckPath = healthCheckPath, - healthCheckClusterName = healthCheckClusterName - ) - ) - } - - return NodeV2.newBuilder() - .setMetadata(meta) - .build() -} - fun nodeV3( serviceDependencies: Set = emptySet(), ads: Boolean? = null, @@ -113,10 +69,12 @@ fun ProxySettings.with( serviceDependencies: Set = emptySet(), domainDependencies: Set = emptySet(), allServicesDependencies: Boolean = false, - defaultServiceSettings: DependencySettings = DependencySettings(timeoutPolicy = Outgoing.TimeoutPolicy( - Durations.fromSeconds(120), - Durations.fromSeconds(120) - )) + defaultServiceSettings: DependencySettings = DependencySettings( + timeoutPolicy = Outgoing.TimeoutPolicy( + Durations.fromSeconds(120), + Durations.fromSeconds(120) + ) + ) ): ProxySettings { return copy( outgoing = Outgoing( @@ -180,7 +138,14 @@ fun proxySettingsProto( } class OutgoingDependenciesProtoScope { - class Dependency(val service: String? = null, val domain: String? = null, val domainPattern: String? = null, val idleTimeout: String? = null, val requestTimeout: String? = null, val handleInternalRedirect: Boolean? = null) + class Dependency( + val service: String? = null, + val domain: String? = null, + val domainPattern: String? = null, + val idleTimeout: String? = null, + val requestTimeout: String? = null, + val handleInternalRedirect: Boolean? = null + ) val dependencies = mutableListOf() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt index 50699dc09..52cc3cc06 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt @@ -43,8 +43,6 @@ internal class RBACFilterFactoryJwtTest : RBACFilterFactoryTestUtils { setOf(), SnapshotResources.create(listOf(), ""), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), SnapshotResources.create(listOf(), "") ) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryTest.kt index f9f3a8502..c6d115cb4 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryTest.kt @@ -90,8 +90,6 @@ internal class RBACFilterFactoryTest : RBACFilterFactoryTestUtils { setOf(), SnapshotResources.create(listOf(), ""), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), SnapshotResources.create(listOf(), "") ) @@ -114,8 +112,6 @@ internal class RBACFilterFactoryTest : RBACFilterFactoryTestUtils { setOf(), SnapshotResources.create(listOf(clusterLoadAssignment), ""), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), SnapshotResources.create(listOf(), "") ) diff --git a/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugController.kt b/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugController.kt index f759f3fa9..2a1c68f89 100644 --- a/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugController.kt +++ b/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugController.kt @@ -34,7 +34,6 @@ import org.springframework.web.bind.annotation.RestController import pl.allegro.tech.servicemesh.envoycontrol.ControlPlane import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotUpdater -import io.envoyproxy.envoy.api.v2.core.Node as NodeV2 import io.envoyproxy.envoy.config.core.v3.Node as NodeV3 import io.envoyproxy.envoy.extensions.filters.http.rbac.v3.RBAC as RBACFilter @@ -49,20 +48,6 @@ class SnapshotDebugController(controlPlane: ControlPlane) { * It contains the versions of XDS resources and the contents for a provided node JSON * extracted from Envoy's config_dump endpoint. */ - @PostMapping(value = ["/snapshot"], consumes = ["application/json"], produces = ["application/v2+json"]) - fun snapshot(@RequestBody node: NodeV2): ResponseEntity { - val nodeHash = nodeGroup.hash(node) - val snapshot = cache.getSnapshot(nodeHash) - return if (snapshot == null) { - throw SnapshotNotFoundException() - } else { - ResponseEntity( - SnapshotDebugInfo(snapshot), - HttpStatus.OK - ) - } - } - @PostMapping( value = ["/snapshot"], consumes = ["application/json"], diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt index 773cd0621..07d41b67f 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlV3SmokeTest.kt @@ -22,9 +22,7 @@ class EnvoyControlV3SmokeTest { @JvmField @RegisterExtension - val envoyControl = EnvoyControlExtension(consul, mapOf( - "envoy-control.envoy.snapshot.support-v2-configuration" to false - )) + val envoyControl = EnvoyControlExtension(consul) @JvmField @RegisterExtension From e801195689e9b343ae9e428fdf5c7bbc35d8afa2 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Tue, 9 Nov 2021 13:25:15 +0100 Subject: [PATCH 30/46] jcenter removed --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 1b92a26a1..a746c77aa 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,6 @@ allprojects { project.version = scmVersion.version repositories { - jcenter() mavenCentral() maven { url "https://oss.sonatype.org/content/repositories/snapshots" From 718a74bdba0f955e71b12d8eb271bf06a3623034 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Tue, 9 Nov 2021 13:45:37 +0100 Subject: [PATCH 31/46] update detekt version and fix warnings --- build.gradle | 4 ++-- .../servicemesh/envoycontrol/ControlPlane.kt | 16 +--------------- .../envoycontrol/groups/NodeMetadataValidator.kt | 4 ++-- .../envoycontrol/snapshot/SnapshotUpdater.kt | 4 +--- .../envoycontrol/groups/NodeMetadataTest.kt | 8 -------- .../envoycontrol/config/service/EchoContainer.kt | 4 ++-- 6 files changed, 8 insertions(+), 32 deletions(-) diff --git a/build.gradle b/build.gradle index a746c77aa..9764b8249 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id 'org.jetbrains.kotlin.plugin.allopen' version '1.3.72' id "org.jlleitschuh.gradle.ktlint" version "10.2.0" id "org.jlleitschuh.gradle.ktlint-idea" version "10.2.0" - id "io.gitlab.arturbosch.detekt" version "1.13.1" + id "io.gitlab.arturbosch.detekt" version "1.18.0" id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' } @@ -188,7 +188,7 @@ subprojects { } detekt { - toolVersion = "1.13.1" + toolVersion = "1.18.0" input = files("src/main/kotlin", "src/test/kotlin") config = files("$rootDir/config/detekt/detekt.yml") buildUponDefaultConfig = true diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt index c192799dc..69b11dddf 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt @@ -183,8 +183,7 @@ class ControlPlane private constructor( groupSnapshotScheduler, groupChangeWatcher.onGroupAdded(), meterRegistry, - snapshotsVersions, - envoyHttpFilters + snapshotsVersions ), nodeGroup, cache, @@ -217,19 +216,6 @@ class ControlPlane private constructor( ) } - private fun createV2Server( - compositeDiscoveryServerCallbacks: List, - groupChangeWatcher: GroupChangeWatcher, - cachedProtoResourcesSerializer: CachedProtoResourcesSerializer - ): V2DiscoveryServer { - return V2DiscoveryServer( - compositeDiscoveryServerCallbacks, - groupChangeWatcher, - executorGroup, - cachedProtoResourcesSerializer - ) - } - private fun buildSnapshotCollectingCallback( cache: SimpleCache ): SnapshotCollectingCallback { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt index 28200f623..de52dd403 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt @@ -62,7 +62,7 @@ class NodeMetadataValidator( } override fun onV2StreamRequest(streamId: Long, request: DiscoveryRequestV2?) { - request?.node?.let { validateV2Metadata(it) } + request?.node?.let { validateV2Metadata() } } override fun onStreamResponse( @@ -80,7 +80,7 @@ class NodeMetadataValidator( validateMetadata(metadata) } - private fun validateV2Metadata(node: NodeV2) { + private fun validateV2Metadata() { throw V2NotSupportedException() } diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt index 78211e910..b7c1baeca 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt @@ -28,9 +28,7 @@ class SnapshotUpdater( private val groupSnapshotScheduler: ParallelizableScheduler, private val onGroupAdded: Flux>, private val meterRegistry: MeterRegistry, - private val versions: SnapshotsVersions, - envoyHttpFilters: EnvoyHttpFilters = EnvoyHttpFilters.emptyFilters, - serviceTagFilter: ServiceTagMetadataGenerator = ServiceTagMetadataGenerator(properties.routing.serviceTags) + private val versions: SnapshotsVersions ) { companion object { private val logger by logger() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataTest.kt index 273b098d2..9252cd37c 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataTest.kt @@ -48,14 +48,6 @@ class NodeMetadataTest { ) } - private val defaultDependencySettings = DependencySettings( - handleInternalRedirect = false, - timeoutPolicy = Outgoing.TimeoutPolicy( - idleTimeout = Durations.fromMillis(1000), - requestTimeout = Durations.fromMillis(3000) - ) - ) - private fun snapshotProperties( allServicesDependenciesIdentifier: String = "*", handleInternalRedirect: Boolean = false, diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt index 048673394..b1f180924 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt @@ -3,7 +3,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.config.service import org.testcontainers.containers.Network import org.testcontainers.containers.wait.strategy.Wait import pl.allegro.tech.servicemesh.envoycontrol.config.testcontainers.GenericContainer -import java.util.UUID +import java.util.* class EchoContainer : GenericContainer("hashicorp/http-echo:latest"), ServiceContainer { @@ -13,7 +13,7 @@ class EchoContainer : GenericContainer("hashicorp/http-echo:lates super.configure() withExposedPorts(PORT) withNetwork(Network.SHARED) - withCommand(String.format("-text=%s", response)) + withCommand(String.format(Locale.getDefault(),"-text=%s", response)) waitingFor(Wait.forHttp("/").forStatusCode(200)) } From 6e14be832c04491e8640860d01f96df9d0ff42c2 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Tue, 9 Nov 2021 13:56:12 +0100 Subject: [PATCH 32/46] ktlint fix --- .../pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt | 1 - .../servicemesh/envoycontrol/groups/NodeMetadataValidator.kt | 1 - .../servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt | 2 -- .../servicemesh/envoycontrol/config/service/EchoContainer.kt | 5 +++-- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt index 69b11dddf..bc51a874b 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt @@ -5,7 +5,6 @@ import io.envoyproxy.controlplane.cache.SnapshotCache import io.envoyproxy.controlplane.cache.v3.Snapshot import io.envoyproxy.controlplane.server.DefaultExecutorGroup import io.envoyproxy.controlplane.server.ExecutorGroup -import io.envoyproxy.controlplane.server.V2DiscoveryServer import io.envoyproxy.controlplane.server.V3DiscoveryServer import io.envoyproxy.controlplane.server.callback.SnapshotCollectingCallback import io.grpc.Server diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt index de52dd403..f55761a18 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt @@ -8,7 +8,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.logger import pl.allegro.tech.servicemesh.envoycontrol.protocol.HttpMethod import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import io.envoyproxy.envoy.api.v2.DiscoveryRequest as DiscoveryRequestV2 -import io.envoyproxy.envoy.api.v2.core.Node as NodeV2 import io.envoyproxy.envoy.config.core.v3.Node as NodeV3 import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest as DiscoveryRequestV3 diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt index b7c1baeca..5f5c914b2 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotUpdater.kt @@ -9,8 +9,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode.XDS import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.logger import pl.allegro.tech.servicemesh.envoycontrol.services.MultiClusterState -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.EnvoyHttpFilters -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.ServiceTagMetadataGenerator import pl.allegro.tech.servicemesh.envoycontrol.utils.ParallelizableScheduler import pl.allegro.tech.servicemesh.envoycontrol.utils.doOnNextScheduledOn import pl.allegro.tech.servicemesh.envoycontrol.utils.measureBuffer diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt index b1f180924..9c1aa15f1 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/service/EchoContainer.kt @@ -3,7 +3,8 @@ package pl.allegro.tech.servicemesh.envoycontrol.config.service import org.testcontainers.containers.Network import org.testcontainers.containers.wait.strategy.Wait import pl.allegro.tech.servicemesh.envoycontrol.config.testcontainers.GenericContainer -import java.util.* +import java.util.UUID +import java.util.Locale class EchoContainer : GenericContainer("hashicorp/http-echo:latest"), ServiceContainer { @@ -13,7 +14,7 @@ class EchoContainer : GenericContainer("hashicorp/http-echo:lates super.configure() withExposedPorts(PORT) withNetwork(Network.SHARED) - withCommand(String.format(Locale.getDefault(),"-text=%s", response)) + withCommand(String.format(Locale.getDefault(), "-text=%s", response)) waitingFor(Wait.forHttp("/").forStatusCode(200)) } From 53f1e148da755ae665e259a0afa64380e015f60e Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Thu, 18 Nov 2021 15:00:14 +0100 Subject: [PATCH 33/46] changes after merge --- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 15 +++++++++------ .../filters/rbac/RBACFilterFactoryJwtTest.kt | 12 +++++++----- .../config/EnvoyControlTestConfiguration.kt | 1 + 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 149ebe629..ac51bb7ad 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -8,8 +8,8 @@ import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource import io.envoyproxy.envoy.config.core.v3.ConfigSource import io.envoyproxy.envoy.config.core.v3.HttpProtocolOptions import io.envoyproxy.envoy.config.core.v3.Metadata -import io.envoyproxy.envoy.config.listener.v3.Listener import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment +import io.envoyproxy.envoy.config.listener.v3.Listener import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.simple.SimpleMeterRegistry import org.assertj.core.api.Assertions.assertThat @@ -28,6 +28,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.EnvoySnapshotFactory import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotsVersions +import pl.allegro.tech.servicemesh.envoycontrol.snapshot.globalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.outgoingTimeoutPolicy import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory @@ -280,11 +281,13 @@ class EnvoySnapshotFactoryTest { private fun createGlobalSnapshot(vararg clusters: Cluster): GlobalSnapshot { return GlobalSnapshot( - SnapshotResources.create(clusters.toList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), clusters.map { it.name }.toSet(), - SnapshotResources.create(emptyList(), "v1"), emptyMap(), - SnapshotResources.create(clusters.toList(), "v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), - SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3") + SnapshotResources.create(clusters.toList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources(), + clusters.map { it.name }.toSet(), + SnapshotResources.create(emptyList(), "v1").resources(), + emptyMap(), + SnapshotResources.create(clusters.toList(), "v3").resources(), + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources(), + SnapshotResources.create(emptyList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources() ) } diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt index 672b544fc..947a44ea9 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/rbac/RBACFilterFactoryJwtTest.kt @@ -1,6 +1,8 @@ package pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.listeners.filters.rbac import io.envoyproxy.controlplane.cache.SnapshotResources +import io.envoyproxy.envoy.config.cluster.v3.Cluster +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment import org.assertj.core.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest @@ -39,13 +41,13 @@ internal class RBACFilterFactoryJwtTest : RBACFilterFactoryTestUtils { ) val snapshot = GlobalSnapshot( - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "").resources(), setOf(), - SnapshotResources.create(listOf(), ""), + SnapshotResources.create(listOf(), "").resources(), mapOf(), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), ""), - SnapshotResources.create(listOf(), "") + SnapshotResources.create(listOf(), "").resources(), + SnapshotResources.create(listOf(), "").resources(), + SnapshotResources.create(listOf(), "").resources() ) @ParameterizedTest(name = "should generate RBAC rules for {arguments} OAuth Policy") diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt index f71409b4a..43ac9d029 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/EnvoyControlTestConfiguration.kt @@ -1,6 +1,7 @@ package pl.allegro.tech.servicemesh.envoycontrol.config import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody From 3c33bb3b3d90a940865320a3ea90b6c3827f4a44 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Fri, 19 Nov 2021 12:21:33 +0100 Subject: [PATCH 34/46] changes after merge --- .../servicemesh/envoycontrol/groups/NodeMetadataValidator.kt | 2 +- .../snapshot/resource/clusters/EnvoyClustersFactory.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt index fc6e878c2..80984ec92 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/groups/NodeMetadataValidator.kt @@ -63,7 +63,7 @@ class NodeMetadataValidator( } override fun onV2StreamDeltaRequest(streamId: Long, request: DeltaDiscoveryRequest?) { - request?.node?.let { validateV2Metadata(it) } + request?.node?.let { validateV2Metadata() } } override fun onV3StreamDeltaRequest( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index b2d093200..ddac4857d 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -162,9 +162,9 @@ class EnvoyClustersFactory( private fun getEdsClustersForGroup(group: Group, globalSnapshot: GlobalSnapshot): List { val clusters: Map = if (enableTlsForGroup(group)) { - globalSnapshot.securedClusters.resources() + globalSnapshot.securedClusters } else { - globalSnapshot.clusters.resources() + globalSnapshot.clusters } val serviceDependencies = group.proxySettings.outgoing.getServiceDependencies().associateBy { it.service } From b20bfc5c5ab9c3f84c7c4b040a614e3d05ca1814 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Thu, 10 Feb 2022 08:47:56 +0100 Subject: [PATCH 35/46] resolve merge conflicts --- .../snapshot/EnvoySnapshotFactory.kt | 3 +- .../envoycontrol/snapshot/GlobalSnapshot.kt | 8 +-- .../resource/clusters/EnvoyClustersFactory.kt | 2 +- .../listeners/EnvoyListenersFactory.kt | 67 ------------------- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 4 +- 5 files changed, 9 insertions(+), 75 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 64751a278..15bea5786 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -12,6 +12,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.groups.AllServicesGroup import pl.allegro.tech.servicemesh.envoycontrol.groups.CommunicationMode import pl.allegro.tech.servicemesh.envoycontrol.groups.DependencySettings import pl.allegro.tech.servicemesh.envoycontrol.groups.Group +import pl.allegro.tech.servicemesh.envoycontrol.groups.IncomingRateLimitEndpoint import pl.allegro.tech.servicemesh.envoycontrol.groups.ServicesGroup import pl.allegro.tech.servicemesh.envoycontrol.services.MultiClusterState import pl.allegro.tech.servicemesh.envoycontrol.services.ServiceInstance @@ -222,7 +223,7 @@ class EnvoySnapshotFactory( if (rateLimitEndpoints.isNotEmpty()) listOf(properties.rateLimit.serviceName) else emptyList() val allClusters = egressRouteClusters + rateLimitClusters - return allClusters.mapNotNull { name -> globalSnapshot.endpoints.resources()[name] } + return allClusters.mapNotNull { name -> globalSnapshot.endpoints[name] } } private fun newSnapshotForGroup( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt index 8cfcefe58..ce352a5a0 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/GlobalSnapshot.kt @@ -14,11 +14,11 @@ data class GlobalSnapshot( @Suppress("LongParameterList") internal fun globalSnapshot( - clusters: Iterable, - endpoints: Iterable, + clusters: Iterable = emptyList(), + endpoints: Iterable = emptyList(), properties: OutgoingPermissionsProperties = OutgoingPermissionsProperties(), - clusterConfigurations: Map, - securedClusters: List + clusterConfigurations: Map = emptyMap(), + securedClusters: List = emptyList() ): GlobalSnapshot { val clusters = SnapshotResources.create(clusters, "").resources() val securedClusters = SnapshotResources.create(securedClusters, "").resources() diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index c418c0665..2670370ac 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -164,7 +164,7 @@ class EnvoyClustersFactory( private fun getRateLimitClusterForGroup(group: Group, globalSnapshot: GlobalSnapshot): List { if (group.proxySettings.incoming.rateLimitEndpoints.containsGlobalRateLimits()) { - val cluster = globalSnapshot.clusters.resources()[properties.rateLimit.serviceName] + val cluster = globalSnapshot.clusters[properties.rateLimit.serviceName] if (cluster != null) { return listOf(Cluster.newBuilder(cluster).build()) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt index f579c0434..cf54d2d09 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/EnvoyListenersFactory.kt @@ -105,21 +105,6 @@ class EnvoyListenersFactory( TLS("tls") } - val defaultApiConfigSourceV3: ApiConfigSource = apiConfigSource(ApiVersion.V3) - - fun apiConfigSource(apiVersion: ApiVersion): ApiConfigSource { - return ApiConfigSource.newBuilder() - .setApiType(ApiConfigSource.ApiType.GRPC) - .setTransportApiVersion(apiVersion) - .addGrpcServices( - GrpcService.newBuilder() - .setEnvoyGrpc( - GrpcService.EnvoyGrpc.newBuilder() - .setClusterName("envoy-control-xds") - ) - ).build() - } - fun createListeners(group: Group, globalSnapshot: GlobalSnapshot): List { if (group.listenersConfig == null) { return listOf() @@ -261,42 +246,6 @@ class EnvoyListenersFactory( .addFilters(createIngressFilter(group, globalSnapshot, statPrefix)) } - private fun createSecuredIngressFilterChain( - group: Group, - listenersConfig: ListenersConfig, - globalSnapshot: GlobalSnapshot - ): FilterChain.Builder { - val filterChain = createIngressFilterChain(group, listenersConfig, globalSnapshot, TransportProtocol.TLS) - filterChain.setTransportSocket(downstreamTlsTransportSocket) - return filterChain - } - - private fun createEgressFilterChain( - group: Group, - listenersConfig: ListenersConfig, - globalSnapshot: GlobalSnapshot - ): FilterChain { - return FilterChain.newBuilder() - .addFilters(createEgressFilter(group, listenersConfig, globalSnapshot)) - .build() - } - - private fun createEgressFilter( - group: Group, - listenersConfig: ListenersConfig, - globalSnapshot: GlobalSnapshot - ): Filter { - val connectionManagerBuilder = HttpConnectionManager.newBuilder() - .setStatPrefix("egress_http") - .setRds(egressRds(group.communicationMode)) - .setHttpProtocolOptions(egressHttp1ProtocolOptions()) - .setPreserveExternalRequestId(listenersConfig.preserveExternalRequestId) - .setGenerateRequestId(boolValue(listenersConfig.generateRequestId)) - - if (group.proxySettings.outgoing.getDomainPatternDependencies().isNotEmpty()) { - connectionManagerBuilder.addHttpFilters(dynamicForwardProxyFilter) - } - private fun createIngressFilter( group: Group, globalSnapshot: GlobalSnapshot, @@ -346,22 +295,6 @@ class EnvoyListenersFactory( .build() } } - private fun egressHttp1ProtocolOptions(): Http1ProtocolOptions? { - return Http1ProtocolOptions.newBuilder() - .setAllowAbsoluteUrl(boolValue(true)) - .build() - } - - private fun egressRds(communicationMode: CommunicationMode): Rds { - val configSource = ConfigSource.newBuilder() - .setInitialFetchTimeout(egressRdsInitialFetchTimeout) - - configSource.resourceApiVersion = ApiVersion.V3 - - when (communicationMode) { - ADS -> configSource.ads = AggregatedConfigSource.getDefaultInstance() - XDS -> configSource.apiConfigSource = defaultApiConfigSourceV3 - } private fun createHttpProxyFilterChainForDomains( group: Group, diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 46fa170df..c13ee17ba 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -260,10 +260,10 @@ class EnvoySnapshotFactoryTest { } private fun GlobalSnapshot.withEndpoint(clusterName: String): GlobalSnapshot = copy( - endpoints = SnapshotResources.create(listOf(ClusterLoadAssignment.newBuilder() + endpoints = SnapshotResources.create(listOf(ClusterLoadAssignment.newBuilder() .setClusterName(clusterName) .build() - ), "v1")) + ), "v1").resources()) private fun createServicesGroup( mode: CommunicationMode = CommunicationMode.XDS, From bdcdd3edd1fc6c1c534140b1bffd44cd6320554d Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Thu, 10 Feb 2022 08:49:00 +0100 Subject: [PATCH 36/46] refactor delta xds --- build.gradle | 2 +- .../envoycontrol/GroupCacheStatusInfo.java | 53 --- .../servicemesh/envoycontrol/SimpleCache.java | 337 ++++++++---------- 3 files changed, 141 insertions(+), 251 deletions(-) delete mode 100644 envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java diff --git a/build.gradle b/build.gradle index 1f53cff6c..0b1a2402e 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ allprojects { project.ext.versions = [ kotlin : '1.3.72', - java_controlplane : '0.1.29-delta-xds-slonka-hash-bytes-SNAPSHOT', + java_controlplane : '0.1.31-delta-xds-slonka-hash-bytes-SNAPSHOT', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', diff --git a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java deleted file mode 100644 index 3c10fc8c3..000000000 --- a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/GroupCacheStatusInfo.java +++ /dev/null @@ -1,53 +0,0 @@ -package pl.allegro.tech.servicemesh.envoycontrol; - -import io.envoyproxy.controlplane.cache.CacheStatusInfo; -import io.envoyproxy.controlplane.cache.StatusInfo; -import java.util.ArrayList; -import java.util.Collection; -import javax.annotation.concurrent.ThreadSafe; - -/** - * {@code GroupCacheStatusInfo} provides an implementation of {@link StatusInfo} for a group of {@link CacheStatusInfo}. - */ -@ThreadSafe -class GroupCacheStatusInfo implements StatusInfo { - private final Collection> statuses; - - public GroupCacheStatusInfo(Collection> statuses) { - this.statuses = new ArrayList<>(statuses); - } - - /** - * {@inheritDoc} - */ - @Override - public long lastWatchRequestTime() { - return statuses.stream().mapToLong(CacheStatusInfo::lastWatchRequestTime).max().orElse(0); - } - - @Override - public long lastDeltaWatchRequestTime() { - return statuses.stream().mapToLong(CacheStatusInfo::lastDeltaWatchRequestTime).max().orElse(0); - } - - /** - * {@inheritDoc} - */ - @Override - public T nodeGroup() { - return statuses.stream().map(CacheStatusInfo::nodeGroup).findFirst().orElse(null); - } - - /** - * {@inheritDoc} - */ - @Override - public int numWatches() { - return statuses.stream().mapToInt(CacheStatusInfo::numWatches).sum(); - } - - @Override - public int numDeltaWatches() { - return statuses.stream().mapToInt(CacheStatusInfo::numDeltaWatches).sum(); - } -} diff --git a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java index f266336c5..e25edfecd 100644 --- a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java +++ b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java @@ -6,20 +6,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.protobuf.Message; -import io.envoyproxy.controlplane.cache.CacheStatusInfo; -import io.envoyproxy.controlplane.cache.DeltaResponse; -import io.envoyproxy.controlplane.cache.DeltaWatch; -import io.envoyproxy.controlplane.cache.DeltaXdsRequest; -import io.envoyproxy.controlplane.cache.NodeGroup; -import io.envoyproxy.controlplane.cache.Resources; -import io.envoyproxy.controlplane.cache.Response; -import io.envoyproxy.controlplane.cache.Snapshot; -import io.envoyproxy.controlplane.cache.SnapshotCache; -import io.envoyproxy.controlplane.cache.StatusInfo; -import io.envoyproxy.controlplane.cache.VersionedResource; -import io.envoyproxy.controlplane.cache.Watch; -import io.envoyproxy.controlplane.cache.WatchCancelledException; -import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.cache.*; +import io.envoyproxy.controlplane.cache.GroupCacheStatusInfo; import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +29,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.function.Function; +import java.util.stream.Stream; import static io.envoyproxy.controlplane.cache.Resources.RESOURCE_TYPES_IN_ORDER; @@ -60,7 +49,7 @@ public class SimpleCache implements SnapshotCache { @GuardedBy("lock") private final Map snapshots = new HashMap<>(); - private final ConcurrentMap>> statuses = new ConcurrentHashMap<>(); + private final CacheStatusInfoAggregator statuses = new CacheStatusInfoAggregator<>(); private AtomicLong watchCount = new AtomicLong(); @@ -83,10 +72,9 @@ public boolean clearSnapshot(T group) { // we take a writeLock to prevent watches from being created writeLock.lock(); try { - Map> status = statuses.get(group); // If we don't know about this group, do nothing. - if (status != null && status.values().stream().mapToLong(CacheStatusInfo::numWatches).sum() > 0) { + if (statuses.hasStatuses(group)) { LOGGER.warn("tried to clear snapshot for group with existing watches, group={}", group); return false; @@ -101,6 +89,14 @@ public boolean clearSnapshot(T group) { } } + public Watch createWatch( + boolean ads, + XdsRequest request, + Set knownResourceNames, + Consumer responseConsumer) { + return createWatch(ads, request, knownResourceNames, responseConsumer, false); + } + /** * {@inheritDoc} */ @@ -111,7 +107,7 @@ public Watch createWatch( Set knownResourceNames, Consumer responseConsumer, boolean hasClusterChanged) { - Resources.ResourceType requestResourceType = request.getResourceType(); + Resources.ResourceType requestResourceType = request.getResourceType(); Preconditions.checkNotNull(requestResourceType, "unsupported type URL %s", request.getTypeUrl()); T group; @@ -125,8 +121,7 @@ public Watch createWatch( // doesn't conflict readLock.lock(); try { - CacheStatusInfo status = statuses.computeIfAbsent(group, g -> new ConcurrentHashMap<>()) - .computeIfAbsent(requestResourceType, s -> new CacheStatusInfo<>(group)); + CacheStatusInfo status = statuses.getOrAddStatusInfo(group, requestResourceType); status.setLastWatchRequestTime(System.currentTimeMillis()); U snapshot = snapshots.get(group); @@ -160,20 +155,7 @@ public Watch createWatch( // If the requested version is up-to-date or missing a response, leave an open watch. if (snapshot == null || request.getVersionInfo().equals(version)) { - long watchId = watchCount.incrementAndGet(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("open watch {} for {}[{}] from node {} for version {}", - watchId, - request.getTypeUrl(), - String.join(", ", request.getResourceNamesList()), - group, - request.getVersionInfo()); - } - - status.setWatch(watchId, watch); - - watch.setStop(() -> status.removeWatch(watchId)); + openWatch(status, watch, request.getTypeUrl(), request.getResourceNamesList(), group, request.getVersionInfo()); return watch; } @@ -182,20 +164,7 @@ public Watch createWatch( boolean responded = respond(watch, snapshot, group); if (!responded) { - long watchId = watchCount.incrementAndGet(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("did not respond immediately, leaving open watch {} for {}[{}] from node {} for version {}", - watchId, - request.getTypeUrl(), - String.join(", ", request.getResourceNamesList()), - group, - request.getVersionInfo()); - } - - status.setWatch(watchId, watch); - - watch.setStop(() -> status.removeWatch(watchId)); + openWatch(status, watch, request.getTypeUrl(), request.getResourceNamesList(), group, request.getVersionInfo()); } return watch; @@ -231,8 +200,8 @@ public DeltaWatch createDeltaWatch( // doesn't conflict readLock.lock(); try { - CacheStatusInfo status = statuses.computeIfAbsent(group, g -> new ConcurrentHashMap<>()) - .computeIfAbsent(requestResourceType, s -> new CacheStatusInfo<>(group)); + DeltaCacheStatusInfo status = statuses.getOrAddDeltaStatusInfo(group, requestResourceType); + status.setLastWatchRequestTime(System.currentTimeMillis()); U snapshot = snapshots.get(group); @@ -248,16 +217,7 @@ public DeltaWatch createDeltaWatch( // If no snapshot, leave an open watch. if (snapshot == null) { - long watchId = setDeltaWatch(status, watch); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("open watch {} for {}[{}] from node {} for version {}", - watchId, - request.getTypeUrl(), - String.join(", ", watch.trackedResources().keySet()), - group, - requesterVersion); - } - + openWatch(status, watch, request.getTypeUrl(), watch.trackedResources().keySet(), group, requesterVersion); return watch; } @@ -277,61 +237,29 @@ public DeltaWatch createDeltaWatch( Collections.emptyList(), version, group); - if (responseState.equals(ResponseState.RESPONDED) || responseState.equals(ResponseState.CANCELLED)) { + if (responseState.isFinished()) { return watch; } } else if (hasClusterChanged && requestResourceType.equals(Resources.ResourceType.ENDPOINT)) { - Map> snapshotResources = snapshot.versionedResources(request.getResourceType()); - List removedResources = findRemovedResources(watch, - snapshotResources); - Map> changedResources = findChangedResources(watch, snapshotResources); - ResponseState responseState = respondDelta( - watch, - changedResources, - removedResources, - version, - group); - if (responseState.equals(ResponseState.RESPONDED) || responseState.equals(ResponseState.CANCELLED)) { + ResponseState responseState = respondDelta(request, watch, snapshot, version, group); + if (responseState.isFinished()) { return watch; } } - long watchId = setDeltaWatch(status, watch); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("open watch {} for {}[{}] from node {} for version {}", - watchId, - request.getTypeUrl(), - String.join(", ", watch.trackedResources().keySet()), - group, - requesterVersion); - } + openWatch(status, watch, request.getTypeUrl(), watch.trackedResources().keySet(), group, requesterVersion); return watch; } // Otherwise, version is different, the watch may be responded immediately - Map> snapshotResources = snapshot.versionedResources(request.getResourceType()); - List removedResources = findRemovedResources(watch, - snapshotResources); - Map> changedResources = findChangedResources(watch, snapshotResources); - ResponseState responseState = respondDelta(watch, - changedResources, - removedResources, - version, - group); - if (responseState.equals(ResponseState.RESPONDED) || responseState.equals(ResponseState.CANCELLED)) { + ResponseState responseState = respondDelta(request, watch, snapshot, version, group); + + if (responseState.isFinished()) { return watch; } - long watchId = setDeltaWatch(status, watch); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("did not respond immediately, leaving open watch {} for {}[{}] from node {} for version {}", - watchId, - request.getTypeUrl(), - String.join(", ", watch.trackedResources().keySet()), - group, - requesterVersion); - } + openWatch(status, watch, request.getTypeUrl(), watch.trackedResources().keySet(), group, requesterVersion); return watch; } finally { @@ -339,11 +267,23 @@ public DeltaWatch createDeltaWatch( } } - private long setDeltaWatch(CacheStatusInfo status, DeltaWatch watch) { + private > void openWatch(MutableStatusInfo status, + V watch, + String url, + Collection resources, + T group, + String version) { long watchId = watchCount.incrementAndGet(); - status.setDeltaWatch(watchId, watch); - watch.setStop(() -> status.removeDeltaWatch(watchId)); - return watchId; + status.setWatch(watchId, watch); + watch.setStop(() -> status.removeWatch(watchId)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("open watch {} for {}[{}] from node {} for version {}", + watchId, + url, + String.join(", ", resources), + group, + version); + } } /** @@ -365,7 +305,7 @@ public U getSnapshot(T group) { */ @Override public Collection groups() { - return ImmutableSet.copyOf(statuses.keySet()); + return ImmutableSet.copyOf(statuses.groups()); } /** @@ -377,40 +317,47 @@ public Collection groups() { @Override public void setSnapshot(T group, U snapshot) { // we take a writeLock to prevent watches from being created while we update the snapshot - ConcurrentMap> status; + Map> status; + Map> deltaStatus; U previousSnapshot; writeLock.lock(); try { // Update the existing snapshot entry. previousSnapshot = snapshots.put(group, snapshot); - status = statuses.get(group); + status = statuses.getStatus(group); + deltaStatus = statuses.getDeltaStatus(group); } finally { writeLock.unlock(); } - if (status == null) { + if (status.isEmpty() && deltaStatus.isEmpty()) { return; } // Responses should be in specific order and typeUrls has a list of resources in the right // order. - respondWithSpecificOrder(group, previousSnapshot, snapshot, status); + respondWithSpecificOrder(group, previousSnapshot, snapshot, status, deltaStatus); } /** * {@inheritDoc} */ @Override - public StatusInfo statusInfo(T group) { + public StatusInfo statusInfo(T group) { readLock.lock(); try { - ConcurrentMap> statusMap = statuses.get(group); - if (statusMap == null || statusMap.isEmpty()) { + Map> statusMap = statuses.getStatus(group); + Map> deltaStatusMap = statuses.getDeltaStatus(group); + + if (statusMap.isEmpty() && deltaStatusMap.isEmpty()) { return null; } - return new GroupCacheStatusInfo<>(statusMap.values()); + List> collection = Stream.concat(statusMap.values().stream(), + deltaStatusMap.values().stream()).collect(Collectors.toList()); + + return new GroupCacheStatusInfo<>(collection); } finally { readLock.unlock(); } @@ -419,83 +366,87 @@ public StatusInfo statusInfo(T group) { @VisibleForTesting protected void respondWithSpecificOrder(T group, U previousSnapshot, U snapshot, - ConcurrentMap> statusMap) { + Map> statusMap, + Map> deltaStatusMap) { for (Resources.ResourceType resourceType : RESOURCE_TYPES_IN_ORDER) { CacheStatusInfo status = statusMap.get(resourceType); - if (status == null) continue; // todo: why this happens? - status.watchesRemoveIf((id, watch) -> { - if (!watch.request().getResourceType().equals(resourceType)) { - return false; - } - String version = snapshot.version(watch.request().getResourceType(), watch.request().getResourceNamesList()); + if (status != null) { + status.watchesRemoveIf((id, watch) -> { + if (!watch.request().getResourceType().equals(resourceType)) { + return false; + } + String version = snapshot.version(watch.request().getResourceType(), watch.request().getResourceNamesList()); - if (!watch.request().getVersionInfo().equals(version)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("responding to open watch {}[{}] with new version {}", + if (!watch.request().getVersionInfo().equals(version)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("responding to open watch {}[{}] with new version {}", id, String.join(", ", watch.request().getResourceNamesList()), version); - } + } - respond(watch, snapshot, group); - - // Discard the watch. A new watch will be created for future snapshots once envoy ACKs the response. - return true; - } + respond(watch, snapshot, group); - // Do not discard the watch. The request version is the same as the snapshot version, so we wait to respond. - return false; - }); - - Map> previousResources = previousSnapshot == null - ? Collections.emptyMap() - : previousSnapshot.versionedResources(resourceType); - Map> snapshotResources = snapshot.versionedResources(resourceType); - - Map> snapshotChangedResources = snapshotResources.entrySet() - .stream() - .filter(entry -> { - VersionedResource versionedResource = previousResources.get(entry.getKey()); - return versionedResource == null || !versionedResource - .version().equals(entry.getValue().version()); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - Set snapshotRemovedResources = previousResources.keySet() - .stream() - .filter(s -> !snapshotResources.containsKey(s)) - .collect(Collectors.toSet()); - - status.deltaWatchesRemoveIf((id, watch) -> { - String version = snapshot.version(watch.request().getResourceType(), Collections.emptyList()); - - if (!watch.version().equals(version)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("responding to open watch {}[{}] with new version {}", - id, - String.join(", ", watch.trackedResources().keySet()), - version); + // Discard the watch. A new watch will be created for future snapshots once envoy ACKs the response. + return true; } + // Do not discard the watch. The request version is the same as the snapshot version, so we wait to respond. + return false; + }); + } + DeltaCacheStatusInfo deltaStatus = deltaStatusMap.get(resourceType); + if (deltaStatus != null) { + Map> previousResources = previousSnapshot == null + ? Collections.emptyMap() + : previousSnapshot.versionedResources(resourceType); + Map> snapshotResources = snapshot.versionedResources(resourceType); + + Map> snapshotChangedResources = snapshotResources.entrySet() + .stream() + .filter(entry -> { + VersionedResource versionedResource = previousResources.get(entry.getKey()); + return versionedResource == null || !versionedResource + .version().equals(entry.getValue().version()); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Set snapshotRemovedResources = previousResources.keySet() + .stream() + .filter(s -> !snapshotResources.containsKey(s)) + .collect(Collectors.toSet()); + + deltaStatus.watchesRemoveIf((id, watch) -> { + String version = snapshot.version(watch.request().getResourceType(), Collections.emptyList()); + + if (!watch.version().equals(version)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("responding to open watch {}[{}] with new version {}", + id, + String.join(", ", watch.trackedResources().keySet()), + version); + } + List removedResources = snapshotRemovedResources.stream() .filter(s -> watch.trackedResources().get(s) != null) .collect(Collectors.toList()); - Map> changedResources = findChangedResources(watch, snapshotChangedResources); + Map> changedResources = findChangedResources(watch, snapshotChangedResources); - ResponseState responseState = respondDelta(watch, - changedResources, - removedResources, - version, - group); - // Discard the watch if it was responded or cancelled. - // A new watch will be created for future snapshots once envoy ACKs the response. - return ResponseState.RESPONDED.equals(responseState) || ResponseState.CANCELLED.equals(responseState); - } + ResponseState responseState = respondDelta(watch, + changedResources, + removedResources, + version, + group); + // Discard the watch if it was responded or cancelled. + // A new watch will be created for future snapshots once envoy ACKs the response. + return responseState.isFinished(); + } - // Do not discard the watch. The request version is the same as the snapshot version, so we wait to respond. - return false; - }); + // Do not discard the watch. The request version is the same as the snapshot version, so we wait to respond. + return false; + }); + } } } @@ -625,28 +576,16 @@ private Map> findChangedResources(DeltaWatch watch, .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private ResponseState respondDeltaTracked(DeltaWatch watch, - Map> snapshotResources, - List removedResources, - String version, - T group) { - - Map> resources = snapshotResources.entrySet() - .stream() - .filter(entry -> { - if (watch.pendingResources().contains(entry.getKey())) { - return true; - } - String resourceVersion = watch.trackedResources().get(entry.getKey()); - if (resourceVersion == null) { - // resource is not tracked, should respond it only if watch is wildcard - return watch.isWildcard(); - } - return !entry.getValue().version().equals(resourceVersion); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - return respondDelta(watch, resources, removedResources, version, group); + private ResponseState respondDelta(DeltaXdsRequest request, DeltaWatch watch, U snapshot, String version, T group) { + Map> snapshotResources = snapshot.versionedResources(request.getResourceType()); + List removedResources = findRemovedResources(watch, + snapshotResources); + Map> changedResources = findChangedResources(watch, snapshotResources); + return respondDelta(watch, + changedResources, + removedResources, + version, + group); } private ResponseState respondDelta(DeltaWatch watch, @@ -681,6 +620,10 @@ private ResponseState respondDelta(DeltaWatch watch, private enum ResponseState { RESPONDED, UNRESPONDED, - CANCELLED + CANCELLED; + + private boolean isFinished() { + return this.equals(RESPONDED) || this.equals(CANCELLED); + } } } From 174e5a61c5fd528fbf1bbb6c517c7429a3e40a48 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Mon, 14 Feb 2022 08:24:10 +0100 Subject: [PATCH 37/46] delta xds property --- docs/configuration.md | 1 + .../servicemesh/envoycontrol/snapshot/SnapshotProperties.kt | 1 + .../snapshot/resource/clusters/EnvoyClustersFactory.kt | 2 +- .../resource/listeners/filters/HttpConnectionManagerFactory.kt | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f771fd709..7916295fb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -82,6 +82,7 @@ Property **envoy-control.envoy.snapshot.max-host-ttl** | The TTL for hosts that are unused. Hosts that have not been used in the configured time interval will be purged | 300s **envoy-control.envoy.snapshot.rate-limit.domain** | Domain name for ratelimit service. | rl **envoy-control.envoy.snapshot.rate-limit.service-name** | ratelimit GRPC service name | ratelimit-grpc +**envoy-control.envoy.snapshot.delta-xds-enabled** | Enable detla xds | false ## Permissions diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt index fb9882fe9..f745ff5c3 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt @@ -32,6 +32,7 @@ class SnapshotProperties { var jwt = JwtFilterProperties() var requireServiceName = false var rateLimit = RateLimitProperties() + var deltaXdsEnabled = false } class MetricsProperties { diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index 2670370ac..e8a4171a2 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -370,7 +370,7 @@ class EnvoyClustersFactory( .setResourceApiVersion(ApiVersion.V3) .setApiConfigSource( ApiConfigSource.newBuilder() - .setApiType(ApiConfigSource.ApiType.GRPC) + .setApiType(if (properties.deltaXdsEnabled) ApiConfigSource.ApiType.DELTA_GRPC else ApiConfigSource.ApiType.GRPC) .setTransportApiVersion(ApiVersion.V3) .addGrpcServices( 0, GrpcService.newBuilder().setEnvoyGrpc( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt index 117b3c0e4..0b793a26e 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt @@ -136,7 +136,7 @@ class HttpConnectionManagerFactory( private fun apiConfigSource(): ApiConfigSource { return ApiConfigSource.newBuilder() - .setApiType(ApiConfigSource.ApiType.GRPC) + .setApiType(if (snapshotProperties.deltaXdsEnabled) ApiConfigSource.ApiType.DELTA_GRPC else ApiConfigSource.ApiType.GRPC) .setTransportApiVersion(ApiVersion.V3) .addGrpcServices( GrpcService.newBuilder() From 2f2cab4cc64ab638dc21f52fb9b341e72cd8a980 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Mon, 14 Feb 2022 08:30:50 +0100 Subject: [PATCH 38/46] fix ktLint --- .../snapshot/resource/clusters/EnvoyClustersFactory.kt | 8 +++++++- .../listeners/filters/HttpConnectionManagerFactory.kt | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index e8a4171a2..6c4508f23 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -370,7 +370,13 @@ class EnvoyClustersFactory( .setResourceApiVersion(ApiVersion.V3) .setApiConfigSource( ApiConfigSource.newBuilder() - .setApiType(if (properties.deltaXdsEnabled) ApiConfigSource.ApiType.DELTA_GRPC else ApiConfigSource.ApiType.GRPC) + .setApiType( + if (properties.deltaXdsEnabled) { + ApiConfigSource.ApiType.DELTA_GRPC + } else { + ApiConfigSource.ApiType.GRPC + } + ) .setTransportApiVersion(ApiVersion.V3) .addGrpcServices( 0, GrpcService.newBuilder().setEnvoyGrpc( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt index 0b793a26e..336ecc73d 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/filters/HttpConnectionManagerFactory.kt @@ -136,7 +136,13 @@ class HttpConnectionManagerFactory( private fun apiConfigSource(): ApiConfigSource { return ApiConfigSource.newBuilder() - .setApiType(if (snapshotProperties.deltaXdsEnabled) ApiConfigSource.ApiType.DELTA_GRPC else ApiConfigSource.ApiType.GRPC) + .setApiType( + if (snapshotProperties.deltaXdsEnabled) { + ApiConfigSource.ApiType.DELTA_GRPC + } else { + ApiConfigSource.ApiType.GRPC + } + ) .setTransportApiVersion(ApiVersion.V3) .addGrpcServices( GrpcService.newBuilder() From 3d099e8ded760be80289afafc24d922d18e5544b Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Mon, 14 Feb 2022 12:50:21 +0100 Subject: [PATCH 39/46] add delta to ads --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7916295fb..65acf7bec 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -82,7 +82,7 @@ Property **envoy-control.envoy.snapshot.max-host-ttl** | The TTL for hosts that are unused. Hosts that have not been used in the configured time interval will be purged | 300s **envoy-control.envoy.snapshot.rate-limit.domain** | Domain name for ratelimit service. | rl **envoy-control.envoy.snapshot.rate-limit.service-name** | ratelimit GRPC service name | ratelimit-grpc -**envoy-control.envoy.snapshot.delta-xds-enabled** | Enable detla xds | false +**envoy-control.envoy.snapshot.delta-xds-enabled** | Enable detla xds | false ## Permissions From 2210be2383b99a0ef2ae668efbcd2039864b2216 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Wed, 16 Feb 2022 07:42:23 +0100 Subject: [PATCH 40/46] change config for DeltaAdsEnvoyControlSmokeTest --- .../tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt index ce7ed4f4f..cf7410a17 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoyControlSmokeTest.kt @@ -7,6 +7,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.assertions.untilAsserted import pl.allegro.tech.servicemesh.envoycontrol.config.Ads import pl.allegro.tech.servicemesh.envoycontrol.config.AdsWithNoDependencies import pl.allegro.tech.servicemesh.envoycontrol.config.AdsWithStaticListeners +import pl.allegro.tech.servicemesh.envoycontrol.config.DeltaAds import pl.allegro.tech.servicemesh.envoycontrol.config.Xds import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension @@ -117,7 +118,7 @@ class DeltaAdsEnvoyControlSmokeTest : EnvoyControlSmokeTest { @JvmField @RegisterExtension - val envoy = EnvoyExtension(envoyControl, service, config = Ads) + val envoy = EnvoyExtension(envoyControl, service, config = DeltaAds) } override fun consul() = consul From 95df981749ee75e0f665ff41818903c7ed7d79c1 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Tue, 22 Feb 2022 10:40:04 +0100 Subject: [PATCH 41/46] change java-control-plane snapshot version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 0b1a2402e..c876f20d6 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ allprojects { project.ext.versions = [ kotlin : '1.3.72', - java_controlplane : '0.1.31-delta-xds-slonka-hash-bytes-SNAPSHOT', + java_controlplane : '0.1.31-delta-xds-slonka-hash-bytes-performance-SNAPSHOT', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', From dd7c6a9b96d50fd77e48b0896ffb0c9e1d8a4390 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Mon, 28 Feb 2022 11:16:30 +0100 Subject: [PATCH 42/46] changelog added --- .github/workflows/changelog.yaml | 38 ++++++++++++++++++++++++++++++++ CHANGELOG.md | 10 +++++++++ docs/changelog_symlink.md | 10 +++++++++ mkdocs.yml | 1 + 4 files changed, 59 insertions(+) create mode 100644 .github/workflows/changelog.yaml create mode 100644 CHANGELOG.md create mode 100644 docs/changelog_symlink.md diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml new file mode 100644 index 000000000..d9627c350 --- /dev/null +++ b/.github/workflows/changelog.yaml @@ -0,0 +1,38 @@ +name: Check changelog + +on: + pull_request: + paths-ignore: + - '.github/**' + +jobs: + changelog: + name: Check changelog + runs-on: [self-hosted, ubuntu-latest] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.head_ref }} + + - uses: jitterbit/get-changed-files@v1 + id: files + continue-on-error: true + + - name: Printing modified files + run: | + echo "Added+Modified:" + echo "${{ steps.files.outputs.added_modified }}" + - name: Check if changelog is updated. + run: | + for changed_file in ${{ steps.files.outputs.added_modified }}; do + if [ ${changed_file} == "CHANGELOG.md" ] + then + exit 0 + fi + done + echo "::error:: Changelog not present in modified files" + exit 1 + + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..d935efdf2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +Lists all changes with user impact. +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## [Unreleased] + +### Changed + +- Added support for Delta XDS diff --git a/docs/changelog_symlink.md b/docs/changelog_symlink.md new file mode 100644 index 000000000..d935efdf2 --- /dev/null +++ b/docs/changelog_symlink.md @@ -0,0 +1,10 @@ +# Changelog + +Lists all changes with user impact. +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## [Unreleased] + +### Changed + +- Added support for Delta XDS diff --git a/mkdocs.yml b/mkdocs.yml index 0c27257e5..9deba272b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ markdown_extensions: nav: - About: index.md + - Changelog: changelog_symlink.md - Quickstart: quickstart.md - Architecture: architecture.md - Configuration: configuration.md From 898f3d9703c5711353f71f1dd55969b0ba768cbc Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Tue, 26 Jul 2022 08:21:46 +0200 Subject: [PATCH 43/46] update java-control-plane --- build.gradle | 2 +- .../allegro/tech/servicemesh/envoycontrol/SimpleCache.java | 6 +----- .../server/callbacks/MetricsDiscoveryServerCallbacks.kt | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 925677470..c1537f57f 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ allprojects { project.ext.versions = [ kotlin : '1.6.10', - java_controlplane : '0.1.32', + java_controlplane : '0.1.33-delta-xds-non-breaking-slonka-hash-bytes-SNAPSHOT', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', diff --git a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java index 272fa3e87..0dbabcc33 100644 --- a/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java +++ b/envoy-control-core/src/main/java/pl/allegro/tech/servicemesh/envoycontrol/SimpleCache.java @@ -186,11 +186,7 @@ public DeltaWatch createDeltaWatch( Preconditions.checkNotNull(requestResourceType, "unsupported type URL %s", request.getTypeUrl()); T group; - if (request.v3Request() != null) { - group = groups.hash(request.v3Request().getNode()); - } else { - group = groups.hash(request.v2Request().getNode()); - } + group = groups.hash(request.v3Request().getNode()); // even though we're modifying, we take a readLock to allow multiple watches to be created in parallel since it // doesn't conflict diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt index ce866c070..c0f65410b 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/callbacks/MetricsDiscoveryServerCallbacks.kt @@ -50,7 +50,7 @@ class MetricsDiscoveryServerCallbacks(private val meterRegistry: MeterRegistry) connectionsByType(typeUrl).decrementAndGet() } - override fun onV3StreamRequest(streamId: Long, request: DiscoveryRequestV3) { + override fun onV3StreamRequest(streamId: Long, request: V3DiscoveryRequest) { meterRegistry.counter("grpc.requests.${StreamType.fromTypeUrl(request.typeUrl).name.toLowerCase()}") .increment() } From ccd59a5741063f254605a9e889f8e7bdbfa96520 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Tue, 26 Jul 2022 10:22:52 +0200 Subject: [PATCH 44/46] revert singe serialization --- .../server/CachedProtoResourcesSerializer.kt | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt index 82254884c..3503b70a4 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/CachedProtoResourcesSerializer.kt @@ -26,7 +26,7 @@ internal class CachedProtoResourcesSerializer( } } - private val cache: Cache, MutableCollection> = createCache("protobuf-cache") + private val cache: Cache = createCache("protobuf-cache") private val timer = createTimer(reportMetrics, meterRegistry, "protobuf-cache.serialize.time") private fun createCache(cacheName: String): Cache { @@ -47,23 +47,13 @@ internal class CachedProtoResourcesSerializer( } } - override fun serialize( - resources: MutableCollection, - apiVersion: Resources.ApiVersion - ): MutableCollection { - return timer.record(Supplier { getResources(resources) }) - } - - private fun getResources(resources: MutableCollection): MutableCollection { - return cache.get(resources) { - resources.asSequence() - .map { Any.pack(it) } - .toMutableList() - } - } - - @Suppress("NotImplementedDeclaration") - override fun serialize(resource: Message?, apiVersion: Resources.ApiVersion?): Any { - throw NotImplementedError("Serializing single messages is not supported") + override fun serialize(resource: Message, apiVersion: Resources.ApiVersion): Any { + return timer.record(Supplier { + cache.get(resource) { + Any.pack( + resource + ) + } + }) } } From bbbdd0132318d6da1f8f344d5e68dfd22069fa99 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Fri, 9 Sep 2022 09:26:10 +0200 Subject: [PATCH 45/46] resolve conflicts --- CHANGELOG.md | 8 ++++++-- build.gradle | 2 +- .../envoycontrol/EnvoySnapshotFactoryTest.kt | 15 +++++---------- .../snapshot/debug/SnapshotDebugService.kt | 3 +-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fec813d04..65abb1826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [0.19.19] +### Changed +- Added support for Delta XDS + +## [0.19.19] + ### Changed - Add default access log filter configuration @@ -31,8 +36,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed - Added flags for lua filters -- Added support for Delta XDS - +- ## [0.19.14] ### Changed diff --git a/build.gradle b/build.gradle index c1537f57f..4eb2072cf 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ allprojects { project.ext.versions = [ kotlin : '1.6.10', - java_controlplane : '0.1.33-delta-xds-non-breaking-slonka-hash-bytes-SNAPSHOT', + java_controlplane : '0.1.35', spring_boot : '2.3.4.RELEASE', grpc : '1.21.0', jaxb : '2.3.1', diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt index 5af5c4c85..41fe92da5 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/EnvoySnapshotFactoryTest.kt @@ -29,7 +29,6 @@ import pl.allegro.tech.servicemesh.envoycontrol.snapshot.EnvoySnapshotFactory import pl.allegro.tech.servicemesh.envoycontrol.snapshot.GlobalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotProperties import pl.allegro.tech.servicemesh.envoycontrol.snapshot.SnapshotsVersions -import pl.allegro.tech.servicemesh.envoycontrol.snapshot.globalSnapshot import pl.allegro.tech.servicemesh.envoycontrol.snapshot.outgoingTimeoutPolicy import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory import pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.endpoints.EnvoyEndpointsFactory @@ -274,14 +273,10 @@ class EnvoySnapshotFactoryTest { } private fun GlobalSnapshot.withEndpoint(clusterName: String): GlobalSnapshot = copy( - endpoints = SnapshotResources.create( - listOf( - ClusterLoadAssignment.newBuilder() + endpoints = SnapshotResources.create(listOf(ClusterLoadAssignment.newBuilder() .setClusterName(clusterName) .build() - ), "v1" - ) - ) + ), "v1").resources()) private fun createServicesGroup( mode: CommunicationMode = CommunicationMode.XDS, @@ -375,11 +370,11 @@ class EnvoySnapshotFactoryTest { private fun createGlobalSnapshot(vararg clusters: Cluster): GlobalSnapshot { return GlobalSnapshot( - SnapshotResources.create(clusters.toList(), "pl/allegro/tech/servicemesh/envoycontrol/v3"), + SnapshotResources.create(clusters.toList(), "pl/allegro/tech/servicemesh/envoycontrol/v3").resources(), clusters.map { it.name }.toSet(), - SnapshotResources.create(emptyList(), "v1"), + SnapshotResources.create(emptyList(), "v1").resources(), emptyMap(), - SnapshotResources.create(clusters.toList(), "v3") + SnapshotResources.create(clusters.toList(), "v3").resources() ) } diff --git a/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugService.kt b/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugService.kt index 52891c9c1..346a74245 100644 --- a/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugService.kt +++ b/envoy-control-runner/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/debug/SnapshotDebugService.kt @@ -75,8 +75,7 @@ class SnapshotDebugService( logger.warn("Global snapshot is missing") throw GlobalSnapshotNotFoundException("Global snapshot is missing") } - val endpoints = globalSnapshot.endpoints - .resources()[service] + val endpoints = globalSnapshot.endpoints[service] if (endpoints == null) { logger.warn("Can not find $service in global snapshot") throw GlobalSnapshotNotFoundException("Service $service not found in global snapshot") From dbfefc95a828046fe2acc8563e597a4865c961b6 Mon Sep 17 00:00:00 2001 From: "radoslaw.chrzanowski" Date: Fri, 9 Sep 2022 10:47:16 +0200 Subject: [PATCH 46/46] configure netty event loops --- build.gradle | 2 +- docs/configuration.md | 4 +- .../servicemesh/envoycontrol/ControlPlane.kt | 18 +++++++++ .../envoycontrol/server/ServerProperties.kt | 1 + .../config/LocalReplyConfigFactoryTest.kt | 40 +++++++++---------- 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/build.gradle b/build.gradle index 4eb2072cf..b809b25c4 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,7 @@ allprojects { kotlin : '1.6.10', java_controlplane : '0.1.35', spring_boot : '2.3.4.RELEASE', - grpc : '1.21.0', + grpc : '1.48.1', jaxb : '2.3.1', javaxactivation : '1.2.0', micrometer : '1.5.5', diff --git a/docs/configuration.md b/docs/configuration.md index 6d51d3f8b..04f586a49 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,8 +7,8 @@ Property ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- **envoy-control.server.executor-group.type** | Group executor type. DIRECT or PARALLEL | DIRECT **envoy-control.server.executor-group.parallel-pool-size** | Pool size used for executor group in PARALLEL mode | 4 -**envoy-control.server.nio-event-loop-thread-count** | The number of threads that will be used by netty's nio event loop | 1 -**envoy-control.server.nio-event-loop-pool-size** | Pool size of NIO Event Loop | 0 (Number of CPUs * 2) +**envoy-control.server.nio-event-loop-thread-count** | The number of threads that will be used by netty's nio worker event loop | 1 +**envoy-control.server.nio-boss-event-loop-thread-count** | The number of threads that will be used by netty's nio boss event loop | 1 **envoy-control.server.netty.keep-alive-time** | Sets a custom keepalive time for Netty server | 15s **envoy-control.server.netty.permit-keep-alive-time** | Specify the most aggressive keep-alive time clients are permitted to configure (in seconds) | 10s **envoy-control.server.netty.permit-keep-alive-without-calls** | Sets whether to allow clients to send keep-alive HTTP/2 PINGs even if there are no outstanding RPCs on the connection | true diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt index c44725443..ac431e297 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ControlPlane.kt @@ -12,6 +12,7 @@ import io.grpc.netty.NettyServerBuilder import io.micrometer.core.instrument.MeterRegistry import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics import io.netty.channel.nio.NioEventLoopGroup +import io.netty.channel.socket.nio.NioServerSocketChannel import pl.allegro.tech.servicemesh.envoycontrol.groups.Group import pl.allegro.tech.servicemesh.envoycontrol.groups.GroupChangeWatcher import pl.allegro.tech.servicemesh.envoycontrol.groups.MetadataNodeGroup @@ -89,6 +90,7 @@ class ControlPlane private constructor( ) { var grpcServerExecutor: Executor? = null var nioEventLoopExecutor: Executor? = null + var nioBossEventLoopExecutor: Executor? = null var executorGroup: ExecutorGroup? = null var globalSnapshotExecutor: Executor? = null var globalSnapshotAuditExecutor: Executor? = null @@ -112,6 +114,12 @@ class ControlPlane private constructor( nioEventLoopExecutor = newMeteredCachedThreadPool("grpc-worker-event-loop") } + if (nioBossEventLoopExecutor == null) { + // unbounded executor - netty will only use configured number of threads + // (by nioEventLoopThreadCount property or default netty value: * 2) + nioBossEventLoopExecutor = newMeteredCachedThreadPool("grpc-boss-event-loop") + } + if (executorGroup == null) { executorGroup = buildExecutorGroup() } @@ -210,6 +218,11 @@ class ControlPlane private constructor( nioEventLoopExecutor ) ) + .bossEventLoopGroup(NioEventLoopGroup( + properties.server.nioBossEventLoopThreadCount, + nioBossEventLoopExecutor + )) + .channelType(NioServerSocketChannel::class.java) .executor(grpcServerExecutor) .keepAliveTime(properties.server.netty.keepAliveTime.toMillis(), TimeUnit.MILLISECONDS) .permitKeepAliveTime(properties.server.netty.permitKeepAliveTime.toMillis(), TimeUnit.MILLISECONDS) @@ -300,6 +313,11 @@ class ControlPlane private constructor( return this } + fun withNioBossEventLoopExecutor(executor: Executor): ControlPlaneBuilder { + nioBossEventLoopExecutor = executor + return this + } + fun withExecutorGroup(executor: ExecutorGroup): ControlPlaneBuilder { executorGroup = executor return this diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt index b7ad29c2f..6ad4a9385 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/server/ServerProperties.kt @@ -7,6 +7,7 @@ import java.time.Duration class ServerProperties { var port = 50000 var nioEventLoopThreadCount = 0 // if set to 0, default Netty value will be used: * 2 + var nioBossEventLoopThreadCount = 0 // if set to 0, default Netty value will be used: * 2 var serverPoolSize = 16 var serverPoolKeepAlive: Duration = Duration.ofMinutes(10) var executorGroup = ExecutorProperties() diff --git a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/config/LocalReplyConfigFactoryTest.kt b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/config/LocalReplyConfigFactoryTest.kt index 1613da04e..08141d3b2 100644 --- a/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/config/LocalReplyConfigFactoryTest.kt +++ b/envoy-control-core/src/test/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/listeners/config/LocalReplyConfigFactoryTest.kt @@ -448,9 +448,19 @@ body_format { value { struct_value { fields { - key: "service-tag" + key: "listOfIntegers" value { - string_value: "test" + list_value { + values { + number_value: 1.0 + } + values { + number_value: 2.0 + } + values { + number_value: 3.0 + } + } } } fields { @@ -467,30 +477,14 @@ body_format { } } fields { - key: "listOfIntegers" + key: "service-tag" value { - list_value { - values { - number_value: 1.0 - } - values { - number_value: 2.0 - } - values { - number_value: 3.0 - } - } + string_value: "test" } } } } } - fields { - key: "reason" - value { - number_value: 1.0 - } - } fields { key: "listOfMap" value { @@ -518,6 +512,12 @@ body_format { } } } + fields { + key: "reason" + value { + number_value: 1.0 + } + } } content_type: "application/envoy+json" }""".trimIndent()