diff --git a/meta-grpc-client/build.gradle b/meta-grpc-client/build.gradle index 68dd7eb0..b9a46fe8 100644 --- a/meta-grpc-client/build.gradle +++ b/meta-grpc-client/build.gradle @@ -45,6 +45,9 @@ def grpcVersion = '1.40.0' def protobufVersion = '3.15.6' def protocVersion = protobufVersion + + + dependencies { api project(':core') @@ -60,6 +63,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.14.1' implementation group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: '2.14.1' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.14.1' + implementation 'org.projectlombok:lombok:1.18.16' testImplementation "io.grpc:grpc-testing:${grpcVersion}" testImplementation "io.grpc:grpc-testing:${grpcVersion}" @@ -76,6 +80,9 @@ dependencies { testCommonImplementation group: 'org.roaringbitmap', name: 'RoaringBitmap', version: '0.9.3' testCommonImplementation "io.grpc:grpc-stub:${grpcVersion}" + compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.0-rc2' + compile 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.0-rc2' + } protobuf { diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/ShardEndPoint.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/ShardEndPoint.java new file mode 100644 index 00000000..b46a3e34 --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/ShardEndPoint.java @@ -0,0 +1,34 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints; + +public interface ShardEndPoint { + + String getHost(); + + boolean equals(Object that); + + int getPort(); + + Protocol getProtocol(); + + boolean mtls(); + + enum Protocol { + http1_1, http2_0, https1_1, https_2_0 + } +} diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/ShardedServiceRegistry.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/ShardedServiceRegistry.java new file mode 100644 index 00000000..b30dfae2 --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/ShardedServiceRegistry.java @@ -0,0 +1,27 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints; + +import java.util.List; +import java.util.Map; + +public interface ShardedServiceRegistry { + + Map> getEndpoints(String namespace); + + Map> getEndpoints(String namespace, long epoch); +} diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/Component.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/Component.java new file mode 100644 index 00000000..802cf8f4 --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/Component.java @@ -0,0 +1,107 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Component { + + @JsonProperty("component") + private String name; + + @JsonProperty("num-shards") + private int numShards; + + @JsonProperty("replicas") + private int numReplicas; + + private static final String[] replicas = {"a", "b", "c", "d", "e"}; + private int port; + + + public Component() { + + } + + public String getName() { + + return name; + } + + public int getNumShards() { + + return numShards; + } + + public int getNumReplicas() { + if(numReplicas < replicas.length) { + return numReplicas; + }else{ + return 1; + } + } + + public void setNumReplicas(int numReplicas) { + if(numReplicas < replicas.length) { + this.numReplicas = numReplicas; + }else{ + this.numReplicas = 1; + } + } + + public String getReplica(int replicaId){ + return replicas[replicaId]; + } + + public int getPort() { + + return port; + } + + public void setPort(int port) { + + this.port = port; + } + + + public void setNumShards(int numShards) { + + this.numShards = numShards; + } + + public void setName(String name) { + + this.name = name; + } + + public void setDefaultComponent(Component defaultComponent) { + if (this.numShards == 0){ + this.numShards = defaultComponent.getNumShards(); + } + if (this.numReplicas == 0){ + this.numReplicas = defaultComponent.getNumReplicas(); + } + if (this.port == 0){ + this.port = defaultComponent.getPort(); + } + } + + @Override + public String toString() { + return "Components {name=" + name + ", num-shards=" + numShards + ", replicas=" + numReplicas + ", port=" + port + "}"; + } +} diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/DeploymentConfig.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/DeploymentConfig.java new file mode 100644 index 00000000..de1d238a --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/DeploymentConfig.java @@ -0,0 +1,86 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DeploymentConfig { + + @JsonProperty("kubernetes.statefulset.registry.cluster") + private String clusterName; + + public String getClusterName() { + return clusterName; + } + + @JsonProperty("kubernetes.statefulset.registry.cluster.domain") + private String clusterDomain; + + @JsonProperty("kubernetes.statefulset.registry.namespaces") + private List namespaces; + + private final Map namespaceMap = new HashMap<>(); + + public DeploymentConfig(){ + + } + + public String getClusterDomain() { + return clusterDomain; + } + + public void setClusterDomain(String clusterDomain) { + this.clusterDomain=clusterDomain; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public List getListOfNamespaces() { + return namespaces; + } + + public void setListOfNamespaces(List namespaces) { + this.namespaces = namespaces; + } + + public Map toMap() { + Namespace defaultNamespace = null; + try { + for (Namespace namespace : namespaces) { + namespace.toMap(); + if (namespace.getName().equals("default")) { + defaultNamespace = namespace; + } else { + namespaceMap.put(namespace.getName(), namespace); + } + } + for (String key : namespaceMap.keySet()) { + namespaceMap.get(key).setDefaultNamespace(defaultNamespace); + } + } catch (Exception e){ + System.out.println(e); + } + + return namespaceMap; + } + +} diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystEndpoint.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystEndpoint.java new file mode 100644 index 00000000..b7584999 --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystEndpoint.java @@ -0,0 +1,106 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints.impl; + +import net.opentsdb.aura.metrics.meta.endpoints.ShardEndPoint; + +import java.util.Objects; + + +public class MystEndpoint implements ShardEndPoint { + + private final String host; + private final int port; + + public MystEndpoint(String host, int port) { + this.host = host; + this.port = port; + } + + + @Override + public String getHost() { + return this.host; + } + + @Override + public int getPort() { + return this.port; + } + + @Override + public Protocol getProtocol() { + return Protocol.http2_0; + } + + @Override + public boolean equals(Object that){ + if (Objects.isNull(that)){ + return false; + } + if (this == that){ + return true; + }else if (!(that instanceof ShardEndPoint)) { + return false; + + }else{ + if (Objects.equals(this.getHost(), ((ShardEndPoint) that).getHost()) && + Objects.equals(this.getPort(), ((ShardEndPoint) that).getPort()) + && Objects.equals(this.getProtocol(), ((ShardEndPoint) that).getProtocol()) && + ((ShardEndPoint) that).mtls() == false && this.mtls() == false ){ + + return true; + }else{ + return false; + } + } +} + + + @Override + public boolean mtls() { + return false; + } + + public static class Builder { + private String l_host; + private int l_port; + + public Builder withHost(String host) { + this.l_host = host; + return this; + } + + public Builder withPort(int port) { + this.l_port = port; + return this; + } + + public MystEndpoint build() { + return new MystEndpoint(l_host, l_port); + } + + public static Builder newBuilder() { + return new Builder(); + } + + } + @Override + public String toString() { + return "Endpoints {host=" + host + ", port=" + port +"}"; + } +} diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystStatefulSetRegistry.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystStatefulSetRegistry.java new file mode 100644 index 00000000..a5f70f70 --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystStatefulSetRegistry.java @@ -0,0 +1,72 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.opentsdb.aura.metrics.meta.endpoints.impl; + +import net.opentsdb.aura.metrics.meta.endpoints.ShardEndPoint; +import net.opentsdb.aura.metrics.meta.endpoints.ShardedServiceRegistry; + +import java.util.*; + + +public class MystStatefulSetRegistry implements ShardedServiceRegistry { + + private static final String pod_name_pattern = "%s-myst-%s-%s.%s.svc.%s"; + + private DeploymentConfig config; + + public MystStatefulSetRegistry(DeploymentConfig config) { + this.config=config; + } + + public MystStatefulSetRegistry() { + } + + @Override + public Map> getEndpoints(String namespace) { + Map> endpointsMap = new HashMap<>(); + + final int numShards = config.toMap().get(namespace).toMap().get("myst").getNumShards(); + final int numReplicas = config.toMap().get(namespace).toMap().get("myst").getNumReplicas(); + for(int i = 0; i < numReplicas; i++) { + String replica = config.toMap().get(namespace).toMap().get("myst").getReplica(i); + List shardEndPoints = new ArrayList<>(); + endpointsMap.put(replica, shardEndPoints); + for (int j = 0; j < numShards; j++) { + final MystEndpoint endpoint = MystEndpoint.Builder.newBuilder() + .withHost( + String.format( + pod_name_pattern, + namespace, + replica, + j, + namespace, + config.getClusterDomain())) + .withPort(config.toMap().get(namespace).toMap().get("myst").getPort()) + .build(); + shardEndPoints.add(endpoint); + + } + } + return endpointsMap; + } + + @Override + public Map> getEndpoints(String namespace, long epoch) { + return getEndpoints(namespace); + } +} diff --git a/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/Namespace.java b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/Namespace.java new file mode 100644 index 00000000..5a1b3495 --- /dev/null +++ b/meta-grpc-client/src/main/java/net/opentsdb/aura/metrics/meta/endpoints/impl/Namespace.java @@ -0,0 +1,102 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class Namespace { + + @JsonProperty("namespace") + private String name; + + @JsonProperty("k8s_namespace") + private String k8sNamespace; + + private List components; + private final Map componentMap = new HashMap<>(); + + public Namespace(){ + + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void setDefaultNamespace(Namespace defaultNamespace) { + if (Objects.nonNull(this.k8sNamespace)) { + this.k8sNamespace = defaultNamespace.getk8sNamespace(); + } + if (Objects.nonNull(defaultNamespace)) { + + final Map defaultComponentMap = defaultNamespace.componentMap; + Component defaultComponent = null; + for (String key : defaultComponentMap.keySet()) { + defaultComponent = defaultComponentMap.get(key); + if (componentMap.containsKey(key)) { + componentMap.get(key).setDefaultComponent(defaultComponent); + } else { + componentMap.put(defaultComponent.getName(), defaultComponent); + } + } + } + } + + public String getk8sNamespace() { + return k8sNamespace; + } + + public void setk8sNamespace(String k8sNamespace) { + this.k8sNamespace = k8sNamespace; + } + + public List getComponents(){ + return components; + } + + public void setComponents(List components){ + this.components = components; + } + + public Map toMap() + { + if (components != null) { + for (Component component : components) { + componentMap.put(component.getName(), component); + } + } + return componentMap; + } + + @Override + public String toString() { + return "Namespaces {name=" + name + ", k8s_namespace=" + k8sNamespace + ", " + componentMap + "}"; + } + + + + + +} diff --git a/meta-grpc-client/src/test/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystStatefulSetRegistryTest.java b/meta-grpc-client/src/test/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystStatefulSetRegistryTest.java new file mode 100644 index 00000000..bdb0be66 --- /dev/null +++ b/meta-grpc-client/src/test/java/net/opentsdb/aura/metrics/meta/endpoints/impl/MystStatefulSetRegistryTest.java @@ -0,0 +1,384 @@ +/* + * This file is part of OpenTSDB. + * Copyright (C) 2021 Yahoo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.opentsdb.aura.metrics.meta.endpoints.impl; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import net.opentsdb.aura.metrics.meta.endpoints.ShardEndPoint; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class MystStatefulSetRegistryTest { + + private static final String pod_name_pattern = "%s-myst-%s-%s.%s.svc.%s"; + private static final String[] replicas = {"a", "b", "c", "d", "e"}; + private static final String cluster_domain = "cluster.local"; + + @Test + public void testListOfEndpoints1() throws IOException { + + ObjectMapper mapper1 = new ObjectMapper(new YAMLFactory()); + + try { + DeploymentConfig cluster_config1 = mapper1.readValue(new File("src/test/resources/ConfigTest1.yaml"), DeploymentConfig.class); + Map namespaceMap = cluster_config1.toMap(); + MystStatefulSetRegistry myststs1 = new MystStatefulSetRegistry(cluster_config1); + HashMap> expectedEndpointsMap = new HashMap<>(); + List componentList1 = new ArrayList<>(); + List componentList2 = new ArrayList<>(); + Component a = new Component(); + a.setNumReplicas(1); + a.setNumShards(1); + a.setName("myst"); + a.setPort(9999); + Component b = new Component(); + b.setNumReplicas(1); + b.setNumShards(1); + b.setName("aura-metrics"); + b.setPort(4080); + componentList1.add(b); + componentList1.add(a); + Component m = new Component(); + m.setNumReplicas(1); + m.setNumShards(1); + m.setName("myst"); + m.setPort(9999); + componentList2.add(m); + Namespace n1 = new Namespace(); + n1.setName("default"); + n1.setk8sNamespace("default"); + n1.setComponents(componentList1); + Namespace n2 = new Namespace(); + n2.setName("ssp"); + n2.setk8sNamespace("ssp"); + n2.setComponents(componentList2); + List namespaceList1 = new ArrayList<>(); + namespaceList1.add(n1); + namespaceList1.add(n2); + DeploymentConfig expectedConfig = new DeploymentConfig(); + expectedConfig.setClusterName("kubernetes.default.GQ"); + expectedConfig.setClusterDomain("cluster.local"); + expectedConfig.setListOfNamespaces(namespaceList1); + + + System.out.println("Test Case 1: "); + for (String key : namespaceMap.keySet()) { + Map> result = myststs1.getEndpoints(key); + final int numShards = expectedConfig.toMap().get(key).toMap().get("myst").getNumShards();; + final int numReplicas = expectedConfig.toMap().get(key).toMap().get("myst").getNumReplicas(); + String namespace = key; + for (int i = 0; i < numReplicas; i++) { + String replica = replicas[i]; + List expectedShardEndPoints = new ArrayList<>(); + expectedEndpointsMap.put(replica, expectedShardEndPoints); + for (int j = 0; j < numShards; j++) { + MystEndpoint endpoint = MystEndpoint.Builder.newBuilder() + .withHost( + String.format( + pod_name_pattern, + namespace, + replica, + j, + namespace, + cluster_domain)) + .withPort(9999) + .build(); + expectedShardEndPoints.add(endpoint); + + } + } + + System.out.println("Result"+myststs1.getEndpoints(key)); + System.out.println("Expected"+expectedEndpointsMap); + assertEquals(expectedEndpointsMap,result); + } + + + } catch (Exception e) { + System.out.println(e); + } + + + } + + + @Test + public void testListOfEndpoints2() throws IOException { + + ObjectMapper mapper1 = new ObjectMapper(new YAMLFactory()); + try { + DeploymentConfig cluster_config1 = mapper1.readValue(new File("src/test/resources/ConfigTest2.yaml"), DeploymentConfig.class); + + Map namespaceMap = cluster_config1.toMap(); + MystStatefulSetRegistry myststs1 = new MystStatefulSetRegistry(cluster_config1); + Map> expectedEndpointsMap = new HashMap<>(); + List componentList1 = new ArrayList<>(); + List componentList2 = new ArrayList<>(); + Component a = new Component(); + a.setNumReplicas(1); + a.setNumShards(1); + a.setName("myst"); + a.setPort(9999); + Component b = new Component(); + b.setNumReplicas(1); + b.setNumShards(1); + b.setName("aura-metrics"); + b.setPort(4080); + componentList1.add(b); + componentList1.add(a); + Component m = new Component(); + m.setNumReplicas(2); + m.setNumShards(9); + m.setName("myst"); + m.setPort(9999); + componentList2.add(m); + Namespace n1 = new Namespace(); + n1.setName("default"); + n1.setk8sNamespace("default"); + n1.setComponents(componentList1); + Namespace n2 = new Namespace(); + n2.setName("Onevideo"); + n2.setk8sNamespace("Onevideo"); + n2.setComponents(componentList2); + List namespaceList1 = new ArrayList<>(); + namespaceList1.add(n1); + namespaceList1.add(n2); + DeploymentConfig expectedConfig = new DeploymentConfig(); + expectedConfig.setClusterName("kubernetes.default.GQ"); + expectedConfig.setClusterDomain("cluster.local"); + expectedConfig.setListOfNamespaces(namespaceList1); + + System.out.println("Test Case 2: "); + for (String key : namespaceMap.keySet()) { + Map> result = myststs1.getEndpoints(key); + + + final int numShards = expectedConfig.toMap().get(key).toMap().get("myst").getNumShards();; + final int numReplicas = expectedConfig.toMap().get(key).toMap().get("myst").getNumReplicas(); + String namespace = key; + + for (int i = 0; i < numReplicas; i++) { + String replica = replicas[i]; + List expectedShardEndPoints = new ArrayList<>(); + expectedEndpointsMap.put(replica, expectedShardEndPoints); + for (int j = 0; j < numShards; j++) { + MystEndpoint endpoint = MystEndpoint.Builder.newBuilder() + .withHost( + String.format( + pod_name_pattern, + namespace, + replica, + j, + namespace, + cluster_domain)) + .withPort(9999) + .build(); + expectedShardEndPoints.add(endpoint); + + } + } + + System.out.println("Result"+myststs1.getEndpoints(key)); + System.out.println("Expected"+expectedEndpointsMap); + assertEquals(expectedEndpointsMap, result); + } + + + } catch (Exception e) { + System.out.println(e); + } + + + } + + @Test + public void testListOfEndpoints3() throws IOException { + + ObjectMapper mapper1 = new ObjectMapper(new YAMLFactory()); + try { + DeploymentConfig cluster_config1 = mapper1.readValue(new File("src/test/resources/ConfigTest3.yaml"), DeploymentConfig.class); + + Map namespaceMap = cluster_config1.toMap(); + MystStatefulSetRegistry myststs1 = new MystStatefulSetRegistry(cluster_config1); + Map> expectedEndpointsMap = new HashMap<>(); + List componentList1 = new ArrayList<>(); + List componentList2 = new ArrayList<>(); + Component a = new Component(); + a.setNumReplicas(1); + a.setNumShards(1); + a.setName("myst"); + a.setPort(9999); + Component b = new Component(); + b.setNumReplicas(1); + b.setNumShards(1); + b.setName("aura-metrics"); + b.setPort(4080); + componentList1.add(b); + componentList1.add(a); + Component m = new Component(); + m.setNumShards(0); + m.setName("myst"); + componentList2.add(m); + Namespace n1 = new Namespace(); + n1.setName("default"); + n1.setk8sNamespace("default"); + n1.setComponents(componentList1); + Namespace n2 = new Namespace(); + n2.setName("Onevideo"); + n2.setk8sNamespace("Onevideo"); + n2.setComponents(componentList2); + List namespaceList1 = new ArrayList<>(); + namespaceList1.add(n1); + namespaceList1.add(n2); + DeploymentConfig expectedConfig = new DeploymentConfig(); + expectedConfig.setClusterName("kubernetes.default.GQ"); + expectedConfig.setClusterDomain("cluster.local"); + expectedConfig.setListOfNamespaces(namespaceList1); + + System.out.println("Test Case 3: "); + for (String key : namespaceMap.keySet()) { + Map> result = myststs1.getEndpoints(key); + final int numShards = expectedConfig.toMap().get(key).toMap().get("myst").getNumShards();; + final int numReplicas = expectedConfig.toMap().get(key).toMap().get("myst").getNumReplicas(); + String namespace = key; + for (int i = 0; i < numReplicas; i++) { + String replica = replicas[i]; + List expectedShardEndPoints = new ArrayList<>(); + expectedEndpointsMap.put(replica, expectedShardEndPoints); + for (int j = 0; j < numShards; j++) { + final MystEndpoint endpoint = MystEndpoint.Builder.newBuilder() + .withHost( + String.format( + pod_name_pattern, + namespace, + replica, + j, + namespace, + cluster_domain)) + .withPort(9999) + .build(); + expectedShardEndPoints.add(endpoint); + + } + } + + System.out.println("Result"+myststs1.getEndpoints(key)); + System.out.println("Expected"+expectedEndpointsMap); + assertEquals(expectedEndpointsMap.toString(), result.toString()); + } + + + } catch (Exception e) { + System.out.println(e); + } + + } + + @Test + public void testListOfEndpoints4() throws IOException { + + ObjectMapper mapper1 = new ObjectMapper(new YAMLFactory()); + try { + DeploymentConfig cluster_config1 = mapper1.readValue(new File("src/test/resources/ConfigTest4.yaml"), DeploymentConfig.class); + + Map namespaceMap = cluster_config1.toMap(); + MystStatefulSetRegistry myststs1 = new MystStatefulSetRegistry(cluster_config1); + Map> expectedEndpointsMap = new HashMap<>(); + List componentList1 = new ArrayList<>(); + List componentList2 = new ArrayList<>(); + Component a = new Component(); + a.setNumReplicas(1); + a.setNumShards(1); + a.setName("myst"); + a.setPort(9999); + Component b = new Component(); + b.setNumReplicas(1); + b.setNumShards(1); + b.setName("aura-metrics"); + b.setPort(4080); + componentList1.add(b); + componentList1.add(a); + Component m = new Component(); + m.setNumReplicas(6); + m.setNumShards(10); + m.setName("myst"); + componentList2.add(m); + Namespace n1 = new Namespace(); + n1.setName("default"); + n1.setk8sNamespace("default"); + n1.setComponents(componentList1); + Namespace n2 = new Namespace(); + n2.setName("Onevideo"); + n2.setk8sNamespace("Onevideo"); + n2.setComponents(componentList2); + List namespaceList1 = new ArrayList<>(); + namespaceList1.add(n1); + namespaceList1.add(n2); + DeploymentConfig expectedConfig = new DeploymentConfig(); + expectedConfig.setClusterName("kubernetes.default.GQ"); + expectedConfig.setClusterDomain("cluster.local"); + expectedConfig.setListOfNamespaces(namespaceList1); + + System.out.println("Test Case 4: "); + for (String key : namespaceMap.keySet()) { + Map> result = myststs1.getEndpoints(key); + final int numShards = expectedConfig.toMap().get(key).toMap().get("myst").getNumShards(); + final int numReplicas = expectedConfig.toMap().get(key).toMap().get("myst").getNumReplicas(); + String namespace = key; + for (int i = 0; i < numReplicas; i++) { + String replica = replicas[i]; + List expectedShardEndPoints = new ArrayList<>(); + expectedEndpointsMap.put(replica, expectedShardEndPoints); + for (int j = 0; j < numShards; j++) { + final MystEndpoint endpoint = MystEndpoint.Builder.newBuilder() + .withHost( + String.format( + pod_name_pattern, + namespace, + replica, + j, + namespace, + cluster_domain)) + .withPort(9999) + .build(); + expectedShardEndPoints.add(endpoint); + + } + } + + + System.out.println("Result"+myststs1.getEndpoints(key)); + System.out.println("Expected"+expectedEndpointsMap); + assertEquals(expectedEndpointsMap, result); + } + + + } catch (Exception e) { + System.out.println(e); + } + + } +} diff --git a/meta-grpc-client/src/test/resources/ConfigTest1.yaml b/meta-grpc-client/src/test/resources/ConfigTest1.yaml new file mode 100644 index 00000000..505a09e4 --- /dev/null +++ b/meta-grpc-client/src/test/resources/ConfigTest1.yaml @@ -0,0 +1,26 @@ +kubernetes.statefulset.registry.cluster: kubernetes.default.GQ +kubernetes.statefulset.registry.cluster.domain: cluster.local +kubernetes.statefulset.registry.namespaces: + - namespace: default + k8s_namespace: default + components: + - component: aura-metrics + num-shards: 1 + replicas: 1 + port: 4080 + - component: myst + num-shards: 1 + replicas: 1 + port: 9999 + - namespace: ssp + k8s_namespace: ssp + components: + - component: aura-metrics + num-shards: 10 + replicas: 1 + - component: myst + + + + + diff --git a/meta-grpc-client/src/test/resources/ConfigTest2.yaml b/meta-grpc-client/src/test/resources/ConfigTest2.yaml new file mode 100644 index 00000000..b67f1fad --- /dev/null +++ b/meta-grpc-client/src/test/resources/ConfigTest2.yaml @@ -0,0 +1,23 @@ +kubernetes.statefulset.registry.cluster: kubernetes.default.GQ +kubernetes.statefulset.registry.cluster.domain: cluster.local +kubernetes.statefulset.registry.namespaces: + - namespace: default + k8s_namespace: default + components: + - component: aura-metrics + num-shards: 1 + replicas: 1 + port: 4080 + - component: myst + num-shards: 1 + replicas: 1 + port: 9999 + - namespace: Onevideo + k8s_namespace: default + components: + - component: myst + num-shards: 9 + replicas: 2 + + + diff --git a/meta-grpc-client/src/test/resources/ConfigTest3.yaml b/meta-grpc-client/src/test/resources/ConfigTest3.yaml new file mode 100644 index 00000000..8be6d0b3 --- /dev/null +++ b/meta-grpc-client/src/test/resources/ConfigTest3.yaml @@ -0,0 +1,21 @@ +kubernetes.statefulset.registry.cluster: kubernetes.default.GQ +kubernetes.statefulset.registry.cluster.domain: cluster.local +kubernetes.statefulset.registry.namespaces: + - namespace: default + k8s_namespace: default + components: + - component: aura-metrics + num-shards: 1 + replicas: 1 + port: 4080 + - component: myst + num-shards: 1 + replicas: 1 + port: 9999 + - namespace: Onevideo + components: + - component: myst + num-shards: 0 + + + diff --git a/meta-grpc-client/src/test/resources/ConfigTest4.yaml b/meta-grpc-client/src/test/resources/ConfigTest4.yaml new file mode 100644 index 00000000..e33d8e03 --- /dev/null +++ b/meta-grpc-client/src/test/resources/ConfigTest4.yaml @@ -0,0 +1,22 @@ +kubernetes.statefulset.registry.cluster: kubernetes.default.GQ +kubernetes.statefulset.registry.cluster.domain: cluster.local +kubernetes.statefulset.registry.namespaces: + - namespace: default + k8s_namespace: default + components: + - component: aura-metrics + num-shards: 1 + replicas: 1 + port: 4080 + - component: myst + num-shards: 1 + replicas: 1 + port: 9999 + - namespace: Onevideo + components: + - component: myst + num-shards: 10 + replicas: 8 + + +