Merge pull request #764 from averevki/coredns-multicluster-tests

Add CoreDNS multicluster tests

Author: averevki

config/settings.local.yaml.tpl

@@ -54,12 +54,17 @@
 #        api_url: "https://api.kubernetes2.com"
 #        token: "KUADRANT_RULEZ"
 #        kubeconfig_path: "~/.kube/config2"
+#    cluster3:                                     # Third cluster for the multicluster tests
+#        api_url: "https://api.kubernetes3.com"
+#        token: "KUADRANT_RULEZ"
+#        kubeconfig_path: "~/.kube/config3"
 #    slow_loadbalancers: false                     # For use in Openshift on AWS: If true, causes all Gateways and LoadBalancer Services to wait longer to become ready
 #    provider_secret: "aws-credentials"            # Name of the Secret resource that contains DNS provider credentials
 #    issuer:                                       # Issuer object for testing TLSPolicy
 #      name: "selfsigned-cluster-issuer"           # Name of Issuer CR
 #      kind: "ClusterIssuer"                       # Kind of Issuer, can be "Issuer" or "ClusterIssuer"
 #  dns:
+#    coredns_zone: "coredns.kuadrant-qe.net"       # hosted zone for coredns delegation tests
 #    dns_server:
 #      geo_code: "DE"                              # dns provider geo code of the dns server
 #      address: "ns1.seolizer.de"                  # dns nameserver hostname or ip

testsuite/capabilities.py

@@ -14,6 +14,9 @@ def has_kuadrant():
     clusters = [settings["control_plane"]["cluster"]]
     if cluster2 := settings["control_plane"]["cluster2"]:
         clusters.append(cluster2)
+    if cluster3 := settings["control_plane"]["cluster3"]:
+        clusters.append(cluster3)
+
     for cluster in clusters:
         system_project = cluster.change_project(project)
         if not system_project.connected:
@@ -32,6 +35,8 @@ def kuadrant_version():
     clusters = [settings["control_plane"]["cluster"]]
     if cluster2 := settings["control_plane"]["cluster2"]:
         clusters.append(cluster2)
+    if cluster3 := settings["control_plane"]["cluster3"]:
+        clusters.append(cluster3)
     versions = []
     for cluster in clusters:
         project = cluster.change_project(settings["service_protection"]["system_project"])

testsuite/config/openshift_loader.py

@@ -43,3 +43,8 @@ def load(obj, env=None, silent=True, key=None, filename=None):
         obj["control_plane"]["cluster2"] = KubernetesClient(
             cluster2.get("project"), cluster2.get("api_url"), cluster2.get("token"), cluster2.get("kubeconfig_path")
         )
+
+    if cluster3 := control_plane.setdefault("cluster3", {}):
+        obj["control_plane"]["cluster3"] = KubernetesClient(
+            cluster3.get("project"), cluster3.get("api_url"), cluster3.get("token"), cluster3.get("kubeconfig_path")
+        )

testsuite/kuadrant/policy/dns.py

@@ -66,6 +66,55 @@ def wait_for_ready(self):
         assert success, "DNSHealthCheckProbe status wasn't ready in time"
 
 
+@dataclass
+class DNSRecordEndpoint:  # pylint: disable=invalid-name
+    """Spec for DNSRecord endpoint"""
+
+    dnsName: str
+    recordTTL: int
+    recordType: str
+    targets: list[str]
+
+
+class DNSRecord(KubernetesObject):
+    """DNSRecord object"""
+
+    @classmethod
+    def create_instance(
+        cls,
+        cluster: KubernetesClient,
+        name: str,
+        root_host: str,
+        endpoints: list[DNSRecordEndpoint] = None,
+        delegate: bool = None,
+        labels: dict[str, str] = None,
+    ):
+        """Creates new instance of DNSRecord"""
+
+        model: dict = {
+            "apiVersion": "kuadrant.io/v1alpha1",
+            "kind": "DNSRecord",
+            "metadata": {"name": name, "labels": labels},
+            "spec": {
+                "rootHost": root_host,
+                "endpoints": [asdict(ep) for ep in endpoints] if endpoints else None,
+            },
+        }
+
+        if delegate is not None:
+            model["spec"]["delegate"] = delegate
+
+        return cls(model, context=cluster.context)
+
+    def wait_for_ready(self):
+        """Waits until DNSRecord is ready"""
+        success = self.wait_until(
+            lambda obj: len(obj.model.status.conditions) > 0
+            and all(condition.status == "True" for condition in obj.model.status.conditions)
+        )
+        assert success, f"DNSRecord {self.name()} did not get ready in time"
+
+
 class DNSPolicy(Policy):
     """DNSPolicy object"""
 
@@ -76,6 +125,7 @@ def create_instance(
         name: str,
         parent: Referencable,
         provider_secret_name: str,
+        delegate: bool = None,
         load_balancing: LoadBalancing = None,
         labels: dict[str, str] = None,
     ):
@@ -91,6 +141,9 @@ def create_instance(
             },
         }
 
+        if delegate is not None:
+            model["spec"]["delegate"] = delegate
+
         if load_balancing:
             model["spec"]["loadBalancing"] = asdict(load_balancing)
 

testsuite/kubernetes/client.py

@@ -8,6 +8,7 @@
 
 from testsuite.kubernetes.openshift.route import OpenshiftRoute
 from testsuite.kubernetes.service import Service
+from .service_account import ServiceAccount
 from .deployment import Deployment
 from .secret import Secret
 
@@ -54,6 +55,11 @@ def token(self):
         """Returns real Kubernetes token"""
         return self._token or self.inspect_context(jsonpath="{.users[*].user.token}", raw=True)
 
+    def get_service_account(self, name: str):
+        """Select service account by the name and return testsuite ServiceAccount object wrapping it"""
+        with self.context:
+            return oc.selector(f"sa/{name}").object(cls=ServiceAccount)
+
     @cached_property
     def apps_url(self):
         """Return URL under which all routes are routed"""
@@ -113,14 +119,12 @@ def do_action(self, verb: str, *args, stdin_str=None, auto_raise: bool = True, p
                 return oc.APIObject(string_to_model=result.out())
             return result
 
-    def inspect_context(self, jsonpath, raw=False):
+    def inspect_context(self, jsonpath=None, raw=False):
         """Returns jsonpath from the current context"""
-        return (
-            self.do_action("config", "view", f'--output=jsonpath="{jsonpath}"', f"--raw={raw}", "--minify=true")
-            .out()
-            .replace('"', "")
-            .strip()
-        )
+        action = ["config", "view", "--minify=true", f"--raw={raw}"]
+        if jsonpath:
+            action.append(f'--output=jsonpath="{jsonpath}"')
+        return self.do_action(*action).out().replace('"', "").strip()
 
     @property
     def project_exists(self):

testsuite/kubernetes/deployment.py

@@ -152,7 +152,10 @@ def wait_for_ready(self, timeout=90):
 
     def wait_for_replicas(self, replicas: int, timeout=90):
         """Waits until Deployment has at least the given number of replicas"""
-        success = self.wait_until(lambda obj: obj.model.status["readyReplicas"] >= replicas, timelimit=timeout)
+        success = self.wait_until(
+            lambda obj: "readyReplicas" in obj.model.status and obj.model.status["readyReplicas"] >= replicas,
+            timelimit=timeout,
+        )
         assert success, f"Deployment {self.name()} did not get {replicas} replicas in time"
 
     @property
@@ -187,6 +190,13 @@ def add_volume(self, volume: Volume):
         mounts = self.template.setdefault("volumes", [])
         mounts.append(asdict(volume))
 
+    def restart(self):
+        """Restarts the deployment by scaling replicas to 0 and back to original"""
+        original_replicas = self.replicas
+        self.set_replicas(0)
+        self.set_replicas(original_replicas)
+        self.wait_for_replicas(original_replicas)
+
     def rollout(self, hard=False, timeout=90):
         """
         Performs rollout on the Deployment and waits until complete.

testsuite/kubernetes/secret.py

@@ -16,7 +16,7 @@ def create_instance(
         cluster,
         name,
         data: dict[str, str],
-        secret_type: Literal["kubernetes.io/tls", "kuadrant.io/aws", "Opaque"] = "Opaque",
+        secret_type: Literal["kubernetes.io/tls", "kuadrant.io/aws", "kuadrant.io/coredns", "Opaque"] = "Opaque",
         labels: dict[str, str] = None,
     ):
         """Creates new Secret"""

testsuite/kubernetes/service_account.py

@@ -1,18 +1,17 @@
 """Service Account object for Kubernetes"""
 
+import yaml
+
+import openshift_client as oc
+
 from testsuite.kubernetes import KubernetesObject
-from testsuite.kubernetes.client import KubernetesClient
 
 
 class ServiceAccount(KubernetesObject):
     """Kubernetest ServiceAccount"""
 
-    def __init__(self, cluster: KubernetesClient, model: dict):
-        self.cluster = cluster
-        super().__init__(model, context=cluster.context)
-
     @classmethod
-    def create_instance(cls, openshift: KubernetesClient, name: str, labels: dict[str, str] = None):
+    def create_instance(cls, cluster, name: str, labels: dict[str, str] = None):
         """Creates new instance of service account"""
         model = {
             "kind": "ServiceAccount",
@@ -23,9 +22,38 @@ def create_instance(cls, openshift: KubernetesClient, name: str, labels: dict[st
             },
         }
 
-        return cls(openshift, model)
+        return cls(model, context=cluster.context)
 
-    def get_auth_token(self, audiences: list[str] = None) -> str:
+    def get_auth_token(self, audiences: list[str] = None, duration: str = None) -> str:
         """Requests and returns bound token for service account"""
-        audiences_args = [f"--audience={a}" for a in audiences or []]
-        return self.cluster.do_action("create", "token", self.name(), *audiences_args).out().strip()
+        args = ["token", self.name()]
+        if audiences:
+            args.extend([f"--audience={a}" for a in audiences])
+        if duration:
+            args.append(f"--duration={duration}")
+        with self.context:
+            return oc.invoke("create", args).out().strip()
+
+    def get_kubeconfig(self, context_name, user_name, cluster_name, api_url) -> str:
+        """Assembles and returns kubeconfig with service account token"""
+        kubeconfig = {
+            "apiVersion": "v1",
+            "kind": "Config",
+            "clusters": [
+                {
+                    "cluster": {"insecure-skip-tls-verify": True, "server": api_url},  # insecure clusters only for now
+                    "name": cluster_name,
+                }
+            ],
+            "contexts": [
+                {
+                    "context": {"cluster": cluster_name, "namespace": self.context.project_name, "user": user_name},
+                    "name": context_name,
+                }
+            ],
+            "current-context": context_name,
+            "preferences": {},
+            "users": [{"name": user_name, "user": {"token": self.get_auth_token(duration="1h")}}],
+        }
+
+        return yaml.dump(kubeconfig)

testsuite/tests/conftest.py

@@ -2,9 +2,11 @@
 
 import signal
 from urllib.parse import urlparse
+import yaml
 
 import pytest
 from pytest_metadata.plugin import metadata_key  # type: ignore
+from openshift_client import selector
 from dynaconf import ValidationError
 from keycloak import KeycloakAuthenticationError
 
@@ -16,8 +18,10 @@
 from testsuite.mockserver import Mockserver
 from testsuite.oidc import OIDCProvider
 from testsuite.oidc.auth0 import Auth0Provider
+from testsuite.prometheus import Prometheus
 from testsuite.oidc.keycloak import Keycloak
 from testsuite.tracing.jaeger import JaegerClient
+from testsuite.kubernetes.config_map import ConfigMap
 from testsuite.tracing.tempo import RemoteTempoClient
 from testsuite.utils import randomize, _whoami
 
@@ -128,6 +132,33 @@ def testconfig():
     return settings
 
 
+@pytest.fixture(scope="session")
+def prometheus(cluster):
+    """
+    Return an instance of Thanos metrics client
+    Skip tests if query route is not properly configured
+    """
+    openshift_monitoring = cluster.change_project("openshift-monitoring")
+    # Check if metrics are enabled
+    try:
+        with openshift_monitoring.context:
+            cm = selector("cm/cluster-monitoring-config").object(cls=ConfigMap)
+            assert yaml.safe_load(cm["config.yaml"])["enableUserWorkload"]
+    except Exception:  # pylint: disable=broad-exception-caught
+        pytest.skip("User workload monitoring is disabled")
+
+    # find thanos-querier route in the openshift-monitoring project
+    # this route allows to query metrics
+
+    routes = openshift_monitoring.get_routes_for_service("thanos-querier")
+    if len(routes) == 0:
+        pytest.skip("Skipping metrics tests as query route is not properly configured")
+
+    url = ("https://" if "tls" in routes[0].model.spec else "http://") + routes[0].model.spec.host
+    with KuadrantClient(headers={"Authorization": f"Bearer {cluster.token}"}, base_url=url, verify=False) as client:
+        yield Prometheus(client)
+
+
 @pytest.fixture(scope="session")
 def keycloak(request, testconfig, blame, skip_or_fail):
     """Keycloak OIDC Provider fixture"""