Merge pull request #433 from pehala/dns_multicluster

Introduce Multi-cluster tests

Makefile

@@ -1,4 +1,4 @@
-.PHONY: commit-acceptance pylint mypy black reformat test authorino poetry poetry-no-dev mgc container-image polish-junit reportportal authorino-standalone limitador kuadrant kuadrant-only disruptive kuadrantctl
+.PHONY: commit-acceptance pylint mypy black reformat test authorino poetry poetry-no-dev mgc container-image polish-junit reportportal authorino-standalone limitador kuadrant kuadrant-only disruptive kuadrantctl multicluster
 
 TB ?= short
 LOGLEVEL ?= INFO
@@ -45,23 +45,27 @@ test pytest tests: kuadrant
 
 authorino: ## Run only authorino related tests
 authorino: poetry-no-dev
-	$(PYTEST) -n4 -m 'authorino' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
+	$(PYTEST) -n4 -m 'authorino and not multicluster' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
 
 authorino-standalone: ## Run only test capable of running with standalone Authorino
 authorino-standalone: poetry-no-dev
 	$(PYTEST) -n4 -m 'authorino and not kuadrant_only' --dist loadfile --enforce --standalone $(flags) testsuite/tests/kuadrant/authorino
 
 limitador: ## Run only Limitador related tests
 limitador: poetry-no-dev
-	$(PYTEST) -n4 -m 'limitador' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
+	$(PYTEST) -n4 -m 'limitador and not multicluster' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
 
 kuadrant: ## Run all tests available on Kuadrant
 kuadrant: poetry-no-dev
-	$(PYTEST) -n4 -m 'not standalone_only and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
+	$(PYTEST) -n4 -m 'not standalone_only and not multicluster and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
 
 kuadrant-only: ## Run Kuadrant-only tests
 kuadrant-only: poetry-no-dev
-	$(PYTEST) -n4 -m 'kuadrant_only and not standalone_only and not disruptive' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
+	$(PYTEST) -n4 -m 'kuadrant_only and not standalone_only and not disruptive and not multicluster' --dist loadfile --enforce $(flags) testsuite/tests/kuadrant
+
+multicluster: ## Run Multicluster only tests
+multicluster: poetry-no-dev
+	$(PYTEST) -m 'multicluster' --dist loadfile --enforce $(flags) testsuite
 
 dnstls: ## Run DNS and TLS tests
 dnstls: poetry-no-dev

config/settings.local.yaml.tpl

@@ -1,7 +1,6 @@
 #default:
 #  tester: "someuser"                          # Optional: name of the user, who is running the tests, defaults to whoami/uid
 #  cluster:                                    # Workload cluster where tests should run, will get overriden if run on Multicluster
-#    project: "kuadrant"                       # Optional: Default namespace for this cluster
 #    api_url: "https://api.openshift.com"      # Optional: OpenShift API URL, if None it will OpenShift that you are logged in
 #    token: "KUADRANT_RULEZ"                   # Optional: OpenShift Token, if None it will OpenShift that you are logged in
 #    kubeconfig_path: "~/.kube/config"         # Optional: Kubeconfig to use, if None the default one is used
@@ -43,6 +42,7 @@
 #      metrics_service_name: ""                    # controller metrics service name for already deployed Authorino
 #  default_exposer: "kubernetes"                   # Force Exposer typem options: 'openshift', 'kind', 'kubernetes'
 #  control_plane:
+#    additional_clusters: []                       # List of additional clusters for Multicluster testing, see 'cluster' option for more details
 #    managedzone: aws-mz                           # Name of the ManagedZone resource
 #    issuer:                                       # Issuer object for testing TLSPolicy
 #      name: "selfsigned-cluster-issuer"           # Name of Issuer CR

config/settings.yaml

@@ -23,6 +23,7 @@ default:
       deploy: true
       log_level: "debug"
   control_plane:
+    additional_clusters: []
     managedzone: "aws-mz"
     issuer:
       name: "selfsigned-issuer"

pyproject.toml

@@ -14,6 +14,7 @@ pytest = "*"
 pytest-asyncio = ">=0.23.4"
 pytest-html = "*"
 dynaconf = "*"
+dnspython = "*"
 python-keycloak = ">=3.6"
 python-jose = "*"
 lxml = "*"
@@ -48,6 +49,7 @@ markers = [
     "tlspolicy: Test is using TLSPolicy",
     "dnspolicy: Test is using DNSPolicy",
     "disruptive: Test is disruptive",
+    "multicluster: Test is specifc to Multicluster deployment",
 ]
 filterwarnings = [
     "ignore: WARNING the new order is not taken into account:UserWarning",

testsuite/capabilities.py

@@ -10,20 +10,20 @@
 @functools.cache
 def has_kuadrant():
     """Returns True, if Kuadrant deployment is present and should be used"""
-    if spokes := weakget(settings)["control_plane"]["spokes"] % {}:
-        for name, openshift in spokes.items():
+    project = settings["service_protection"]["system_project"]
+    if clusters := weakget(settings)["control_plane"]["additional_clusters"] % []:
+        for openshift in clusters:
+            name = openshift.api_url
             # Try if Kuadrant is deployed
             if not openshift.connected:
-                return False, f"Spoke {name} is not connected"
-            project = settings["service_protection"]["system_project"]
+                return False, f"Cluster {name} is not connected"
             kuadrant_openshift = openshift.change_project(project)
             kuadrants = kuadrant_openshift.do_action("get", "kuadrant", "-o", "json", parse_output=True)
             if len(kuadrants.model["items"]) == 0:
-                return False, f"Spoke {name} does not have Kuadrant resource in project {project}"
+                return False, f"Cluster {name} does not have Kuadrant resource in project {project}"
 
     else:
-        openshift = settings["service_protection"]["project"]
-        project = settings["service_protection"]["system_project"]
+        openshift = settings["cluster"]
         kuadrant_openshift = openshift.change_project(project)
         kuadrants = kuadrant_openshift.do_action("get", "kuadrant", "-o", "json", parse_output=True)
         if len(kuadrants.model["items"]) == 0:

testsuite/config/openshift_loader.py

@@ -26,20 +26,14 @@ def load(obj, env=None, silent=True, key=None, filename=None):
         tools = client.change_project(obj["tools"]["project"])
     obj["tools"] = tools
 
-    service_protection = obj.setdefault("service_protection", {})
-    inject_client(service_protection, client, "project")
-    inject_client(service_protection, client, "project2")
-
     control_plane = obj.setdefault("control_plane", {})
-    hub = control_plane.get("hub", {})
-    hub_client = OpenShiftClient(hub.get("project"), hub.get("api_url"), hub.get("token"), hub.get("kubeconfig_path"))
-    obj["control_plane"]["hub"] = hub_client
-
-    clients = {}
-    spokes = control_plane.setdefault("spokes", {})
-    for name, value in spokes.items():
-        clients[name] = OpenShiftClient(
-            value.get("project"), value.get("api_url"), value.get("token"), value.get("kubeconfig_path")
+    clients = []
+    clusters = control_plane.setdefault("additional_clusters", [])
+    for value in clusters:
+        clients.append(
+            OpenShiftClient(
+                value.get("project"), value.get("api_url"), value.get("token"), value.get("kubeconfig_path")
+            )
         )
     if len(clients) > 0:
-        control_plane["spokes"] = clients
+        control_plane["additional_clusters"] = clients

testsuite/tests/conftest.py

@@ -6,23 +6,16 @@
 import pytest
 from dynaconf import ValidationError
 from keycloak import KeycloakAuthenticationError
-from openshift_client import selector, OpenShiftPythonException
 
-from testsuite.backend.httpbin import Httpbin
 from testsuite.capabilities import has_kuadrant
 from testsuite.certificates import CFSSLClient
 from testsuite.config import settings
-from testsuite.gateway import Gateway, GatewayRoute, Hostname, Exposer
-from testsuite.gateway.envoy import Envoy
-from testsuite.gateway.envoy.route import EnvoyVirtualRoute
-from testsuite.gateway.gateway_api.gateway import KuadrantGateway
-from testsuite.gateway.gateway_api.route import HTTPRoute
+from testsuite.gateway import Exposer, CustomReference
 from testsuite.httpx import KuadrantClient
 from testsuite.mockserver import Mockserver
 from testsuite.oidc import OIDCProvider
 from testsuite.oidc.auth0 import Auth0Provider
 from testsuite.oidc.keycloak import Keycloak
-from testsuite.openshift.kuadrant import KuadrantCR
 from testsuite.tracing import TracingClient
 from testsuite.utils import randomize, _whoami
 
@@ -117,32 +110,6 @@ def testconfig():
     return settings
 
 
-@pytest.fixture(scope="session")
-def hub_openshift(testconfig):
-    """OpenShift client for the primary namespace"""
-    client = testconfig["service_protection"]["project"]
-    if not client.connected:
-        pytest.fail("You are not logged into Openshift or the namespace doesn't exist")
-    return client
-
-
-@pytest.fixture(scope="session")
-def openshift(hub_openshift):
-    """OpenShift client for the primary namespace"""
-    return hub_openshift
-
-
-@pytest.fixture(scope="session")
-def openshift2(testconfig, skip_or_fail):
-    """OpenShift client for the secondary namespace located on the same cluster as primary Openshift"""
-    client = testconfig["service_protection"]["project2"]
-    if client is None:
-        skip_or_fail("Openshift2 required but second_project was not set")
-    if not client.connected:
-        pytest.fail("You are not logged into Openshift or the namespace for Openshift2 doesn't exist")
-    return client
-
-
 @pytest.fixture(scope="session")
 def keycloak(request, testconfig, blame, skip_or_fail):
     """Keycloak OIDC Provider fixture"""
@@ -263,65 +230,12 @@ def module_label(label):
 
 
 @pytest.fixture(scope="session")
-def kuadrant(request, testconfig):
-    """Returns Kuadrant instance if exists, or None"""
-    if request.config.getoption("--standalone"):
-        return None
-
-    ocp = testconfig["service_protection"]["project"]
-    project = testconfig["service_protection"]["system_project"]
-    kuadrant_openshift = ocp.change_project(project)
-
-    try:
-        with kuadrant_openshift.context:
-            kuadrant = selector("kuadrant").object(cls=KuadrantCR)
-    except OpenShiftPythonException:
-        pytest.fail("Running Kuadrant tests, but Kuadrant resource was not found")
-
-    return kuadrant
-
-
-@pytest.fixture(scope="session")
-def backend(request, openshift, blame, label):
-    """Deploys Httpbin backend"""
-    httpbin = Httpbin(openshift, blame("httpbin"), label)
-    request.addfinalizer(httpbin.delete)
-    httpbin.commit()
-    return httpbin
-
-
-@pytest.fixture(scope="session")
-def gateway(request, kuadrant, openshift, blame, label, testconfig, wildcard_domain) -> Gateway:
-    """Deploys Gateway that wires up the Backend behind the reverse-proxy and Authorino instance"""
-    if kuadrant:
-        gw = KuadrantGateway.create_instance(openshift, blame("gw"), wildcard_domain, {"app": label})
-    else:
-        authorino = request.getfixturevalue("authorino")
-        gw = Envoy(
-            openshift,
-            blame("gw"),
-            authorino,
-            testconfig["service_protection"]["envoy"]["image"],
-            labels={"app": label},
-        )
-    request.addfinalizer(gw.delete)
-    gw.commit()
-    gw.wait_for_ready()
-    return gw
-
-
-@pytest.fixture(scope="module")
-def route(request, kuadrant, gateway, blame, hostname, backend, module_label) -> GatewayRoute:
-    """Route object"""
-    if kuadrant:
-        route = HTTPRoute.create_instance(gateway.openshift, blame("route"), gateway, {"app": module_label})
-    else:
-        route = EnvoyVirtualRoute.create_instance(gateway.openshift, blame("route"), gateway)
-    route.add_hostname(hostname.hostname)
-    route.add_backend(backend)
-    request.addfinalizer(route.delete)
-    route.commit()
-    return route
+def hub_openshift(testconfig):
+    """OpenShift client for the primary namespace"""
+    client = testconfig["cluster"].change_project(testconfig["service_protection"]["project"])
+    if not client.connected:
+        pytest.fail("You are not logged into Openshift or the namespace doesn't exist")
+    return client
 
 
 @pytest.fixture(scope="session")
@@ -333,13 +247,6 @@ def exposer(request, testconfig, hub_openshift) -> Exposer:
     return exposer
 
 
-@pytest.fixture(scope="module")
-def hostname(gateway, exposer, blame) -> Hostname:
-    """Exposed Hostname object"""
-    hostname = exposer.expose_hostname(blame("hostname"), gateway)
-    return hostname
-
-
 @pytest.fixture(scope="session")
 def base_domain(exposer):
     """Returns preconfigured base domain"""
@@ -348,15 +255,15 @@ def base_domain(exposer):
 
 @pytest.fixture(scope="session")
 def wildcard_domain(base_domain):
-    """
-    Wildcard domain of openshift cluster
-    """
+    """Wildcard domain"""
     return f"*.{base_domain}"
 
 
-@pytest.fixture(scope="module")
-def client(route, hostname):  # pylint: disable=unused-argument
-    """Returns httpx client to be used for requests, it also commits AuthConfig"""
-    client = hostname.client()
-    yield client
-    client.close()
+@pytest.fixture(scope="session")
+def cluster_issuer(testconfig):
+    """Reference to cluster self-signed certificate issuer"""
+    return CustomReference(
+        group="cert-manager.io",
+        kind=testconfig["control_plane"]["issuer"]["kind"],
+        name=testconfig["control_plane"]["issuer"]["name"],
+    )