Merge pull request #535 from azgabur/gateway_listeners

Add Gateway listeners support and new test

testsuite/gateway/__init__.py

@@ -177,6 +177,44 @@ def remove_all_backend(self):
         """Sets match for a specific backend"""
 
 
+@dataclass
+class GatewayListener:
+    """
+    Dataclass of Gateway listener object.
+    When used in `add_listener()` function you MUST specify a unique name!
+    """
+
+    hostname: str
+    name: str = "api"
+    port: int = 80
+    protocol: str = "HTTP"
+    allowedRoutes = {"namespaces": {"from": "All"}}
+
+
+@dataclass(kw_only=True)
+class TLSGatewayListener(GatewayListener):
+    """
+    Dataclass for Gateway listener with TLS support.
+    When used in `add_listener()` function you MUST specify a unique name!
+    """
+
+    gateway_name: str
+    mode: str = "Terminate"
+    port: int = 443
+    protocol: str = "HTTPS"
+
+    def asdict(self):
+        """Custom asdict to easily add tls certificateRefs"""
+        return {
+            "name": self.name,
+            "hostname": self.hostname,
+            "port": self.port,
+            "protocol": self.protocol,
+            "allowedRoutes": self.allowedRoutes,
+            "tls": {"mode": self.mode, "certificateRefs": [{"name": f"{self.gateway_name}-tls", "kind": "Secret"}]},
+        }
+
+
 class Hostname(ABC):
     """
     Abstraction layer on top of externally exposed hostname

testsuite/gateway/gateway_api/gateway.py

@@ -5,66 +5,45 @@
 import openshift_client as oc
 
 from testsuite.certificates import Certificate
-from testsuite.gateway import Gateway
+from testsuite.gateway import Gateway, GatewayListener
 from testsuite.kubernetes.client import KubernetesClient
-from testsuite.kubernetes import KubernetesObject
+from testsuite.kubernetes import KubernetesObject, modify
 from testsuite.kuadrant.policy import Policy
-from testsuite.utils import check_condition
+from testsuite.utils import check_condition, asdict
 
 
 class KuadrantGateway(KubernetesObject, Gateway):
     """Gateway object for Kuadrant"""
 
     @classmethod
-    def create_instance(cls, cluster: KubernetesClient, name, hostname, labels, tls=False):
+    def create_instance(cls, cluster: KubernetesClient, name, labels):
         """Creates new instance of Gateway"""
 
         model: dict[Any, Any] = {
             "apiVersion": "gateway.networking.k8s.io/v1beta1",
             "kind": "Gateway",
             "metadata": {"name": name, "labels": labels},
-            "spec": {
-                "gatewayClassName": "istio",
-                "listeners": [
-                    {
-                        "name": "api",
-                        "port": 80,
-                        "protocol": "HTTP",
-                        "hostname": hostname,
-                        "allowedRoutes": {"namespaces": {"from": "All"}},
-                    }
-                ],
-            },
+            "spec": {"gatewayClassName": "istio", "listeners": []},
         }
-
-        if tls:
-            model["spec"]["listeners"] = [
-                {
-                    "name": "api",
-                    "port": 443,
-                    "protocol": "HTTPS",
-                    "hostname": hostname,
-                    "allowedRoutes": {"namespaces": {"from": "All"}},
-                    "tls": {
-                        "mode": "Terminate",
-                        "certificateRefs": [{"name": f"{name}-tls", "kind": "Secret"}],
-                    },
-                }
-            ]
-
-        return cls(model, context=cluster.context)
-
-    def add_listener(self, name: str, hostname: str):
-        """Adds new listener to the Gateway"""
-        self.model.spec.listeners.append(
-            {
-                "name": name,
-                "port": 80,
-                "protocol": "HTTP",
-                "hostname": hostname,
-                "allowedRoutes": {"namespaces": {"from": "All"}},
-            }
+        gateway = cls(model, context=cluster.context)
+        return gateway
+
+    @modify
+    def add_listener(self, listener: GatewayListener):
+        """Adds a listener to Gateway."""
+        self.model.spec.listeners.append(asdict(listener))
+
+    @modify
+    def remove_listener(self, listener_name: str):
+        """Removes a listener from Gateway."""
+        self.model.spec.listeners = list(filter(lambda i: i["name"] != listener_name, self.model.spec.listeners))
+
+    def get_listener_dns_ttl(self, listener_name: str) -> int:
+        """Returns TTL stored in DNSRecord CR under the specified Listener."""
+        dns_record = self.cluster.do_action(
+            "get", ["-o", "yaml", f"dnsrecords.kuadrant.io/{self.name()}-{listener_name}"], parse_output=True
         )
+        return dns_record.model.spec.endpoints[0].recordTTL
 
     @property
     def service_name(self) -> str:

testsuite/httpx/__init__.py

@@ -50,6 +50,7 @@ def should_backoff(self):
             or (self.error is None and self.status_code in self.retry_codes)
             or self.has_error("Server disconnected without sending a response.")
             or self.has_error("timed out")
+            or self.has_error("SSL: UNEXPECTED_EOF_WHILE_READING")
         )
 
     def has_error(self, error_msg: str) -> bool:

testsuite/tests/multicluster/conftest.py

@@ -8,6 +8,7 @@
 from testsuite.backend.httpbin import Httpbin
 from testsuite.certificates import Certificate
 from testsuite.gateway import Exposer, CustomReference, Hostname
+from testsuite.gateway import TLSGatewayListener
 from testsuite.gateway.gateway_api.gateway import KuadrantGateway
 from testsuite.gateway.gateway_api.hostname import DNSPolicyExposer
 from testsuite.gateway.gateway_api.route import HTTPRoute
@@ -106,7 +107,9 @@ def routes(request, gateway, gateway2, blame, hostname, backends, module_label)
 @pytest.fixture(scope="module")
 def gateway(request, cluster, blame, label, wildcard_domain):
     """Deploys Gateway to first Kubernetes cluster"""
-    gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": label}, tls=True)
+    name = blame("gw")
+    gw = KuadrantGateway.create_instance(cluster, name, {"app": label})
+    gw.add_listener(TLSGatewayListener(hostname=wildcard_domain, gateway_name=name))
     request.addfinalizer(gw.delete)
     gw.commit()
     gw.wait_for_ready()
@@ -116,7 +119,9 @@ def gateway(request, cluster, blame, label, wildcard_domain):
 @pytest.fixture(scope="module")
 def gateway2(request, cluster2, blame, label, wildcard_domain):
     """Deploys Gateway to second Kubernetes cluster"""
-    gw = KuadrantGateway.create_instance(cluster2, blame("gw"), wildcard_domain, {"app": label}, tls=True)
+    name = blame("gw")
+    gw = KuadrantGateway.create_instance(cluster2, name, {"app": label})
+    gw.add_listener(TLSGatewayListener(hostname=wildcard_domain, gateway_name=name))
     request.addfinalizer(gw.delete)
     gw.commit()
     gw.wait_for_ready()

testsuite/tests/singlecluster/conftest.py

@@ -6,7 +6,7 @@
 from openshift_client import selector
 
 from testsuite.backend.httpbin import Httpbin
-from testsuite.gateway import GatewayRoute, Gateway, Hostname
+from testsuite.gateway import GatewayRoute, Gateway, Hostname, GatewayListener
 from testsuite.gateway.envoy import Envoy
 from testsuite.gateway.envoy.route import EnvoyVirtualRoute
 from testsuite.gateway.gateway_api.gateway import KuadrantGateway
@@ -130,7 +130,8 @@ def backend(request, cluster, blame, label, testconfig):
 def gateway(request, kuadrant, cluster, blame, label, testconfig, wildcard_domain) -> Gateway:
     """Deploys Gateway that wires up the Backend behind the reverse-proxy and Authorino instance"""
     if kuadrant:
-        gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": label})
+        gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": label})
+        gw.add_listener(GatewayListener(wildcard_domain))
     else:
         authorino = request.getfixturevalue("authorino")
         gw = Envoy(

testsuite/tests/singlecluster/gateway/conftest.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from testsuite.gateway import Exposer
+from testsuite.gateway import Exposer, TLSGatewayListener
 from testsuite.gateway.gateway_api.gateway import KuadrantGateway
 from testsuite.gateway.gateway_api.hostname import DNSPolicyExposer
 from testsuite.httpx.auth import HttpxOidcClientAuth
@@ -13,7 +13,13 @@
 @pytest.fixture(scope="module")
 def gateway(request, cluster, blame, wildcard_domain, module_label):
     """Returns ready gateway"""
-    gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": module_label}, tls=True)
+    gateway_name = blame("gw")
+    gw = KuadrantGateway.create_instance(
+        cluster,
+        gateway_name,
+        {"app": module_label},
+    )
+    gw.add_listener(TLSGatewayListener(hostname=wildcard_domain, gateway_name=gateway_name))
     request.addfinalizer(gw.delete)
     gw.commit()
     gw.wait_for_ready()

testsuite/tests/singlecluster/gateway/dnspolicy/test_dnspolicy_removal.py

@@ -4,15 +4,16 @@
 
 import pytest
 
-from testsuite.gateway.gateway_api.gateway import KuadrantGateway
+from testsuite.gateway.gateway_api.gateway import KuadrantGateway, GatewayListener
 
 pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy]
 
 
 @pytest.fixture(scope="module")
 def gateway(request, cluster, blame, wildcard_domain, module_label):
     """Create gateway without TLS enabled"""
-    gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": module_label}, tls=False)
+    gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": module_label})
+    gw.add_listener(GatewayListener(wildcard_domain))
     request.addfinalizer(gw.delete)
     gw.commit()
     gw.wait_for_ready()
@@ -38,8 +39,10 @@ def test_dnspolicy_removal(gateway, dns_policy, client):
     response = client.get("/get")
     assert response.status_code == 200
 
+    dns_ttl = gateway.get_listener_dns_ttl(GatewayListener.name)
     dns_policy.delete()
-    sleep(60)  # wait for records deletion/ttl expiration from the previous request
+    # wait for records deletion/ttl expiration from the previous request
+    sleep(dns_ttl)
 
     assert not gateway.refresh().is_affected_by(dns_policy)
     response = client.get("/get")

testsuite/tests/singlecluster/gateway/dnspolicy/test_invalid_credentials.py

@@ -5,15 +5,16 @@
 from testsuite.kubernetes.secret import Secret
 from testsuite.kuadrant.policy import has_condition
 from testsuite.kuadrant.policy.dns import has_record_condition
-from testsuite.gateway.gateway_api.gateway import KuadrantGateway
+from testsuite.gateway.gateway_api.gateway import KuadrantGateway, GatewayListener
 
 pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy]
 
 
 @pytest.fixture(scope="module")
 def gateway(request, cluster, blame, wildcard_domain, module_label):
     """Create gateway without TLS enabled"""
-    gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": module_label}, tls=False)
+    gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": module_label})
+    gw.add_listener(GatewayListener(wildcard_domain, name="api"))
     request.addfinalizer(gw.delete)
     gw.commit()
     gw.wait_for_ready()

testsuite/tests/singlecluster/gateway/reconciliation/listeners/conftest.py

@@ -0,0 +1,59 @@
+"""
+Conftest for Gateway listeners tests.
+The main change consists of replacing the default wildcard domain for an exact one.
+"""
+
+import pytest
+
+from testsuite.gateway.gateway_api.hostname import StaticHostname
+
+
+@pytest.fixture(scope="module")
+def wildcard_domain(base_domain, blame):
+    """
+    For these tests we want specific default domain, not wildcard.
+    """
+    return f'{blame("prefix1")}.{base_domain}'
+
+
+@pytest.fixture(scope="module")
+def second_domain(base_domain, blame):
+    """Second domain string, not used in any object yet. To be assigned inside test."""
+    return f'{blame("prefix2")}.{base_domain}'
+
+
+@pytest.fixture(scope="module")
+def custom_client(gateway):
+    """
+    While changing TLS listeners the TLS certificate changes so a new client needs to be generated
+    to fetch newest tls cert from cluster.
+    """
+
+    def _client_new(hostname: str):
+        return StaticHostname(hostname, gateway.get_tls_cert).client()
+
+    return _client_new
+
+
+@pytest.fixture(scope="module")
+def check_ok_https(custom_client, auth):
+    """
+    Assert that HTTPS connection to domain works and returns 200. Authorization is used.
+    Assert that no DNS and TLS errors happened.
+    """
+
+    def _check_ok_https(domain: str):
+        response = custom_client(domain).get("/get", auth=auth)
+        assert not response.has_dns_error()
+        assert not response.has_cert_verify_error()
+        assert response.status_code == 200
+
+    return _check_ok_https
+
+
+@pytest.fixture(scope="module")
+def route(route, wildcard_domain):
+    """Ensure that route hostname matches the gateway hostname."""
+    route.remove_all_hostnames()
+    route.add_hostname(wildcard_domain)
+    return route

testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_basic_listeners.py

@@ -0,0 +1,46 @@
+"""
+Test case:
+- Add new listener and add it to HTTPRoute and test both work
+- Remove the new listener and remove it from HTTPRoute and test removed one is not working
+"""
+
+from time import sleep
+import pytest
+
+from testsuite.gateway import TLSGatewayListener
+from testsuite.utils import is_nxdomain
+
+
+pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy, pytest.mark.tlspolicy]
+
+LISTENER_NAME = "api-second"
+
+
+def test_listeners(custom_client, check_ok_https, gateway, route, wildcard_domain, second_domain):
+    """
+    This test checks reconciliation of dns/tls policy on addition and removal of listeners in gateway and HTTPRoute.
+    """
+
+    # Check the default domain works and second domain does not exist yet
+    check_ok_https(wildcard_domain)
+    assert is_nxdomain(second_domain)
+    assert custom_client(second_domain).get("/get").has_dns_error()
+
+    # Add second domain to gateway and route
+    gateway.add_listener(TLSGatewayListener(hostname=second_domain, gateway_name=gateway.name(), name=LISTENER_NAME))
+    route.add_hostname(second_domain)
+
+    # Check both domains work
+    for domain in [wildcard_domain, second_domain]:
+        check_ok_https(domain)
+
+    # Remove second domain, store TTL value of to be removed DNS record
+    second_domain_ttl = gateway.get_listener_dns_ttl(LISTENER_NAME)
+    route.remove_hostname(second_domain)
+    gateway.remove_listener(LISTENER_NAME)
+
+    # Check the default domain still works and second domain does not exist anymore
+    sleep(second_domain_ttl)
+    check_ok_https(wildcard_domain)
+    assert is_nxdomain(second_domain)
+    assert custom_client(second_domain).get("/get").has_dns_error()