diff --git a/testsuite/gateway/__init__.py b/testsuite/gateway/__init__.py index 452f803a..db48e294 100644 --- a/testsuite/gateway/__init__.py +++ b/testsuite/gateway/__init__.py @@ -177,6 +177,44 @@ def remove_all_backend(self): """Sets match for a specific backend""" +@dataclass +class GatewayListener: + """ + Dataclass of Gateway listener object. + When used in `add_listener()` function you MUST specify a unique name! + """ + + hostname: str + name: str = "api" + port: int = 80 + protocol: str = "HTTP" + allowedRoutes = {"namespaces": {"from": "All"}} + + +@dataclass(kw_only=True) +class TLSGatewayListener(GatewayListener): + """ + Dataclass for Gateway listener with TLS support. + When used in `add_listener()` function you MUST specify a unique name! + """ + + gateway_name: str + mode: str = "Terminate" + port: int = 443 + protocol: str = "HTTPS" + + def asdict(self): + """Custom asdict to easily add tls certificateRefs""" + return { + "name": self.name, + "hostname": self.hostname, + "port": self.port, + "protocol": self.protocol, + "allowedRoutes": self.allowedRoutes, + "tls": {"mode": self.mode, "certificateRefs": [{"name": f"{self.gateway_name}-tls", "kind": "Secret"}]}, + } + + class Hostname(ABC): """ Abstraction layer on top of externally exposed hostname diff --git a/testsuite/gateway/gateway_api/gateway.py b/testsuite/gateway/gateway_api/gateway.py index 9b4e9c93..e699ef77 100644 --- a/testsuite/gateway/gateway_api/gateway.py +++ b/testsuite/gateway/gateway_api/gateway.py @@ -5,66 +5,45 @@ import openshift_client as oc from testsuite.certificates import Certificate -from testsuite.gateway import Gateway +from testsuite.gateway import Gateway, GatewayListener from testsuite.kubernetes.client import KubernetesClient -from testsuite.kubernetes import KubernetesObject +from testsuite.kubernetes import KubernetesObject, modify from testsuite.kuadrant.policy import Policy -from testsuite.utils import check_condition +from testsuite.utils import check_condition, asdict class KuadrantGateway(KubernetesObject, Gateway): """Gateway object for Kuadrant""" @classmethod - def create_instance(cls, cluster: KubernetesClient, name, hostname, labels, tls=False): + def create_instance(cls, cluster: KubernetesClient, name, labels): """Creates new instance of Gateway""" model: dict[Any, Any] = { "apiVersion": "gateway.networking.k8s.io/v1beta1", "kind": "Gateway", "metadata": {"name": name, "labels": labels}, - "spec": { - "gatewayClassName": "istio", - "listeners": [ - { - "name": "api", - "port": 80, - "protocol": "HTTP", - "hostname": hostname, - "allowedRoutes": {"namespaces": {"from": "All"}}, - } - ], - }, + "spec": {"gatewayClassName": "istio", "listeners": []}, } - - if tls: - model["spec"]["listeners"] = [ - { - "name": "api", - "port": 443, - "protocol": "HTTPS", - "hostname": hostname, - "allowedRoutes": {"namespaces": {"from": "All"}}, - "tls": { - "mode": "Terminate", - "certificateRefs": [{"name": f"{name}-tls", "kind": "Secret"}], - }, - } - ] - - return cls(model, context=cluster.context) - - def add_listener(self, name: str, hostname: str): - """Adds new listener to the Gateway""" - self.model.spec.listeners.append( - { - "name": name, - "port": 80, - "protocol": "HTTP", - "hostname": hostname, - "allowedRoutes": {"namespaces": {"from": "All"}}, - } + gateway = cls(model, context=cluster.context) + return gateway + + @modify + def add_listener(self, listener: GatewayListener): + """Adds a listener to Gateway.""" + self.model.spec.listeners.append(asdict(listener)) + + @modify + def remove_listener(self, listener_name: str): + """Removes a listener from Gateway.""" + self.model.spec.listeners = list(filter(lambda i: i["name"] != listener_name, self.model.spec.listeners)) + + def get_listener_dns_ttl(self, listener_name: str) -> int: + """Returns TTL stored in DNSRecord CR under the specified Listener.""" + dns_record = self.cluster.do_action( + "get", ["-o", "yaml", f"dnsrecords.kuadrant.io/{self.name()}-{listener_name}"], parse_output=True ) + return dns_record.model.spec.endpoints[0].recordTTL @property def service_name(self) -> str: diff --git a/testsuite/httpx/__init__.py b/testsuite/httpx/__init__.py index 05c06104..90e25ac6 100644 --- a/testsuite/httpx/__init__.py +++ b/testsuite/httpx/__init__.py @@ -50,6 +50,7 @@ def should_backoff(self): or (self.error is None and self.status_code in self.retry_codes) or self.has_error("Server disconnected without sending a response.") or self.has_error("timed out") + or self.has_error("SSL: UNEXPECTED_EOF_WHILE_READING") ) def has_error(self, error_msg: str) -> bool: diff --git a/testsuite/tests/multicluster/conftest.py b/testsuite/tests/multicluster/conftest.py index 0df5fe24..784fa5fc 100644 --- a/testsuite/tests/multicluster/conftest.py +++ b/testsuite/tests/multicluster/conftest.py @@ -8,6 +8,7 @@ from testsuite.backend.httpbin import Httpbin from testsuite.certificates import Certificate from testsuite.gateway import Exposer, CustomReference, Hostname +from testsuite.gateway import TLSGatewayListener from testsuite.gateway.gateway_api.gateway import KuadrantGateway from testsuite.gateway.gateway_api.hostname import DNSPolicyExposer from testsuite.gateway.gateway_api.route import HTTPRoute @@ -106,7 +107,9 @@ def routes(request, gateway, gateway2, blame, hostname, backends, module_label) @pytest.fixture(scope="module") def gateway(request, cluster, blame, label, wildcard_domain): """Deploys Gateway to first Kubernetes cluster""" - gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": label}, tls=True) + name = blame("gw") + gw = KuadrantGateway.create_instance(cluster, name, {"app": label}) + gw.add_listener(TLSGatewayListener(hostname=wildcard_domain, gateway_name=name)) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready() @@ -116,7 +119,9 @@ def gateway(request, cluster, blame, label, wildcard_domain): @pytest.fixture(scope="module") def gateway2(request, cluster2, blame, label, wildcard_domain): """Deploys Gateway to second Kubernetes cluster""" - gw = KuadrantGateway.create_instance(cluster2, blame("gw"), wildcard_domain, {"app": label}, tls=True) + name = blame("gw") + gw = KuadrantGateway.create_instance(cluster2, name, {"app": label}) + gw.add_listener(TLSGatewayListener(hostname=wildcard_domain, gateway_name=name)) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready() diff --git a/testsuite/tests/singlecluster/conftest.py b/testsuite/tests/singlecluster/conftest.py index 3c57dcaa..4c6534e5 100644 --- a/testsuite/tests/singlecluster/conftest.py +++ b/testsuite/tests/singlecluster/conftest.py @@ -6,7 +6,7 @@ from openshift_client import selector from testsuite.backend.httpbin import Httpbin -from testsuite.gateway import GatewayRoute, Gateway, Hostname +from testsuite.gateway import GatewayRoute, Gateway, Hostname, GatewayListener from testsuite.gateway.envoy import Envoy from testsuite.gateway.envoy.route import EnvoyVirtualRoute from testsuite.gateway.gateway_api.gateway import KuadrantGateway @@ -130,7 +130,8 @@ def backend(request, cluster, blame, label, testconfig): def gateway(request, kuadrant, cluster, blame, label, testconfig, wildcard_domain) -> Gateway: """Deploys Gateway that wires up the Backend behind the reverse-proxy and Authorino instance""" if kuadrant: - gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": label}) + gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": label}) + gw.add_listener(GatewayListener(wildcard_domain)) else: authorino = request.getfixturevalue("authorino") gw = Envoy( diff --git a/testsuite/tests/singlecluster/gateway/conftest.py b/testsuite/tests/singlecluster/gateway/conftest.py index f79d8f73..b5f68b3c 100644 --- a/testsuite/tests/singlecluster/gateway/conftest.py +++ b/testsuite/tests/singlecluster/gateway/conftest.py @@ -2,7 +2,7 @@ import pytest -from testsuite.gateway import Exposer +from testsuite.gateway import Exposer, TLSGatewayListener from testsuite.gateway.gateway_api.gateway import KuadrantGateway from testsuite.gateway.gateway_api.hostname import DNSPolicyExposer from testsuite.httpx.auth import HttpxOidcClientAuth @@ -13,7 +13,13 @@ @pytest.fixture(scope="module") def gateway(request, cluster, blame, wildcard_domain, module_label): """Returns ready gateway""" - gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": module_label}, tls=True) + gateway_name = blame("gw") + gw = KuadrantGateway.create_instance( + cluster, + gateway_name, + {"app": module_label}, + ) + gw.add_listener(TLSGatewayListener(hostname=wildcard_domain, gateway_name=gateway_name)) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready() diff --git a/testsuite/tests/singlecluster/gateway/dnspolicy/test_dnspolicy_removal.py b/testsuite/tests/singlecluster/gateway/dnspolicy/test_dnspolicy_removal.py index 47bfad93..e7b14df3 100644 --- a/testsuite/tests/singlecluster/gateway/dnspolicy/test_dnspolicy_removal.py +++ b/testsuite/tests/singlecluster/gateway/dnspolicy/test_dnspolicy_removal.py @@ -4,7 +4,7 @@ import pytest -from testsuite.gateway.gateway_api.gateway import KuadrantGateway +from testsuite.gateway.gateway_api.gateway import KuadrantGateway, GatewayListener pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy] @@ -12,7 +12,8 @@ @pytest.fixture(scope="module") def gateway(request, cluster, blame, wildcard_domain, module_label): """Create gateway without TLS enabled""" - gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": module_label}, tls=False) + gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": module_label}) + gw.add_listener(GatewayListener(wildcard_domain)) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready() @@ -38,8 +39,10 @@ def test_dnspolicy_removal(gateway, dns_policy, client): response = client.get("/get") assert response.status_code == 200 + dns_ttl = gateway.get_listener_dns_ttl(GatewayListener.name) dns_policy.delete() - sleep(60) # wait for records deletion/ttl expiration from the previous request + # wait for records deletion/ttl expiration from the previous request + sleep(dns_ttl) assert not gateway.refresh().is_affected_by(dns_policy) response = client.get("/get") diff --git a/testsuite/tests/singlecluster/gateway/dnspolicy/test_invalid_credentials.py b/testsuite/tests/singlecluster/gateway/dnspolicy/test_invalid_credentials.py index 146cccb9..17403cb3 100644 --- a/testsuite/tests/singlecluster/gateway/dnspolicy/test_invalid_credentials.py +++ b/testsuite/tests/singlecluster/gateway/dnspolicy/test_invalid_credentials.py @@ -5,7 +5,7 @@ from testsuite.kubernetes.secret import Secret from testsuite.kuadrant.policy import has_condition from testsuite.kuadrant.policy.dns import has_record_condition -from testsuite.gateway.gateway_api.gateway import KuadrantGateway +from testsuite.gateway.gateway_api.gateway import KuadrantGateway, GatewayListener pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy] @@ -13,7 +13,8 @@ @pytest.fixture(scope="module") def gateway(request, cluster, blame, wildcard_domain, module_label): """Create gateway without TLS enabled""" - gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": module_label}, tls=False) + gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": module_label}) + gw.add_listener(GatewayListener(wildcard_domain, name="api")) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready() diff --git a/testsuite/tests/singlecluster/gateway/reconciliation/listeners/__init__.py b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testsuite/tests/singlecluster/gateway/reconciliation/listeners/conftest.py b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/conftest.py new file mode 100644 index 00000000..23213035 --- /dev/null +++ b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/conftest.py @@ -0,0 +1,59 @@ +""" +Conftest for Gateway listeners tests. +The main change consists of replacing the default wildcard domain for an exact one. +""" + +import pytest + +from testsuite.gateway.gateway_api.hostname import StaticHostname + + +@pytest.fixture(scope="module") +def wildcard_domain(base_domain, blame): + """ + For these tests we want specific default domain, not wildcard. + """ + return f'{blame("prefix1")}.{base_domain}' + + +@pytest.fixture(scope="module") +def second_domain(base_domain, blame): + """Second domain string, not used in any object yet. To be assigned inside test.""" + return f'{blame("prefix2")}.{base_domain}' + + +@pytest.fixture(scope="module") +def custom_client(gateway): + """ + While changing TLS listeners the TLS certificate changes so a new client needs to be generated + to fetch newest tls cert from cluster. + """ + + def _client_new(hostname: str): + return StaticHostname(hostname, gateway.get_tls_cert).client() + + return _client_new + + +@pytest.fixture(scope="module") +def check_ok_https(custom_client, auth): + """ + Assert that HTTPS connection to domain works and returns 200. Authorization is used. + Assert that no DNS and TLS errors happened. + """ + + def _check_ok_https(domain: str): + response = custom_client(domain).get("/get", auth=auth) + assert not response.has_dns_error() + assert not response.has_cert_verify_error() + assert response.status_code == 200 + + return _check_ok_https + + +@pytest.fixture(scope="module") +def route(route, wildcard_domain): + """Ensure that route hostname matches the gateway hostname.""" + route.remove_all_hostnames() + route.add_hostname(wildcard_domain) + return route diff --git a/testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_basic_listeners.py b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_basic_listeners.py new file mode 100644 index 00000000..e0a65df2 --- /dev/null +++ b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_basic_listeners.py @@ -0,0 +1,46 @@ +""" +Test case: +- Add new listener and add it to HTTPRoute and test both work +- Remove the new listener and remove it from HTTPRoute and test removed one is not working +""" + +from time import sleep +import pytest + +from testsuite.gateway import TLSGatewayListener +from testsuite.utils import is_nxdomain + + +pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy, pytest.mark.tlspolicy] + +LISTENER_NAME = "api-second" + + +def test_listeners(custom_client, check_ok_https, gateway, route, wildcard_domain, second_domain): + """ + This test checks reconciliation of dns/tls policy on addition and removal of listeners in gateway and HTTPRoute. + """ + + # Check the default domain works and second domain does not exist yet + check_ok_https(wildcard_domain) + assert is_nxdomain(second_domain) + assert custom_client(second_domain).get("/get").has_dns_error() + + # Add second domain to gateway and route + gateway.add_listener(TLSGatewayListener(hostname=second_domain, gateway_name=gateway.name(), name=LISTENER_NAME)) + route.add_hostname(second_domain) + + # Check both domains work + for domain in [wildcard_domain, second_domain]: + check_ok_https(domain) + + # Remove second domain, store TTL value of to be removed DNS record + second_domain_ttl = gateway.get_listener_dns_ttl(LISTENER_NAME) + route.remove_hostname(second_domain) + gateway.remove_listener(LISTENER_NAME) + + # Check the default domain still works and second domain does not exist anymore + sleep(second_domain_ttl) + check_ok_https(wildcard_domain) + assert is_nxdomain(second_domain) + assert custom_client(second_domain).get("/get").has_dns_error() diff --git a/testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_listeners_dns.py b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_listeners_dns.py new file mode 100644 index 00000000..ee5f9413 --- /dev/null +++ b/testsuite/tests/singlecluster/gateway/reconciliation/listeners/test_gateway_listeners_dns.py @@ -0,0 +1,34 @@ +"""Testing specific bug that happens when listener hostname in Gateway gets changed while DNSPolicy is applied.""" + +from time import sleep +import pytest + +from testsuite.gateway import TLSGatewayListener +from testsuite.utils import is_nxdomain + +pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy, pytest.mark.tlspolicy] + +DEFAULT_LISTENER_NAME = TLSGatewayListener.name + + +@pytest.mark.issue("https://github.com/Kuadrant/kuadrant-operator/issues/794") +def test_change_listener(custom_client, check_ok_https, gateway, route, second_domain, wildcard_domain): + """ + This test checks if after change of listener hostname in a Gateway while having DNSPolicy applied, that + the old hostname gets deleted from DNS provider. After editing the hostname in HTTPRoute to the new value + this test checks the reconciliation of such procedure. + """ + check_ok_https(wildcard_domain) + wildcard_domain_ttl = gateway.get_listener_dns_ttl(DEFAULT_LISTENER_NAME) + + gateway.remove_listener(DEFAULT_LISTENER_NAME) + route.remove_hostname(wildcard_domain) + gateway.add_listener( + TLSGatewayListener(hostname=second_domain, gateway_name=gateway.name(), name=DEFAULT_LISTENER_NAME) + ) + route.add_hostname(second_domain) + + sleep(wildcard_domain_ttl) + check_ok_https(second_domain) + assert is_nxdomain(wildcard_domain) + assert custom_client(wildcard_domain).get("/get").has_dns_error() diff --git a/testsuite/tests/singlecluster/gateway/reconciliation/test_gateway_listeners_dns.py b/testsuite/tests/singlecluster/gateway/reconciliation/test_gateway_listeners_dns.py deleted file mode 100644 index 87bb749c..00000000 --- a/testsuite/tests/singlecluster/gateway/reconciliation/test_gateway_listeners_dns.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Testing specific bug that happens when listener hostname in Gateway gets changed while DNSPolicy is applied.""" - -import pytest -from testsuite.gateway.gateway_api.hostname import StaticHostname -from testsuite.utils import is_nxdomain, sleep_ttl - -pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy, pytest.mark.tlspolicy] - - -@pytest.fixture(scope="module") -def wildcard_domain(base_domain, blame): - """ - For this test we want specific domain, not wildcard. - This will be used in the first iteration of Gateway and HTTPRoute. - """ - return f"{blame('dnsbug1')}.{base_domain}" - - -@pytest.fixture(scope="module") -def new_domain(base_domain, blame): - """In the test the Gateway and HTTPRoute will change their hostnames to this one.""" - return f"{blame('dnsbug2')}.{base_domain}" - - -@pytest.fixture(scope="module") -def client(gateway, wildcard_domain): - """Make client point to correct domain route domain.""" - return StaticHostname(wildcard_domain, gateway.get_tls_cert).client() - - -@pytest.fixture(scope="module") -def client_new(gateway, new_domain): - """Second client that will be created after Gateway gets updated pointing to new_domain.""" - - def _client_new(): - return StaticHostname(new_domain, gateway.get_tls_cert).client() - - return _client_new - - -@pytest.fixture(scope="module") -def route(route, wildcard_domain): - """So that route hostname matches the gateway hostname.""" - route.remove_all_hostnames() - route.add_hostname(wildcard_domain) - return route - - -@pytest.mark.issue("https://github.com/Kuadrant/kuadrant-operator/issues/794") -def test_change_hostname(client, client_new, auth, gateway, route, new_domain, wildcard_domain): - """ - This test checks if after change of listener hostname in a Gateway while having DNSPolicy applied, that - the old hostname gets deleted from DNS provider. After editing the hostname in HTTPRoute to the new value - this test checks the reconciliation of such procedure. - - WARNING - Running this test in unpatched Kuadrant will leave orphaned DNS records in DNS provider. - If you want to delete them you need to do it manually. The DNS records will contain string 'dnsbug' - """ - result = client.get("/get", auth=auth) - assert not result.has_dns_error() - assert not result.has_cert_verify_error() - assert result.status_code == 200 - - gateway.refresh().model.spec.listeners[0].hostname = new_domain - gateway.apply() - route.refresh().model.spec.hostnames[0] = new_domain - route.apply() - - result = client_new().get("/get", auth=auth) - assert not result.has_dns_error() - assert not result.has_cert_verify_error() - assert result.status_code == 200 - - sleep_ttl(wildcard_domain) - assert is_nxdomain(wildcard_domain) diff --git a/testsuite/tests/singlecluster/gateway/test_scale_listeners.py b/testsuite/tests/singlecluster/gateway/test_scale_listeners.py index 1670222d..75294d82 100644 --- a/testsuite/tests/singlecluster/gateway/test_scale_listeners.py +++ b/testsuite/tests/singlecluster/gateway/test_scale_listeners.py @@ -4,7 +4,7 @@ from testsuite.httpx import KuadrantClient from testsuite.gateway.gateway_api.route import HTTPRoute -from testsuite.gateway.gateway_api.gateway import KuadrantGateway +from testsuite.gateway.gateway_api.gateway import KuadrantGateway, GatewayListener from testsuite.kuadrant.policy.dns import DNSPolicy pytestmark = [pytest.mark.kuadrant_only, pytest.mark.dnspolicy] @@ -15,9 +15,10 @@ @pytest.fixture(scope="module") def gateway(request, cluster, blame, base_domain, module_label): """Create first gateway with 64 listeners""" - gw = KuadrantGateway.create_instance(cluster, blame("gw"), f"gw1-api.{base_domain}", {"app": module_label}) + gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": module_label}) + gw.add_listener(GatewayListener(f"gw1-api.{base_domain}")) for i in range(1, MAX_GATEWAY_LISTENERS): - gw.add_listener(f"api{i}", f"gw1-api{i}.{base_domain}") + gw.add_listener(GatewayListener(f"api{i}", f"gw1-api{i}.{base_domain}")) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready() @@ -27,9 +28,10 @@ def gateway(request, cluster, blame, base_domain, module_label): @pytest.fixture(scope="module") def gateway2(request, cluster, blame, base_domain, module_label): """Create second gateway with 64 listeners""" - gw = KuadrantGateway.create_instance(cluster, blame("gw"), f"gw2-api.{base_domain}", {"app": module_label}) + gw = KuadrantGateway.create_instance(cluster, blame("gw"), {"app": module_label}) + gw.add_listener(GatewayListener(f"gw2-api.{base_domain}")) for i in range(1, MAX_GATEWAY_LISTENERS): - gw.add_listener(f"api{i}", f"gw2-api{i}.{base_domain}") + gw.add_listener(GatewayListener(f"api{i}", f"gw2-api{i}.{base_domain}")) request.addfinalizer(gw.delete) gw.commit() gw.wait_for_ready()