Merge pull request #320 from pehala/remove_templates_envoy

Migrate Envoy to Deployment object

config/settings.yaml

@@ -16,7 +16,7 @@ default:
     project: "kuadrant"
     project2: "kuadrant2"
     envoy:
-      image: "docker.io/envoyproxy/envoy:v1.23-latest"
+      image: "quay.io/phala/envoy:v1.28-latest"
     gateway:
        project: "istio-system"
        name: "istio-ingressgateway"

testsuite/gateway/envoy/__init__.py

@@ -1,14 +1,16 @@
 """Envoy Gateway module"""
 import time
-from importlib import resources
 from typing import Optional
 
 import openshift as oc
 
 from testsuite.certificates import Certificate
 from testsuite.gateway import Gateway
+from testsuite.openshift import Selector
 from testsuite.openshift.client import OpenShiftClient
 from testsuite.gateway.envoy.config import EnvoyConfig
+from testsuite.openshift.deployment import Deployment, VolumeMount, ConfigMapVolume
+from testsuite.openshift.service import Service, ServicePort
 
 
 class Envoy(Gateway):  # pylint: disable=too-many-instance-attributes
@@ -24,9 +26,10 @@ def __init__(self, openshift: "OpenShiftClient", name, authorino, image, labels:
         self.name = name
         self.image = image
         self.labels = labels
-        self.template = resources.files("testsuite.resources").joinpath("envoy.yaml")
+
+        self.deployment = None
+        self.service = None
         self._config = None
-        self.envoy_objects: oc.Selector = None  # type: ignore
 
     @property
     def config(self):
@@ -35,11 +38,6 @@ def config(self):
             self._config = EnvoyConfig.create_instance(self.openshift, self.name, self.authorino, self.labels)
         return self._config
 
-    @property
-    def app_label(self):
-        """Returns App label that should be applied to all resources"""
-        return self.labels.get("app")
-
     @property
     def openshift(self) -> "OpenShiftClient":
         return self._openshift
@@ -60,17 +58,41 @@ def wait_for_ready(self, timeout: int = 180):
                 "rollout", ["status", f"deployment/{self.name}"]
             ), "Envoy wasn't ready in time"
 
+    def create_deployment(self) -> Deployment:
+        """Creates Deployment object for Envoy, which is then committed"""
+        return Deployment.create_instance(
+            self.openshift,
+            self.name,
+            container_name="envoy",
+            image=self.image,
+            ports={"api": 8080, "admin": 8001},
+            selector=Selector(matchLabels={"deployment": self.name, **self.labels}),
+            labels=self.labels,
+            command_args=[
+                "--config-path /usr/local/etc/envoy/envoy.yaml",
+                "--log-level info",
+                "--component-log-level filter:trace,http:debug,router:debug",
+            ],
+            volumes=[ConfigMapVolume(config_map_name=self.name, name="config", items={"envoy.yaml": "envoy.yaml"})],
+            volume_mounts=[VolumeMount(mountPath="/usr/local/etc/envoy", name="config", readOnly=True)],
+            readiness_probe={"httpGet": {"path": "/ready", "port": 8001}, "initialDelaySeconds": 3, "periodSeconds": 4},
+        )
+
     def commit(self):
         """Deploy all required objects into OpenShift"""
         self.config.commit()
-        self.envoy_objects = self.openshift.new_app(
-            self.template,
-            {
-                "NAME": self.name,
-                "LABEL": self.app_label,
-                "ENVOY_IMAGE": self.image,
-            },
+
+        self.deployment = self.create_deployment()
+        self.deployment.commit()
+        self.deployment.wait_for_ready()
+
+        self.service = Service.create_instance(
+            self.openshift,
+            self.name,
+            selector={"deployment": self.name, **self.labels},
+            ports=[ServicePort(name="api", port=8080, targetPort="api")],
         )
+        self.service.commit()
 
     def get_tls_cert(self) -> Optional[Certificate]:
         return None
@@ -80,6 +102,9 @@ def delete(self):
         self.config.delete()
         self._config = None
         with self.openshift.context:
-            if self.envoy_objects:
-                self.envoy_objects.delete()
-        self.envoy_objects = None
+            if self.deployment:
+                self.deployment.delete()
+                self.deployment = None
+            if self.service:
+                self.service.delete()
+                self.service = None

testsuite/gateway/envoy/config.py

@@ -12,7 +12,7 @@
         - address:
             socket_address:
               address: 0.0.0.0
-              port_value: 8000
+              port_value: 8080
           filter_chains:
           - filters:
             - name: envoy.http_connection_manager

testsuite/gateway/envoy/tls.py

@@ -1,9 +1,9 @@
 """Envoy Gateway implementation with TLS setup"""
-from importlib import resources
 from typing import TYPE_CHECKING
 
 import yaml
 
+from testsuite.openshift.deployment import Deployment, SecretVolume, VolumeMount
 from . import Envoy, EnvoyConfig
 
 if TYPE_CHECKING:
@@ -30,7 +30,7 @@
   common_tls_context:
     validation_context:
       trusted_ca:
-        filename: /etc/ssl/certs/authorino/tls.crt
+        filename: /etc/ssl/certs/authorino-ca/tls.crt
 """
 
 
@@ -67,19 +67,14 @@ def config(self):
             self._config["envoy.yaml"] = yaml.dump(config)
         return self._config
 
-    def commit(self):
-        self.config.commit()
-        self.envoy_objects = self.openshift.new_app(
-            resources.files("testsuite.resources.tls").joinpath("envoy.yaml"),
-            {
-                "NAME": self.name,
-                "LABEL": self.app_label,
-                "AUTHORINO_CA_SECRET": self.authorino_ca_secret,
-                "ENVOY_CA_SECRET": self.backend_ca_secret,
-                "ENVOY_CERT_SECRET": self.envoy_cert_secret,
-                "ENVOY_IMAGE": self.image,
-            },
-        )
+    def create_deployment(self) -> Deployment:
+        deployment = super().create_deployment()
+        deployment.add_volume(SecretVolume(secret_name=self.authorino_ca_secret, name="authorino-ca", readOnly=True))
+        deployment.add_volume(SecretVolume(secret_name=self.backend_ca_secret, name="envoy-ca", readOnly=True))
+        deployment.add_volume(SecretVolume(secret_name=self.envoy_cert_secret, name="envoy-cert", readOnly=True))
 
-        with self.openshift.context:
-            assert self.openshift.is_ready(self.envoy_objects.narrow("deployment")), "Envoy wasn't ready in time"
+        deployment.add_mount(VolumeMount(mountPath="/etc/ssl/certs/authorino-ca", name="authorino-ca"))
+        deployment.add_mount(VolumeMount(mountPath="/etc/ssl/certs/envoy-ca", name="envoy-ca"))
+        deployment.add_mount(VolumeMount(mountPath="/etc/ssl/certs/envoy", name="envoy-cert"))
+
+        return deployment

testsuite/openshift/deployment.py

@@ -1,16 +1,76 @@
 """Deployment related objects"""
+from dataclasses import dataclass
+from typing import Any
+
 import openshift as oc
 
-from testsuite.openshift import OpenShiftObject, Selector
+from testsuite.openshift import OpenShiftObject, Selector, modify
 from testsuite.utils import asdict
 
+# pylint: disable=invalid-name
+
+
+@dataclass
+class VolumeMount:
+    """Deployment VolumeMount object"""
+
+    mountPath: str
+    name: str
+    readOnly: bool = True
+
+
+@dataclass
+class ConfigMapVolume:
+    """Deployment ConfigMapVolume object"""
+
+    config_map_name: str
+    items: dict[str, str]
+    name: str
+
+    def asdict(self):
+        """Custom asdict because of needing to put location as parent dict key for inner dict"""
+        return {
+            "configMap": {
+                "items": [{"key": key, "path": value} for key, value in self.items.items()],
+                "name": self.config_map_name,
+            },
+            "name": self.name,
+        }
+
+
+@dataclass
+class SecretVolume:
+    """Deployment SecretVolume object"""
+
+    secret_name: str
+    name: str
+    readOnly: bool = True
+
+    def asdict(self):
+        """Custom asdict because of needing to put location as parent dict key for inner dict"""
+        return {"secret": {"readOnly": self.readOnly, "secretName": self.secret_name}, "name": self.name}
+
+
+Volume = SecretVolume | ConfigMapVolume
+
 
 class Deployment(OpenShiftObject):
     """Kubernetes Deployment object"""
 
     @classmethod
     def create_instance(
-        cls, openshift, name, container_name, image, ports: dict[str, int], selector: Selector, labels: dict[str, str]
+        cls,
+        openshift,
+        name,
+        container_name,
+        image,
+        ports: dict[str, int],
+        selector: Selector,
+        labels: dict[str, str],
+        command_args: list[str] = None,
+        volumes: list[Volume] = None,
+        volume_mounts: list[VolumeMount] = None,
+        readiness_probe: dict[str, Any] = None,
     ):
         """
         Creates new instance of Deployment
@@ -40,6 +100,21 @@ def create_instance(
                 },
             },
         }
+        template = model["spec"]["template"]["spec"]
+
+        if volumes:
+            template["volumes"] = [asdict(volume) for volume in volumes]
+
+        container = template["containers"][0]
+
+        if command_args:
+            container["args"] = command_args
+
+        if volume_mounts:
+            container["volumeMounts"] = [asdict(mount) for mount in volume_mounts]
+
+        if readiness_probe:
+            container["readinessProbe"] = readiness_probe
 
         return cls(model, context=openshift.context)
 
@@ -48,3 +123,25 @@ def wait_for_ready(self, timeout=90):
         with oc.timeout(timeout):
             success, _, _ = self.self_selector().until_all(success_func=lambda obj: "readyReplicas" in obj.model.status)
             assert success, f"Deployment {self.name()} did not get ready in time"
+
+    @property
+    def template(self):
+        """Returns spec.template.spec part of Deployment"""
+        return self.model.spec.template.spec
+
+    @property
+    def container(self):
+        """Returns spec.template.spec.container[0] part of Deployment"""
+        return self.template.containers[0]
+
+    @modify
+    def add_mount(self, mount: VolumeMount):
+        """Adds volume mount"""
+        mounts = self.container.setdefault("volumeMounts", [])
+        mounts.append(asdict(mount))
+
+    @modify
+    def add_volume(self, volume: Volume):
+        """Adds volume"""
+        mounts = self.template.setdefault("volumes", [])
+        mounts.append(asdict(volume))