Enable compliance tests to use plugins for cluster provisioning (#753)

Signed-off-by: Toni Finger <[email protected]>
Signed-off-by: Matthias Büchse <[email protected]>
Co-authored-by: Matthias Büchse <[email protected]>

Tests/config.toml

@@ -34,11 +34,51 @@ subjects = [
 workers = 4
 
 
+[presets.kaas-dev]
+scopes = [
+    "scs-compatible-kaas",
+]
+subjects = [
+    "kind-current",
+    "kind-current-1",
+    "kind-current-2",
+]
+workers = 1  # better restrict this with clusters running on local machine
+
+
 [scopes.scs-compatible-iaas]
 spec = "./scs-compatible-iaas.yaml"
 
 
+[scopes.scs-compatible-kaas]
+spec = "./scs-compatible-kaas.yaml"
+
+
 # default subject (not a real subject, but used to declare a default mapping)
 # (this is the only mapping declaration that supports using Python string interpolation)
 [subjects._.mapping]
 os_cloud = "{subject}"
+subject_root = "{subject}"
+
+
+[subjects._.kubernetes_setup]
+clusterspec         = "kaas/clusterspec.yaml"
+
+
+[subjects.kind-current.kubernetes_setup]
+kube_plugin         = "kind"
+kube_plugin_config  = "kaas/kind_config.yaml"
+clusterspec_cluster = "current-k8s-release"
+
+
+[subjects.kind-current-1.kubernetes_setup]
+kube_plugin         = "kind"
+kube_plugin_config  = "kaas/kind_config.yaml"
+clusterspec_cluster = "current-k8s-release-1"
+
+
+[subjects.kind-current-2.kubernetes_setup]
+kube_plugin         = "kind"
+kube_plugin_config  = "kaas/kind_config.yaml"
+clusterspec_cluster = "current-k8s-release-2"
+

Tests/kaas/clusterspec.yaml

@@ -0,0 +1,11 @@
+# this file specifies all clusters that have to be provisioned for the tests to run
+clusters:
+  current-k8s-release:
+    branch: "1.31"
+    kubeconfig: kubeconfig.yaml
+  current-k8s-release-1:
+    branch: "1.30"
+    kubeconfig: kubeconfig.yaml
+  current-k8s-release-2:
+    branch: "1.29"
+    kubeconfig: kubeconfig.yaml

Tests/kaas/kind_config.yaml

@@ -0,0 +1,5 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker

Tests/kaas/plugin/README.md

@@ -0,0 +1,38 @@
+# Plugin for provisioning k8s clusters and performing conformance tests on these clusters
+
+## Development environment
+
+### requirements
+
+* [docker](https://docs.docker.com/engine/install/)
+* [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+
+### setup for development
+
+1. Generate python 3.10 env
+
+   ```bash
+   sudo apt-get install python3.10-dev
+   virtualenv -p /usr/bin/python3.10 venv
+   echo "*" >> venv/.gitignore
+   source venv/bin/activate
+   (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+   (venv) python3.10 -m pip install --upgrade pip
+   (venv) python3.10 -m pip --version
+
+   ```
+
+2. Install dependencies:
+
+   ```bash
+   (venv) pip install pip-tools
+   (venv) pip-compile requirements.in
+   (venv) pip-sync requirements.txt
+   ```
+
+3. Set environment variables and launch the process:
+
+   ```bash
+   (venv) export CLUSTER_PROVIDER="kind"
+   (venv) python run.py
+   ```

Tests/kaas/plugin/interface.py

@@ -0,0 +1,54 @@
+
+
+class KubernetesClusterPlugin():
+    """
+    An abstract base class for custom Kubernetes cluster provider plugins.
+    It represents an interface class from which the api provider-specific
+    plugins must be derived as child classes
+
+    To implement fill the methods `create_cluster` and `delete_cluster` with
+    api provider-specific functionalities for creating and deleting clusters.
+    The `create_cluster` method must ensure that the kubeconfigfile is provided
+    at the position in the file system defined by the parameter
+    `kubeconfig_filepath`
+
+    - Implement `create_cluster` and `delete_cluster` methods
+    - Create `__init__(self, config_file)` method to handle api specific
+      configurations.
+
+    Example:
+        .. code:: python
+
+          from interface import KubernetesClusterPlugin
+          from apiX_library import cluster_api_class as ClusterAPI
+
+          class PluginX(KubernetesClusterPlugin):
+
+              def __init__(self, config_file):
+                  self.config = config_file
+
+              def create_cluster(self, cluster_name, version, kubeconfig_filepath):
+                  self.cluster = ClusterAPI(name=cluster_name, image=cluster_image, kubeconfig_filepath)
+                  self.cluster.create(self.config)
+
+              def delete_cluster(self, cluster_name):
+                  self.cluster = ClusterAPI(cluster_name)
+                  self.cluster.delete()
+        ..
+    """
+
+    def create_cluster(self, cluster_name, version, kubeconfig_filepath):
+        """
+        This method is to be called to create a k8s cluster
+        :param: cluster_name:
+        :param: version:
+        :param: kubeconfig_filepath:
+        """
+        raise NotImplementedError
+
+    def delete_cluster(self, cluster_name):
+        """
+        This method is to be called in order to unprovision a cluster
+        :param: cluster_name:
+        """
+        raise NotImplementedError

Tests/kaas/plugin/plugin_kind.py

@@ -0,0 +1,50 @@
+import logging
+import os
+import os.path
+from pathlib import Path
+
+from interface import KubernetesClusterPlugin
+from pytest_kind import KindCluster
+
+logger = logging.getLogger(__name__)
+
+
+class PluginKind(KubernetesClusterPlugin):
+    """
+    Plugin to handle the provisioning of kubernetes cluster for
+    conformance testing purpose with the use of Kind
+    """
+    def __init__(self, config_path):
+        logger.info("Init PluginKind")
+        self.config = config_path
+        logger.debug(self.config)
+        self.working_directory = os.getcwd()
+        logger.debug(f"Working from {self.working_directory}")
+
+    def create_cluster(self, cluster_name, version, kubeconfig):
+        """
+        This method is to be called to create a k8s cluster
+        :param: kubernetes_version:
+        :return: kubeconfig_filepath
+        """
+        cluster_version = version
+        if cluster_version == '1.29':
+            cluster_version = 'v1.29.8'
+        elif cluster_version == '1.30':
+            cluster_version = 'v1.30.4'
+        elif cluster_version == '1.31' or cluster_version == 'default':
+            cluster_version = 'v1.31.1'
+        cluster_image = f"kindest/node:{cluster_version}"
+        kubeconfig_filepath = Path(kubeconfig)
+        if kubeconfig_filepath is None:
+            raise ValueError("kubeconfig_filepath is missing")
+        else:
+            self.cluster = KindCluster(name=cluster_name, image=cluster_image, kubeconfig=kubeconfig_filepath)
+        if self.config is None:
+            self.cluster.create()
+        else:
+            self.cluster.create(self.config)
+
+    def delete_cluster(self, cluster_name):
+        self.cluster = KindCluster(cluster_name)
+        self.cluster.delete()

Tests/kaas/plugin/plugin_static.py

@@ -0,0 +1,19 @@
+import shutil
+
+from interface import KubernetesClusterPlugin
+
+
+class PluginStatic(KubernetesClusterPlugin):
+    """
+    Plugin to handle the provisioning of kubernetes
+    using a kubeconfig file
+    """
+
+    def __init__(self, config_path):
+        self.kubeconfig_path = config_path
+
+    def create_cluster(self, cluster_name, version, kubeconfig):
+        shutil.copyfile(self.kubeconfig_path, kubeconfig)
+
+    def delete_cluster(self, cluster_name, version):
+        pass

Tests/kaas/plugin/requirements.in

@@ -0,0 +1,2 @@
+pytest-kind
+kubernetes

Tests/kaas/plugin/requirements.txt

@@ -0,0 +1,60 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+cachetools==5.5.0
+    # via google-auth
+certifi==2024.8.30
+    # via
+    #   kubernetes
+    #   requests
+charset-normalizer==3.3.2
+    # via requests
+google-auth==2.34.0
+    # via kubernetes
+idna==3.8
+    # via requests
+kubernetes==30.1.0
+    # via -r requirements.in
+oauthlib==3.2.2
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+pyasn1==0.6.0
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.4.0
+    # via google-auth
+pykube-ng==23.6.0
+    # via pytest-kind
+pytest-kind==22.11.1
+    # via -r requirements.in
+python-dateutil==2.9.0.post0
+    # via kubernetes
+pyyaml==6.0.2
+    # via
+    #   kubernetes
+    #   pykube-ng
+requests==2.32.3
+    # via
+    #   kubernetes
+    #   pykube-ng
+    #   requests-oauthlib
+requests-oauthlib==2.0.0
+    # via kubernetes
+rsa==4.9
+    # via google-auth
+six==1.16.0
+    # via
+    #   kubernetes
+    #   python-dateutil
+urllib3==2.2.2
+    # via
+    #   kubernetes
+    #   pykube-ng
+    #   requests
+websocket-client==1.8.0
+    # via kubernetes

Tests/kaas/plugin/run_plugin.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import logging
+import os.path
+
+import click
+import yaml
+
+from plugin_kind import PluginKind
+from plugin_static import PluginStatic
+
+PLUGIN_LOOKUP = {
+    "kind": PluginKind,
+    "static": PluginStatic,
+}
+
+
+def init_plugin(plugin_kind, config_path):
+    plugin_maker = PLUGIN_LOOKUP.get(plugin_kind)
+    if plugin_maker is None:
+        raise ValueError(f"unknown plugin '{plugin_kind}'")
+    return plugin_maker(config_path)
+
+
+def load_spec(clusterspec_path):
+    with open(clusterspec_path, "rb") as fileobj:
+        return yaml.load(fileobj, Loader=yaml.SafeLoader)
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False))
+@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False))
+@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False))
+@click.argument('cluster_id', type=str, default="default")
+def create(plugin_kind, plugin_config, clusterspec_path, cluster_id):
+    clusterspec = load_spec(clusterspec_path)['clusters']
+    plugin = init_plugin(plugin_kind, plugin_config)
+    clusterinfo = clusterspec[cluster_id]
+    plugin.create_cluster(cluster_id, clusterinfo['branch'], os.path.abspath(clusterinfo['kubeconfig']))
+
+
+@cli.command()
+@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False))
+@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False))
+@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False))
+@click.argument('cluster_id', type=str, default="default")
+def delete(plugin_kind, plugin_config, clusterspec_path, cluster_id):
+    plugin = init_plugin(plugin_kind, plugin_config)
+    plugin.delete_cluster(cluster_id)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
+    cli()