feat: namespace per test (#260)

* feat(ns): initial commit Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): random ns Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): random ns Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): clientset init Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): priritize the ns creation Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): add logs Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): add logs Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): keep it only in k8s.Initialize Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): add log info + delete ns after the tests is done Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update log msg Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update from mainh Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update from main Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update order Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): add && Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): remove second wait Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): remove second wait Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): remove secon2d wait Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): uniq id as ns Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): fix typo+ Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): layout Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): fix non ns Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): remove && Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): clenaup Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update docs Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update command Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update command func Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): comments and tests Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update docs Signed-off-by: Jose Ramon Mañes <[email protected]> * feat(ns): update docs Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: remove comment Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: func if duplicated check Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: check if ns exists Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: move func and update Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: update docs+ Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: update docs Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: add comments Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: add comments Signed-off-by: Jose Ramon Mañes <[email protected]> * fix: add comments Signed-off-by: Jose Ramon Mañes <[email protected]> --------- Signed-off-by: Jose Ramon Mañes <[email protected]>
celestiaorg · Apr 4, 2024 · 80efc4f · 80efc4f
1 parent 2fd263b
commit 80efc4f
Show file tree

Hide file tree

Showing 4 changed files with 158 additions and 30 deletions.
diff --git a/README.md b/README.md
@@ -173,12 +173,19 @@ go test -v ./...
 
 You can set the following environment variables to change the behavior of knuu:
 
-| Environment Variable       | Description                               | Possible Values         | Default |
-|----------------------------|-------------------------------------------|-------------------------|---------|
-| `KNUU_TIMEOUT`             | The timeout for the tests.                | Any valid duration       | `60m`    |
-| `KNUU_NAMESPACE`           | The namespace where the instances will be created. | Any valid namespace name | `test`  |
-| `KNUU_BUILDER`             | The builder to use for building images.   | `docker`, `kubernetes`  | `docker` |
-| `DEBUG_LEVEL`              | The debug level.                          | `debug`, `info`, `warn`, `error` | `info`  |
+| Environment Variable        | Description                                                                                                     | Possible Values                  | Default  |
+|-----------------------------|-----------------------------------------------------------------------------------------------------------------|----------------------------------|----------|
+| `KNUU_TIMEOUT`              | The timeout for the tests.                                                                                      | Any valid duration               | `60m`    |
+| `KNUU_NAMESPACE`            | The namespace where the instances will be created.                                                              | Any valid namespace name         | `test`   |
+| `KNUU_BUILDER`              | The builder to use for building images.                                                                         | `docker`, `kubernetes`           | `docker` |
+| `KNUU_DEDICATED_NAMESPACE`  | Creates and deletes a dedicated namespace for the test after `KNUU_TIMEOUT`. See note below for more details.*	 | `true`, `false`                  | `false`  |º
+| `DEBUG_LEVEL`               | The debug level.                                                                                                | `debug`, `info`, `warn`, `error` | `info`   |
+
+* Note on KNUU_DEDICATED_NAMESPACE: 
+
+When set to true, this environment variable tells knuu to create a unique namespace for running tests, which will be automatically deleted after the specified timeout period (`KNUU_TIMEOUT`).
+The created namespace will have a prefix of `knuu-` followed by a unique identifier (unique id of the test), which ensures that each test run has its isolated environment within the Kubernetes cluster.
+The unique identifier is an autogenerated timestamp that uniquely identifies each test run, however, developers have the option to specify this identifier.
 
 ## Contributing
 

diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go
@@ -5,7 +5,11 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
 
+	"github.com/sirupsen/logrus"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -25,7 +29,7 @@ var (
 )
 
 // Initialize sets up the Kubernetes client with the appropriate configuration.
-func Initialize() error {
+func Initialize(identifier string) error {
 	k8sConfig, err := getClusterConfig()
 	if err != nil {
 		return fmt.Errorf("retrieving the Kubernetes config: %w", err)
@@ -36,22 +40,38 @@ func Initialize() error {
 		return fmt.Errorf("creating clientset for Kubernetes: %w", err)
 	}
 
+	var namespaceName string
+	useDedicatedNamespace, _ := strconv.ParseBool(os.Getenv("KNUU_DEDICATED_NAMESPACE"))
+
 	// Check if the program is running in a Kubernetes cluster environment
 	if isClusterEnvironment() {
 		// Read the namespace from the pod's spec
 		namespaceBytes, err := os.ReadFile(namespacePath)
 		if err != nil {
 			return fmt.Errorf("reading namespace from pod's spec: %w", err)
 		}
-		setNamespace(string(namespaceBytes))
+		namespaceName = string(namespaceBytes)
+		logrus.Debugf("Using namespace from pod spec: %s", namespaceName)
+	} else if useDedicatedNamespace {
+		namespaceName, err = InitializeNamespace(identifier)
+		if err != nil {
+			return fmt.Errorf("initializing dedicated namespace: %w", err)
+		}
+		logrus.Debugf("KNUU_DEDICATED_NAMESPACE enabled, namespace generated: %s", namespaceName)
 	} else {
-		// Read the namespace from KNUU_NAMESPACE environment variable
-		if os.Getenv("KNUU_NAMESPACE") != "" {
-			setNamespace(os.Getenv("KNUU_NAMESPACE"))
-		} else {
-			setNamespace("test")
+		// Use KNUU_NAMESPACE or fallback to a default if it's not set
+		namespaceName = os.Getenv("KNUU_NAMESPACE")
+		if namespaceName == "" {
+			namespaceName = "test"
 		}
+		logrus.Debugf("KNUU_DEDICATED_NAMESPACE not specified, namespace to use: %s", namespaceName)
 	}
+
+	// Set the namespace
+	setNamespace(namespaceName)
+
+	logrus.Infof("Namespace where the test runs: %s", namespaceName)
+
 	return nil
 }
 
@@ -93,3 +113,28 @@ func getClusterConfig() (*rest.Config, error) {
 func isNotFound(err error) bool {
 	return apierrs.IsNotFound(err)
 }
+
+// precompile the regular expression to avoid recompiling it on every function call
+var invalidCharsRegexp = regexp.MustCompile(`[^a-z0-9-]+`)
+
+// sanitizeName ensures compliance with Kubernetes DNS-1123 subdomain names. It:
+//  1. Converts the input string to lowercase.
+//  2. Replaces underscores and any non-DNS-1123 compliant characters with hyphens.
+//  3. Trims leading and trailing hyphens.
+//  4. Ensures the name does not exceed 63 characters, trimming excess characters if necessary
+//     and ensuring it does not end with a hyphen after trimming.
+//
+// Use this function to sanitize strings to be used as Kubernetes names for resources.
+func sanitizeName(name string) string {
+	sanitized := strings.ToLower(name)
+	// Replace underscores and any other disallowed characters with hyphens
+	sanitized = invalidCharsRegexp.ReplaceAllString(sanitized, "-")
+	// Trim leading and trailing hyphens
+	sanitized = strings.Trim(sanitized, "-")
+	if len(sanitized) > 63 {
+		sanitized = sanitized[:63]
+		// Ensure it does not end with a hyphen after cutting it to the max length
+		sanitized = strings.TrimRight(sanitized, "-")
+	}
+	return sanitized
+}
diff --git a/pkg/k8s/k8s_namespace.go b/pkg/k8s/k8s_namespace.go
@@ -0,0 +1,47 @@
+package k8s
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/api/errors"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// InitializeNamespace sets up the namespace based on the KNUU_DEDICATED_NAMESPACE environment variable
+func InitializeNamespace(identifier string) (string, error) {
+	namespaceName := "knuu-" + sanitizeName(identifier)
+	logrus.Debugf("namespace random generated: %s", namespaceName)
+	if err := createNamespace(Clientset(), namespaceName); err != nil {
+		return "", fmt.Errorf("failed to create dedicated namespace: %v", err)
+	}
+
+	logrus.Debugf("full namespace name generated: %s", namespaceName)
+
+	return namespaceName, nil
+}
+
+// createNamespace creates a new namespace if it does not exist
+func createNamespace(clientset *kubernetes.Clientset, name string) error {
+	ctx := context.TODO()
+	namespace := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+
+	_, err := clientset.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{})
+	if err != nil {
+		if errors.IsAlreadyExists(err) {
+			fmt.Printf("Namespace %s already exists, continuing.\n", name)
+			return nil
+		}
+		return fmt.Errorf("error creating namespace %s: %v", name, err)
+	}
+
+	return nil
+}
diff --git a/pkg/knuu/knuu.go b/pkg/knuu/knuu.go
@@ -4,15 +4,18 @@ package knuu
 import (
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
+	"github.com/sirupsen/logrus"
+	rbacv1 "k8s.io/api/rbac/v1"
+
 	"github.com/celestiaorg/knuu/pkg/builder"
 	"github.com/celestiaorg/knuu/pkg/builder/docker"
 	"github.com/celestiaorg/knuu/pkg/builder/kaniko"
 	"github.com/celestiaorg/knuu/pkg/k8s"
 	"github.com/celestiaorg/knuu/pkg/minio"
-	"github.com/sirupsen/logrus"
-	rbacv1 "k8s.io/api/rbac/v1"
 )
 
 var (
@@ -26,7 +29,7 @@ var (
 // Initialize initializes knuug
 func Initialize() error {
 	t := time.Now()
-	identifier = fmt.Sprintf("%s_%03d", t.Format("20060102_150405"), t.Nanosecond()/1e6)
+	identifier = fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6)
 	return InitializeWithIdentifier(identifier)
 }
 
@@ -44,7 +47,7 @@ func InitializeWithIdentifier(uniqueIdentifier string) error {
 	identifier = uniqueIdentifier
 
 	t := time.Now()
-	startTime = fmt.Sprintf("%s_%03d", t.Format("20060102_150405"), t.Nanosecond()/1e6)
+	startTime = fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6)
 
 	switch os.Getenv("LOG_LEVEL") {
 	case "debug":
@@ -59,7 +62,15 @@ func InitializeWithIdentifier(uniqueIdentifier string) error {
 		logrus.SetLevel(logrus.InfoLevel)
 	}
 
-	err := k8s.Initialize()
+	useDedicatedNamespaceEnv := os.Getenv("KNUU_DEDICATED_NAMESPACE")
+	useDedicatedNamespace, err := strconv.ParseBool(useDedicatedNamespaceEnv)
+	if err != nil {
+		useDedicatedNamespace = false
+	}
+
+	logrus.Debugf("Use dedicated namespace: %t", useDedicatedNamespace)
+
+	err = k8s.Initialize(identifier)
 	if err != nil {
 		return err
 	}
@@ -130,18 +141,36 @@ func handleTimeout() error {
 	if err := instance.Commit(); err != nil {
 		return fmt.Errorf("cannot commit instance: %s", err)
 	}
-	timeoutSeconds := int64(timeout.Seconds())
-
-	// command to wait for timeout and delete all resources with the identifier
-	var command = []string{"sh", "-c"}
-	// Command runs in-cluster to delete resources post-test. Chosen for simplicity over a separate Go app.
-	wait := fmt.Sprintf("sleep %d", timeoutSeconds)
-	deleteAllButTimeOutType := fmt.Sprintf("kubectl get all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/test-run-id=%s -n %s -o json | jq -r '.items[] | select(.metadata.labels.\"knuu.sh/type\" != \"%s\") | \"\\(.kind)/\\(.metadata.name)\"' | xargs -r kubectl delete -n %s", identifier, k8s.Namespace(), TimeoutHandlerInstance.String(), k8s.Namespace())
-	deleteAll := fmt.Sprintf("kubectl delete all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/test-run-id=%s -n %s", identifier, k8s.Namespace())
-	cmd := fmt.Sprintf("%s && %s && %s", wait, deleteAllButTimeOutType, deleteAll)
-	command = append(command, cmd)
-
-	if err := instance.SetCommand(command...); err != nil {
+
+	var commands []string
+
+	// Wait for a specific period before executing the next operation.
+	// This is useful to ensure that any previous operation has time to complete.
+	commands = append(commands, fmt.Sprintf("sleep %d", int64(timeout.Seconds())))
+	// Collects all resources (pods, services, etc.) within the specified namespace that match a specific label, excluding certain types,
+	// and then deletes them. This is useful for cleaning up specific test resources before proceeding to delete the namespace.
+	commands = append(commands, fmt.Sprintf("kubectl get all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/test-run-id=%s -n %s -o json | jq -r '.items[] | select(.metadata.labels.\"knuu.sh/type\" != \"%s\") | \"\\(.kind)/\\(.metadata.name)\"' | xargs -r kubectl delete -n %s", identifier, k8s.Namespace(), TimeoutHandlerInstance.String(), k8s.Namespace()))
+
+	// Get KNUU_DEDICATED_NAMESPACE from the environment
+	useDedicatedNamespace, _ := strconv.ParseBool(os.Getenv("KNUU_DEDICATED_NAMESPACE"))
+
+	// If KNUU_DEDICATED_NAMESPACE is true, it indicates that a dedicated namespace is being used for this run.
+	// Therefore, if it is set to be deleted, this command will delete the dedicated namespace.
+	// This helps ensure that all resources within the namespace are deleted, and the namespace itself as well.
+	if useDedicatedNamespace {
+		logrus.Debugf("The namespace generated [%s] will be deleted", k8s.Namespace())
+		commands = append(commands, fmt.Sprintf("kubectl delete namespace %s", k8s.Namespace()))
+	}
+
+	// Delete all labeled resources within the namespace.
+	// Unlike the previous command that excludes certain types, this command ensures that everything remaining is deleted.
+	commands = append(commands, fmt.Sprintf("kubectl delete all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/test-run-id=%s -n %s", identifier, k8s.Namespace()))
+
+	finalCmd := strings.Join(commands, " && ")
+
+	// Run the command
+	if err := instance.SetCommand("sh", "-c", finalCmd); err != nil {
+		logrus.Debugf("The full command generated is [%s]", finalCmd)
 		return fmt.Errorf("cannot set command: %s", err)
 	}