chore: Add e2e tests to validate hubble-relay and hubble-ui deployment (

#896)

# Description

Add e2e tests to validate Hubble-relay and Hubble-UI deployment, in
order not to touch the legacy test, the install-helm-chart func was
re-written again. The test Hubble job is able to validate Hubble
resource and basic metrics but not migrating all the advanced metrics
validation since there are many heavy liftings work remain and could be
addressed in the future migration.

## Related Issue

#422

## Checklist

- [ ] I have read the [contributing
documentation](https://retina.sh/docs/contributing).
- [ ] I signed and signed-off the commits (`git commit -S -s ...`). See
[this
documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification)
on signing commits.
- [ ] I have correctly attributed the author(s) of the code.
- [ ] I have tested the changes locally.
- [ ] I have followed the project's style guidelines.
- [ ] I have updated the documentation, if necessary.
- [ ] I have added tests, if applicable.

## Screenshots (if applicable) or Testing Completed

<img width="925" alt="image"
src="https://github.com/user-attachments/assets/fa0207b7-c004-4a34-a9d4-8cef09fff327">

## Additional Notes

Add any additional notes or context about the pull request here.

---

Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more
information on how to contribute to this project.

---------

Signed-off-by: Yilin <[email protected]>

test/e2e/framework/kubernetes/install-hubble-helm.go

@@ -0,0 +1,155 @@
+package kubernetes
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/microsoft/retina/test/e2e/common"
+	generic "github.com/microsoft/retina/test/e2e/framework/generic"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/cli"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	HubbleNamespace = "kube-system"
+	HubbleUIApp     = "hubble-ui"
+	HubbleRelayApp  = "hubble-relay"
+)
+
+type ValidateHubbleStep struct {
+	Namespace          string
+	ReleaseName        string
+	KubeConfigFilePath string
+	ChartPath          string
+	TagEnv             string
+}
+
+func (v *ValidateHubbleStep) Run() error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second)
+	defer cancel()
+
+	settings := cli.New()
+	settings.KubeConfig = v.KubeConfigFilePath
+	actionConfig := new(action.Configuration)
+
+	err := actionConfig.Init(settings.RESTClientGetter(), v.Namespace, os.Getenv("HELM_DRIVER"), log.Printf)
+	if err != nil {
+		return fmt.Errorf("failed to initialize helm action config: %w", err)
+	}
+
+	// Creating extra namespace to deploy test pods
+	err = CreateNamespaceFn(v.KubeConfigFilePath, common.TestPodNamespace)
+	if err != nil {
+		return fmt.Errorf("failed to create namespace %s: %w", v.Namespace, err)
+	}
+
+	tag := os.Getenv(generic.DefaultTagEnv)
+	if tag == "" {
+		return fmt.Errorf("tag is not set: %w", errEmpty)
+	}
+	imageRegistry := os.Getenv(generic.DefaultImageRegistry)
+	if imageRegistry == "" {
+		return fmt.Errorf("image registry is not set: %w", errEmpty)
+	}
+
+	imageNamespace := os.Getenv(generic.DefaultImageNamespace)
+	if imageNamespace == "" {
+		return fmt.Errorf("image namespace is not set: %w", errEmpty)
+	}
+
+	// load chart from the path
+	chart, err := loader.Load(v.ChartPath)
+	if err != nil {
+		return fmt.Errorf("failed to load chart from path %s: %w", v.ChartPath, err)
+	}
+
+	chart.Values["imagePullSecrets"] = []map[string]interface{}{
+		{
+			"name": "acr-credentials",
+		},
+	}
+	chart.Values["operator"].(map[string]interface{})["enabled"] = true
+	chart.Values["operator"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-operator"
+	chart.Values["operator"].(map[string]interface{})["tag"] = tag
+	chart.Values["agent"].(map[string]interface{})["enabled"] = true
+	chart.Values["agent"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-agent"
+	chart.Values["agent"].(map[string]interface{})["tag"] = tag
+	chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["enabled"] = true
+	chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-init"
+	chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["tag"] = tag
+	chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["enabled"] = false
+	chart.Values["hubble"].(map[string]interface{})["relay"].(map[string]interface{})["tls"].(map[string]interface{})["server"].(map[string]interface{})["enabled"] = false
+	chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["auto"].(map[string]interface{})["enabled"] = false
+
+	getclient := action.NewGet(actionConfig)
+	release, err := getclient.Run(v.ReleaseName)
+	if err == nil && release != nil {
+		log.Printf("found existing release by same name, removing before installing %s", release.Name)
+		delclient := action.NewUninstall(actionConfig)
+		delclient.Wait = true
+		delclient.Timeout = deleteTimeout
+		_, err = delclient.Run(v.ReleaseName)
+		if err != nil {
+			return fmt.Errorf("failed to delete existing release %s: %w", v.ReleaseName, err)
+		}
+	} else if err != nil && !strings.Contains(err.Error(), "not found") {
+		return fmt.Errorf("failed to get release %s: %w", v.ReleaseName, err)
+	}
+
+	client := action.NewInstall(actionConfig)
+	client.Namespace = v.Namespace
+	client.ReleaseName = v.ReleaseName
+	client.Timeout = createTimeout
+	client.Wait = true
+	client.WaitForJobs = true
+
+	// install the chart here
+	rel, err := client.RunWithContext(ctx, chart, chart.Values)
+	if err != nil {
+		return fmt.Errorf("failed to install chart: %w", err)
+	}
+
+	log.Printf("installed chart from path: %s in namespace: %s\n", rel.Name, rel.Namespace)
+	// this will confirm the values set during installation
+	log.Printf("chart values: %v\n", rel.Config)
+
+	// ensure all pods are running, since helm doesn't care about windows
+	config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath)
+	if err != nil {
+		return fmt.Errorf("error building kubeconfig: %w", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("error creating Kubernetes client: %w", err)
+	}
+
+	// Validate Hubble Relay Pod
+	if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleRelayApp); err != nil {
+		return fmt.Errorf("error waiting for Hubble Relay pods to be ready: %w", err)
+	}
+	log.Printf("Hubble Relay Pod is ready")
+
+	// Validate Hubble UI Pod
+	if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleUIApp); err != nil {
+		return fmt.Errorf("error waiting for Hubble UI pods to be ready: %w", err)
+	}
+	log.Printf("Hubble UI Pod is ready")
+
+	return nil
+}
+
+func (v *ValidateHubbleStep) Prevalidate() error {
+	return nil
+}
+
+func (v *ValidateHubbleStep) Stop() error {
+	return nil
+}

test/e2e/framework/kubernetes/validate-service.go

@@ -0,0 +1,83 @@
+package kubernetes
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+type ResourceTypes string
+
+const (
+	ResourceTypePod     = "pod"
+	ResourceTypeService = "service"
+)
+
+type ValidateResource struct {
+	ResourceName       string
+	ResourceNamespace  string
+	ResourceType       string
+	Labels             string
+	KubeConfigFilePath string
+}
+
+func (v *ValidateResource) Run() error {
+	config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath)
+	if err != nil {
+		return fmt.Errorf("error building kubeconfig: %w", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("error creating Kubernetes client: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second)
+	defer cancel()
+
+	switch v.ResourceType {
+	case ResourceTypePod:
+		err = WaitForPodReady(ctx, clientset, v.ResourceNamespace, v.Labels)
+		if err != nil {
+			return fmt.Errorf("pod not found: %w", err)
+		}
+	case ResourceTypeService:
+		exists, err := serviceExists(ctx, clientset, v.ResourceNamespace, v.ResourceName, v.Labels)
+		if err != nil || !exists {
+			return fmt.Errorf("service not found: %w", err)
+		}
+
+	default:
+		return fmt.Errorf("resource type %s not supported", v.ResourceType)
+	}
+
+	if err != nil {
+		return fmt.Errorf("error waiting for pod to be ready: %w", err)
+	}
+	return nil
+}
+
+func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) {
+	var serviceList *corev1.ServiceList
+	serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels})
+	if err != nil {
+		return false, fmt.Errorf("error listing Services: %w", err)
+	}
+	if len(serviceList.Items) == 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (v *ValidateResource) Prevalidate() error {
+	return nil
+}
+
+func (v *ValidateResource) Stop() error {
+	return nil
+}

test/e2e/framework/kubernetes/validateHttp.go

@@ -0,0 +1,50 @@
+package kubernetes
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+)
+
+const (
+	RequestTimeout = 30 * time.Second
+)
+
+type ValidateHTTPResponse struct {
+	URL            string
+	ExpectedStatus int
+}
+
+func (v *ValidateHTTPResponse) Run() error {
+	ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.URL, nil)
+	if err != nil {
+		return fmt.Errorf("error creating HTTP request: %w", err)
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error making HTTP request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != v.ExpectedStatus {
+		return fmt.Errorf("unexpected status code: got %d, want %d", resp.StatusCode, v.ExpectedStatus)
+	}
+	log.Printf("HTTP validation succeeded for URL: %s with status code %d\n", v.URL, resp.StatusCode)
+
+	return nil
+}
+
+func (v *ValidateHTTPResponse) Prevalidate() error {
+	return nil
+}
+
+func (v *ValidateHTTPResponse) Stop() error {
+	return nil
+}

test/e2e/hubble/scenario.go

@@ -0,0 +1,56 @@
+package hubble
+
+import (
+	"net/http"
+
+	k8s "github.com/microsoft/retina/test/e2e/framework/kubernetes"
+	"github.com/microsoft/retina/test/e2e/framework/types"
+)
+
+func ValidateHubbleRelayService() *types.Scenario {
+	name := "Validate Hubble Relay Service"
+	steps := []*types.StepWrapper{
+		{
+			Step: &k8s.ValidateResource{
+				ResourceName:      "hubble-relay-service",
+				ResourceNamespace: k8s.HubbleNamespace,
+				ResourceType:      k8s.ResourceTypeService,
+				Labels:            "k8s-app=" + k8s.HubbleRelayApp,
+			},
+		},
+	}
+
+	return types.NewScenario(name, steps...)
+}
+
+func ValidateHubbleUIService(kubeConfigFilePath string) *types.Scenario {
+	name := "Validate Hubble UI Services"
+	steps := []*types.StepWrapper{
+		{
+			Step: &k8s.ValidateResource{
+				ResourceName:      k8s.HubbleUIApp,
+				ResourceNamespace: k8s.HubbleNamespace,
+				ResourceType:      k8s.ResourceTypeService,
+				Labels:            "k8s-app=" + k8s.HubbleUIApp,
+			},
+		},
+		{
+			Step: &k8s.PortForward{
+				LabelSelector:         "k8s-app=hubble-ui",
+				LocalPort:             "8080",
+				RemotePort:            "8081",
+				OptionalLabelAffinity: "k8s-app=hubble-ui",
+				Endpoint:              "?namespace=default", // set as default namespace query since endpoint can't be nil
+				KubeConfigFilePath:    kubeConfigFilePath,
+			},
+		},
+		{
+			Step: &k8s.ValidateHTTPResponse{
+				URL:            "http://localhost:8080",
+				ExpectedStatus: http.StatusOK,
+			},
+		},
+	}
+
+	return types.NewScenario(name, steps...)
+}