From 8db4de4752fcdf2d8bd2a445299b603199bee6bd Mon Sep 17 00:00:00 2001 From: Yilin <76769345+xiaozhiche320@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:30:12 -0800 Subject: [PATCH] chore: Add e2e tests to validate hubble-relay and hubble-ui deployment (#896) # Description Add e2e tests to validate Hubble-relay and Hubble-UI deployment, in order not to touch the legacy test, the install-helm-chart func was re-written again. The test Hubble job is able to validate Hubble resource and basic metrics but not migrating all the advanced metrics validation since there are many heavy liftings work remain and could be addressed in the future migration. ## Related Issue https://github.com/microsoft/retina/issues/422 ## Checklist - [ ] I have read the [contributing documentation](https://retina.sh/docs/contributing). - [ ] I signed and signed-off the commits (`git commit -S -s ...`). See [this documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) on signing commits. - [ ] I have correctly attributed the author(s) of the code. - [ ] I have tested the changes locally. - [ ] I have followed the project's style guidelines. - [ ] I have updated the documentation, if necessary. - [ ] I have added tests, if applicable. ## Screenshots (if applicable) or Testing Completed image ## Additional Notes Add any additional notes or context about the pull request here. --- Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more information on how to contribute to this project. --------- Signed-off-by: Yilin <76769345+xiaozhiche320@users.noreply.github.com> --- .../kubernetes/install-hubble-helm.go | 155 ++++++++++++++++++ .../framework/kubernetes/validate-service.go | 83 ++++++++++ test/e2e/framework/kubernetes/validateHttp.go | 50 ++++++ test/e2e/hubble/scenario.go | 56 +++++++ test/e2e/jobs/jobs.go | 75 +++++++++ test/e2e/retina_e2e_test.go | 5 + 6 files changed, 424 insertions(+) create mode 100644 test/e2e/framework/kubernetes/install-hubble-helm.go create mode 100644 test/e2e/framework/kubernetes/validate-service.go create mode 100644 test/e2e/framework/kubernetes/validateHttp.go create mode 100644 test/e2e/hubble/scenario.go diff --git a/test/e2e/framework/kubernetes/install-hubble-helm.go b/test/e2e/framework/kubernetes/install-hubble-helm.go new file mode 100644 index 0000000000..5e225d08e6 --- /dev/null +++ b/test/e2e/framework/kubernetes/install-hubble-helm.go @@ -0,0 +1,155 @@ +package kubernetes + +import ( + "context" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/microsoft/retina/test/e2e/common" + generic "github.com/microsoft/retina/test/e2e/framework/generic" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + HubbleNamespace = "kube-system" + HubbleUIApp = "hubble-ui" + HubbleRelayApp = "hubble-relay" +) + +type ValidateHubbleStep struct { + Namespace string + ReleaseName string + KubeConfigFilePath string + ChartPath string + TagEnv string +} + +func (v *ValidateHubbleStep) Run() error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second) + defer cancel() + + settings := cli.New() + settings.KubeConfig = v.KubeConfigFilePath + actionConfig := new(action.Configuration) + + err := actionConfig.Init(settings.RESTClientGetter(), v.Namespace, os.Getenv("HELM_DRIVER"), log.Printf) + if err != nil { + return fmt.Errorf("failed to initialize helm action config: %w", err) + } + + // Creating extra namespace to deploy test pods + err = CreateNamespaceFn(v.KubeConfigFilePath, common.TestPodNamespace) + if err != nil { + return fmt.Errorf("failed to create namespace %s: %w", v.Namespace, err) + } + + tag := os.Getenv(generic.DefaultTagEnv) + if tag == "" { + return fmt.Errorf("tag is not set: %w", errEmpty) + } + imageRegistry := os.Getenv(generic.DefaultImageRegistry) + if imageRegistry == "" { + return fmt.Errorf("image registry is not set: %w", errEmpty) + } + + imageNamespace := os.Getenv(generic.DefaultImageNamespace) + if imageNamespace == "" { + return fmt.Errorf("image namespace is not set: %w", errEmpty) + } + + // load chart from the path + chart, err := loader.Load(v.ChartPath) + if err != nil { + return fmt.Errorf("failed to load chart from path %s: %w", v.ChartPath, err) + } + + chart.Values["imagePullSecrets"] = []map[string]interface{}{ + { + "name": "acr-credentials", + }, + } + chart.Values["operator"].(map[string]interface{})["enabled"] = true + chart.Values["operator"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-operator" + chart.Values["operator"].(map[string]interface{})["tag"] = tag + chart.Values["agent"].(map[string]interface{})["enabled"] = true + chart.Values["agent"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-agent" + chart.Values["agent"].(map[string]interface{})["tag"] = tag + chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["enabled"] = true + chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-init" + chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["tag"] = tag + chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["enabled"] = false + chart.Values["hubble"].(map[string]interface{})["relay"].(map[string]interface{})["tls"].(map[string]interface{})["server"].(map[string]interface{})["enabled"] = false + chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["auto"].(map[string]interface{})["enabled"] = false + + getclient := action.NewGet(actionConfig) + release, err := getclient.Run(v.ReleaseName) + if err == nil && release != nil { + log.Printf("found existing release by same name, removing before installing %s", release.Name) + delclient := action.NewUninstall(actionConfig) + delclient.Wait = true + delclient.Timeout = deleteTimeout + _, err = delclient.Run(v.ReleaseName) + if err != nil { + return fmt.Errorf("failed to delete existing release %s: %w", v.ReleaseName, err) + } + } else if err != nil && !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("failed to get release %s: %w", v.ReleaseName, err) + } + + client := action.NewInstall(actionConfig) + client.Namespace = v.Namespace + client.ReleaseName = v.ReleaseName + client.Timeout = createTimeout + client.Wait = true + client.WaitForJobs = true + + // install the chart here + rel, err := client.RunWithContext(ctx, chart, chart.Values) + if err != nil { + return fmt.Errorf("failed to install chart: %w", err) + } + + log.Printf("installed chart from path: %s in namespace: %s\n", rel.Name, rel.Namespace) + // this will confirm the values set during installation + log.Printf("chart values: %v\n", rel.Config) + + // ensure all pods are running, since helm doesn't care about windows + config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + // Validate Hubble Relay Pod + if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleRelayApp); err != nil { + return fmt.Errorf("error waiting for Hubble Relay pods to be ready: %w", err) + } + log.Printf("Hubble Relay Pod is ready") + + // Validate Hubble UI Pod + if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleUIApp); err != nil { + return fmt.Errorf("error waiting for Hubble UI pods to be ready: %w", err) + } + log.Printf("Hubble UI Pod is ready") + + return nil +} + +func (v *ValidateHubbleStep) Prevalidate() error { + return nil +} + +func (v *ValidateHubbleStep) Stop() error { + return nil +} diff --git a/test/e2e/framework/kubernetes/validate-service.go b/test/e2e/framework/kubernetes/validate-service.go new file mode 100644 index 0000000000..4ba7463bc2 --- /dev/null +++ b/test/e2e/framework/kubernetes/validate-service.go @@ -0,0 +1,83 @@ +package kubernetes + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type ResourceTypes string + +const ( + ResourceTypePod = "pod" + ResourceTypeService = "service" +) + +type ValidateResource struct { + ResourceName string + ResourceNamespace string + ResourceType string + Labels string + KubeConfigFilePath string +} + +func (v *ValidateResource) Run() error { + config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second) + defer cancel() + + switch v.ResourceType { + case ResourceTypePod: + err = WaitForPodReady(ctx, clientset, v.ResourceNamespace, v.Labels) + if err != nil { + return fmt.Errorf("pod not found: %w", err) + } + case ResourceTypeService: + exists, err := serviceExists(ctx, clientset, v.ResourceNamespace, v.ResourceName, v.Labels) + if err != nil || !exists { + return fmt.Errorf("service not found: %w", err) + } + + default: + return fmt.Errorf("resource type %s not supported", v.ResourceType) + } + + if err != nil { + return fmt.Errorf("error waiting for pod to be ready: %w", err) + } + return nil +} + +func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) { + var serviceList *corev1.ServiceList + serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels}) + if err != nil { + return false, fmt.Errorf("error listing Services: %w", err) + } + if len(serviceList.Items) == 0 { + return false, nil + } + return true, nil +} + +func (v *ValidateResource) Prevalidate() error { + return nil +} + +func (v *ValidateResource) Stop() error { + return nil +} diff --git a/test/e2e/framework/kubernetes/validateHttp.go b/test/e2e/framework/kubernetes/validateHttp.go new file mode 100644 index 0000000000..39e2e400b2 --- /dev/null +++ b/test/e2e/framework/kubernetes/validateHttp.go @@ -0,0 +1,50 @@ +package kubernetes + +import ( + "context" + "fmt" + "log" + "net/http" + "time" +) + +const ( + RequestTimeout = 30 * time.Second +) + +type ValidateHTTPResponse struct { + URL string + ExpectedStatus int +} + +func (v *ValidateHTTPResponse) Run() error { + ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.URL, nil) + if err != nil { + return fmt.Errorf("error creating HTTP request: %w", err) + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error making HTTP request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != v.ExpectedStatus { + return fmt.Errorf("unexpected status code: got %d, want %d", resp.StatusCode, v.ExpectedStatus) + } + log.Printf("HTTP validation succeeded for URL: %s with status code %d\n", v.URL, resp.StatusCode) + + return nil +} + +func (v *ValidateHTTPResponse) Prevalidate() error { + return nil +} + +func (v *ValidateHTTPResponse) Stop() error { + return nil +} diff --git a/test/e2e/hubble/scenario.go b/test/e2e/hubble/scenario.go new file mode 100644 index 0000000000..92de650b32 --- /dev/null +++ b/test/e2e/hubble/scenario.go @@ -0,0 +1,56 @@ +package hubble + +import ( + "net/http" + + k8s "github.com/microsoft/retina/test/e2e/framework/kubernetes" + "github.com/microsoft/retina/test/e2e/framework/types" +) + +func ValidateHubbleRelayService() *types.Scenario { + name := "Validate Hubble Relay Service" + steps := []*types.StepWrapper{ + { + Step: &k8s.ValidateResource{ + ResourceName: "hubble-relay-service", + ResourceNamespace: k8s.HubbleNamespace, + ResourceType: k8s.ResourceTypeService, + Labels: "k8s-app=" + k8s.HubbleRelayApp, + }, + }, + } + + return types.NewScenario(name, steps...) +} + +func ValidateHubbleUIService(kubeConfigFilePath string) *types.Scenario { + name := "Validate Hubble UI Services" + steps := []*types.StepWrapper{ + { + Step: &k8s.ValidateResource{ + ResourceName: k8s.HubbleUIApp, + ResourceNamespace: k8s.HubbleNamespace, + ResourceType: k8s.ResourceTypeService, + Labels: "k8s-app=" + k8s.HubbleUIApp, + }, + }, + { + Step: &k8s.PortForward{ + LabelSelector: "k8s-app=hubble-ui", + LocalPort: "8080", + RemotePort: "8081", + OptionalLabelAffinity: "k8s-app=hubble-ui", + Endpoint: "?namespace=default", // set as default namespace query since endpoint can't be nil + KubeConfigFilePath: kubeConfigFilePath, + }, + }, + { + Step: &k8s.ValidateHTTPResponse{ + URL: "http://localhost:8080", + ExpectedStatus: http.StatusOK, + }, + }, + } + + return types.NewScenario(name, steps...) +} diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index bd921df2a3..925df6dc01 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -9,6 +9,7 @@ import ( "github.com/microsoft/retina/test/e2e/framework/generic" "github.com/microsoft/retina/test/e2e/framework/kubernetes" "github.com/microsoft/retina/test/e2e/framework/types" + "github.com/microsoft/retina/test/e2e/hubble" "github.com/microsoft/retina/test/e2e/scenarios/dns" "github.com/microsoft/retina/test/e2e/scenarios/drop" "github.com/microsoft/retina/test/e2e/scenarios/latency" @@ -232,6 +233,79 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi return job } +func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job { + job := types.NewJob("Validate Hubble") + + job.AddStep(&kubernetes.ValidateHubbleStep{ + Namespace: common.KubeSystemNamespace, + ReleaseName: "retina", + KubeConfigFilePath: kubeConfigFilePath, + ChartPath: chartPath, + TagEnv: generic.DefaultTagEnv, + }, nil) + + job.AddScenario(hubble.ValidateHubbleRelayService()) + + job.AddScenario(hubble.ValidateHubbleUIService(kubeConfigFilePath)) + + job.AddScenario(drop.ValidateDropMetric(testPodNamespace)) + + job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace)) + + dnsScenarios := []struct { + name string + req *dns.RequestValidationParams + resp *dns.ResponseValidationParams + }{ + { + name: "Validate basic DNS request and response metrics for a valid domain", + req: &dns.RequestValidationParams{ + NumResponse: "0", + Query: "kubernetes.default.svc.cluster.local.", + QueryType: "A", + Command: "nslookup kubernetes.default", + ExpectError: false, + }, + resp: &dns.ResponseValidationParams{ + NumResponse: "1", + Query: "kubernetes.default.svc.cluster.local.", + QueryType: "A", + ReturnCode: "No Error", + Response: "10.0.0.1", + }, + }, + { + name: "Validate basic DNS request and response metrics for a non-existent domain", + req: &dns.RequestValidationParams{ + NumResponse: "0", + Query: "some.non.existent.domain.", + QueryType: "A", + Command: "nslookup some.non.existent.domain", + ExpectError: true, + }, + resp: &dns.ResponseValidationParams{ + NumResponse: "0", + Query: "some.non.existent.domain.", + QueryType: "A", + Response: dns.EmptyResponse, // hacky way to bypass the framework for now + ReturnCode: "Non-Existent Domain", + }, + }, + } + + for _, scenario := range dnsScenarios { + job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace)) + } + + job.AddStep(&kubernetes.EnsureStableComponent{ + PodNamespace: common.KubeSystemNamespace, + LabelSelector: "k8s-app=retina", + IgnoreContainerRestart: false, + }, nil) + + return job +} + func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { job := types.NewJob("Run performance tests") @@ -279,3 +353,4 @@ func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { return job } + diff --git a/test/e2e/retina_e2e_test.go b/test/e2e/retina_e2e_test.go index 7b13d1782b..d1b1b16e95 100644 --- a/test/e2e/retina_e2e_test.go +++ b/test/e2e/retina_e2e_test.go @@ -57,6 +57,7 @@ func TestE2ERetina(t *testing.T) { rootDir := filepath.Dir(filepath.Dir(cwd)) chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") + hubblechartPath := filepath.Join(rootDir, "deploy", "hubble", "manifests", "controller", "helm", "retina") profilePath := filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml") kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") @@ -77,4 +78,8 @@ func TestE2ERetina(t *testing.T) { // Upgrade and test Retina with advanced metrics advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace)) advanceMetricsE2E.Run(ctx) + + // Install and test Hubble basic metrics + validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace)) + validatehubble.Run(ctx) }