From 2eeddbf60ac829abbe18ece5142b6ae91447602b Mon Sep 17 00:00:00 2001 From: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:49:57 +0100 Subject: [PATCH] Fix: list of managedClusters in e2e tests for AlertManager (#1820) * fix list of managedClusters Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * fix Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * fix getting cluster ID, add logs Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> --------- Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> --- tests/pkg/tests/observability_alert_test.go | 9 +- tests/pkg/utils/kube_debug.go | 22 +++++ tests/pkg/utils/mco_managedcluster.go | 102 ++++++++++++++------ 3 files changed, 97 insertions(+), 36 deletions(-) diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go index 11be37e82..adc501385 100644 --- a/tests/pkg/tests/observability_alert_test.go +++ b/tests/pkg/tests/observability_alert_test.go @@ -335,21 +335,22 @@ var _ = Describe("Observability:", func() { alertGetReq.Header.Set("Authorization", "Bearer "+BearerToken) } - expectedOCPClusterIDs, err := utils.ListOCPManagedClusterIDs(testOptions) + expectedOCPClusterIDs, err := utils.ListAvailableOCPManagedClusterIDs(testOptions) Expect(err).NotTo(HaveOccurred()) - expectedKSClusterNames, err := utils.ListKSManagedClusterNames(testOptions) + expectedKSClusterNames, err := utils.ListAvailableKSManagedClusterNames(testOptions) Expect(err).NotTo(HaveOccurred()) expectClusterIdentifiers := append(expectedOCPClusterIDs, expectedKSClusterNames...) missingClusters := slices.Clone(expectClusterIdentifiers) klog.Infof("List of cluster IDs expected to send the alert is: %s", expectClusterIdentifiers) - // Ensure we have all the managed clusters in the list - Expect(len(expectClusterIdentifiers)).To(Equal(len(testOptions.ManagedClusters) + 1)) + // Ensure we have at least a managedCluster + Expect(expectClusterIdentifiers).To(Not(BeEmpty())) // install watchdog PrometheusRule to *KS clusters watchDogRuleKustomizationPath := "../../../examples/alerts/watchdog_rule" yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: watchDogRuleKustomizationPath}) Expect(err).NotTo(HaveOccurred()) + klog.Infof("List of cluster IDs to install the watchdog alert: %s", expectedKSClusterNames) for _, ks := range expectedKSClusterNames { for idx, mc := range testOptions.ManagedClusters { if mc.Name == ks { diff --git a/tests/pkg/utils/kube_debug.go b/tests/pkg/utils/kube_debug.go index 14471825c..a3452aafd 100644 --- a/tests/pkg/utils/kube_debug.go +++ b/tests/pkg/utils/kube_debug.go @@ -16,6 +16,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/klog" ) @@ -48,6 +49,7 @@ func LogFailingTestStandardDebugInfo(opt TestOptions) { CheckPodsInNamespace(hubClient, MCO_NAMESPACE, []string{}, map[string]string{}) printConfigMapsInNamespace(hubClient, MCO_NAMESPACE) printSecretsInNamespace(hubClient, MCO_NAMESPACE) + LogManagedClusters(hubDynClient) for _, mc := range opt.ManagedClusters { if mc.Name == "local-cluster" { @@ -302,6 +304,26 @@ func LogObjectEvents(client kubernetes.Interface, ns string, kind string, name s klog.V(1).Infof("%s %q events: \n%s", kind, name, formattedEvents) } +func LogManagedClusters(client dynamic.Interface) { + objs, err := client.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("failed to get ManagedClusters: %v", err) + return + } + + var managedClustersData strings.Builder + managedClustersData.WriteString(">>>>>>>>>> Managed Clusters >>>>>>>>>>\n") + for _, obj := range objs.Items { + mc, err := json.MarshalIndent(obj, "", " ") + if err != nil { + klog.Errorf("Failed to marshal ManagedCluster %q: %s", obj.GetName(), err.Error()) + } + managedClustersData.WriteString(string(mc)) + } + managedClustersData.WriteString("<<<<<<<<<< Managed Clusters <<<<<<<<<<") + klog.Info(managedClustersData.String()) +} + func printPodsStatuses(pods []corev1.Pod) { writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) fmt.Fprintln(writer, "NAME\tSTATUS\tRESTARTS\tAGE") diff --git a/tests/pkg/utils/mco_managedcluster.go b/tests/pkg/utils/mco_managedcluster.go index a839d89de..c5cb5e506 100644 --- a/tests/pkg/utils/mco_managedcluster.go +++ b/tests/pkg/utils/mco_managedcluster.go @@ -7,11 +7,27 @@ package utils import ( "context" "errors" + "fmt" "os" + "slices" + "github.com/onsi/ginkgo" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "open-cluster-management.io/api/cluster/v1" ) +const ( + availableManagedClusterCondition = "ManagedClusterConditionAvailable" + idClusterClaim = "id.k8s.io" +) + +var openshiftLabelSelector = labels.SelectorFromValidatedSet(map[string]string{ + "vendor": "OpenShift", +}) + func UpdateObservabilityFromManagedCluster(opt TestOptions, enableObservability bool) error { clusterName := GetManagedClusterName(opt) if clusterName != "" { @@ -95,55 +111,77 @@ func ListManagedClusters(opt TestOptions) ([]string, error) { return clusterNames, nil } -func ListOCPManagedClusterIDs(opt TestOptions) ([]string, error) { - clientDynamic := GetKubeClientDynamic(opt, true) - objs, err := clientDynamic.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{}) +func ListAvailableOCPManagedClusterIDs(opt TestOptions) ([]string, error) { + managedClusters, err := GetManagedClusters(opt) if err != nil { return nil, err } - clusterIDs := []string{} - for _, obj := range objs.Items { - metadata := obj.Object["metadata"].(map[string]interface{}) - labels := metadata["labels"].(map[string]interface{}) - if labels == nil { - continue - } + // Filter out unavailable and non openshift clusters. + // This is necessary for some e2e testing environments where some managed clusters might not be available. + managedClusters = slices.DeleteFunc(managedClusters, func(e *clusterv1.ManagedCluster) bool { + return !meta.IsStatusConditionTrue(e.Status.Conditions, availableManagedClusterCondition) || !isOpenshiftVendor(e) + }) - if vendor, ok := labels["vendor"]; !ok || vendor != "OpenShift" { - continue - } + ret := make([]string, 0, len(managedClusters)) + for _, mc := range managedClusters { + ret = append(ret, getManagedClusterID(mc)) + } + + return ret, nil +} + +func ListAvailableKSManagedClusterNames(opt TestOptions) ([]string, error) { + managedClusters, err := GetManagedClusters(opt) + if err != nil { + return nil, err + } + + // Filter out unavailable and non openshift clusters. + // This is necessary for some e2e testing environments where some managed clusters might not be available. + managedClusters = slices.DeleteFunc(managedClusters, func(e *clusterv1.ManagedCluster) bool { + return !meta.IsStatusConditionTrue(e.Status.Conditions, availableManagedClusterCondition) || isOpenshiftVendor(e) + }) - if clusterID, ok := labels["clusterID"]; ok { - clusterIDs = append(clusterIDs, clusterID.(string)) + ret := make([]string, 0, len(managedClusters)) + for _, mc := range managedClusters { + ret = append(ret, getManagedClusterID(mc)) + } + + return ret, nil +} + +func isOpenshiftVendor(mc *clusterv1.ManagedCluster) bool { + return openshiftLabelSelector.Matches(labels.Set(mc.GetLabels())) +} + +func getManagedClusterID(mc *clusterv1.ManagedCluster) string { + for _, cc := range mc.Status.ClusterClaims { + if cc.Name == idClusterClaim { + return cc.Value } } - return clusterIDs, nil + ginkgo.Fail(fmt.Sprintf("failed to get the managedCluster %q ID", mc.Name)) + + return "" } -func ListKSManagedClusterNames(opt TestOptions) ([]string, error) { +func GetManagedClusters(opt TestOptions) ([]*clusterv1.ManagedCluster, error) { clientDynamic := GetKubeClientDynamic(opt, true) objs, err := clientDynamic.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{}) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get ManagedClusters: %w", err) } - clusterNames := []string{} - for _, obj := range objs.Items { - metadata := obj.Object["metadata"].(map[string]interface{}) - labels := metadata["labels"].(map[string]interface{}) - if labels == nil { - continue - } - if vendor, ok := labels["vendor"]; ok && vendor == "OpenShift" { - continue - } - - if clusterName, ok := labels["name"]; ok { - clusterNames = append(clusterNames, clusterName.(string)) + ret := make([]*clusterv1.ManagedCluster, 0, len(objs.Items)) + for _, obj := range objs.Items { + mc := &clusterv1.ManagedCluster{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, mc); err != nil { + return nil, fmt.Errorf("failed to convert Unstructured to ManagedCluster: %w", err) } + ret = append(ret, mc) } - return clusterNames, nil + return ret, nil }