Fix: list of managedClusters in e2e tests for AlertManager (#1820)

* fix list of managedClusters Signed-off-by: Thibault Mange <[email protected]> * fix Signed-off-by: Thibault Mange <[email protected]> * fix getting cluster ID, add logs Signed-off-by: Thibault Mange <[email protected]> --------- Signed-off-by: Thibault Mange <[email protected]>
stolostron · Feb 3, 2025 · 2eeddbf · 2eeddbf
1 parent c47b61c
commit 2eeddbf
Show file tree

Hide file tree

Showing 3 changed files with 97 additions and 36 deletions.
diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go
@@ -335,21 +335,22 @@ var _ = Describe("Observability:", func() {
 			alertGetReq.Header.Set("Authorization", "Bearer "+BearerToken)
 		}
 
-		expectedOCPClusterIDs, err := utils.ListOCPManagedClusterIDs(testOptions)
+		expectedOCPClusterIDs, err := utils.ListAvailableOCPManagedClusterIDs(testOptions)
 		Expect(err).NotTo(HaveOccurred())
-		expectedKSClusterNames, err := utils.ListKSManagedClusterNames(testOptions)
+		expectedKSClusterNames, err := utils.ListAvailableKSManagedClusterNames(testOptions)
 		Expect(err).NotTo(HaveOccurred())
 		expectClusterIdentifiers := append(expectedOCPClusterIDs, expectedKSClusterNames...)
 		missingClusters := slices.Clone(expectClusterIdentifiers)
 		klog.Infof("List of cluster IDs expected to send the alert is: %s", expectClusterIdentifiers)
 
-		// Ensure we have all the managed clusters in the list
-		Expect(len(expectClusterIdentifiers)).To(Equal(len(testOptions.ManagedClusters) + 1))
+		// Ensure we have at least a managedCluster
+		Expect(expectClusterIdentifiers).To(Not(BeEmpty()))
 
 		// install watchdog PrometheusRule to *KS clusters
 		watchDogRuleKustomizationPath := "../../../examples/alerts/watchdog_rule"
 		yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: watchDogRuleKustomizationPath})
 		Expect(err).NotTo(HaveOccurred())
+		klog.Infof("List of cluster IDs to install the watchdog alert: %s", expectedKSClusterNames)
 		for _, ks := range expectedKSClusterNames {
 			for idx, mc := range testOptions.ManagedClusters {
 				if mc.Name == ks {

diff --git a/tests/pkg/utils/kube_debug.go b/tests/pkg/utils/kube_debug.go
@@ -16,6 +16,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 )
@@ -48,6 +49,7 @@ func LogFailingTestStandardDebugInfo(opt TestOptions) {
 	CheckPodsInNamespace(hubClient, MCO_NAMESPACE, []string{}, map[string]string{})
 	printConfigMapsInNamespace(hubClient, MCO_NAMESPACE)
 	printSecretsInNamespace(hubClient, MCO_NAMESPACE)
+	LogManagedClusters(hubDynClient)
 
 	for _, mc := range opt.ManagedClusters {
 		if mc.Name == "local-cluster" {
@@ -302,6 +304,26 @@ func LogObjectEvents(client kubernetes.Interface, ns string, kind string, name s
 	klog.V(1).Infof("%s %q events: \n%s", kind, name, formattedEvents)
 }
 
+func LogManagedClusters(client dynamic.Interface) {
+	objs, err := client.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		klog.Errorf("failed to get ManagedClusters: %v", err)
+		return
+	}
+
+	var managedClustersData strings.Builder
+	managedClustersData.WriteString(">>>>>>>>>> Managed Clusters >>>>>>>>>>\n")
+	for _, obj := range objs.Items {
+		mc, err := json.MarshalIndent(obj, "", "  ")
+		if err != nil {
+			klog.Errorf("Failed to marshal ManagedCluster %q: %s", obj.GetName(), err.Error())
+		}
+		managedClustersData.WriteString(string(mc))
+	}
+	managedClustersData.WriteString("<<<<<<<<<< Managed Clusters <<<<<<<<<<")
+	klog.Info(managedClustersData.String())
+}
+
 func printPodsStatuses(pods []corev1.Pod) {
 	writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
 	fmt.Fprintln(writer, "NAME\tSTATUS\tRESTARTS\tAGE")

diff --git a/tests/pkg/utils/mco_managedcluster.go b/tests/pkg/utils/mco_managedcluster.go
@@ -7,11 +7,27 @@ package utils
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
+	"slices"
 
+	"github.com/onsi/ginkgo"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	clusterv1 "open-cluster-management.io/api/cluster/v1"
 )
 
+const (
+	availableManagedClusterCondition = "ManagedClusterConditionAvailable"
+	idClusterClaim                   = "id.k8s.io"
+)
+
+var openshiftLabelSelector = labels.SelectorFromValidatedSet(map[string]string{
+	"vendor": "OpenShift",
+})
+
 func UpdateObservabilityFromManagedCluster(opt TestOptions, enableObservability bool) error {
 	clusterName := GetManagedClusterName(opt)
 	if clusterName != "" {
@@ -95,55 +111,77 @@ func ListManagedClusters(opt TestOptions) ([]string, error) {
 	return clusterNames, nil
 }
 
-func ListOCPManagedClusterIDs(opt TestOptions) ([]string, error) {
-	clientDynamic := GetKubeClientDynamic(opt, true)
-	objs, err := clientDynamic.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{})
+func ListAvailableOCPManagedClusterIDs(opt TestOptions) ([]string, error) {
+	managedClusters, err := GetManagedClusters(opt)
 	if err != nil {
 		return nil, err
 	}
 
-	clusterIDs := []string{}
-	for _, obj := range objs.Items {
-		metadata := obj.Object["metadata"].(map[string]interface{})
-		labels := metadata["labels"].(map[string]interface{})
-		if labels == nil {
-			continue
-		}
+	// Filter out unavailable and non openshift clusters.
+	// This is necessary for some e2e testing environments where some managed clusters might not be available.
+	managedClusters = slices.DeleteFunc(managedClusters, func(e *clusterv1.ManagedCluster) bool {
+		return !meta.IsStatusConditionTrue(e.Status.Conditions, availableManagedClusterCondition) || !isOpenshiftVendor(e)
+	})
 
-		if vendor, ok := labels["vendor"]; !ok || vendor != "OpenShift" {
-			continue
-		}
+	ret := make([]string, 0, len(managedClusters))
+	for _, mc := range managedClusters {
+		ret = append(ret, getManagedClusterID(mc))
+	}
+
+	return ret, nil
+}
+
+func ListAvailableKSManagedClusterNames(opt TestOptions) ([]string, error) {
+	managedClusters, err := GetManagedClusters(opt)
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter out unavailable and non openshift clusters.
+	// This is necessary for some e2e testing environments where some managed clusters might not be available.
+	managedClusters = slices.DeleteFunc(managedClusters, func(e *clusterv1.ManagedCluster) bool {
+		return !meta.IsStatusConditionTrue(e.Status.Conditions, availableManagedClusterCondition) || isOpenshiftVendor(e)
+	})
 
-		if clusterID, ok := labels["clusterID"]; ok {
-			clusterIDs = append(clusterIDs, clusterID.(string))
+	ret := make([]string, 0, len(managedClusters))
+	for _, mc := range managedClusters {
+		ret = append(ret, getManagedClusterID(mc))
+	}
+
+	return ret, nil
+}
+
+func isOpenshiftVendor(mc *clusterv1.ManagedCluster) bool {
+	return openshiftLabelSelector.Matches(labels.Set(mc.GetLabels()))
+}
+
+func getManagedClusterID(mc *clusterv1.ManagedCluster) string {
+	for _, cc := range mc.Status.ClusterClaims {
+		if cc.Name == idClusterClaim {
+			return cc.Value
 		}
 	}
 
-	return clusterIDs, nil
+	ginkgo.Fail(fmt.Sprintf("failed to get the managedCluster %q ID", mc.Name))
+
+	return ""
 }
 
-func ListKSManagedClusterNames(opt TestOptions) ([]string, error) {
+func GetManagedClusters(opt TestOptions) ([]*clusterv1.ManagedCluster, error) {
 	clientDynamic := GetKubeClientDynamic(opt, true)
 	objs, err := clientDynamic.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get ManagedClusters: %w", err)
 	}
-	clusterNames := []string{}
-	for _, obj := range objs.Items {
-		metadata := obj.Object["metadata"].(map[string]interface{})
-		labels := metadata["labels"].(map[string]interface{})
-		if labels == nil {
-			continue
-		}
 
-		if vendor, ok := labels["vendor"]; ok && vendor == "OpenShift" {
-			continue
-		}
-
-		if clusterName, ok := labels["name"]; ok {
-			clusterNames = append(clusterNames, clusterName.(string))
+	ret := make([]*clusterv1.ManagedCluster, 0, len(objs.Items))
+	for _, obj := range objs.Items {
+		mc := &clusterv1.ManagedCluster{}
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, mc); err != nil {
+			return nil, fmt.Errorf("failed to convert Unstructured to ManagedCluster: %w", err)
 		}
+		ret = append(ret, mc)
 	}
 
-	return clusterNames, nil
+	return ret, nil
 }