fix test logging

Signed-off-by: Thibault Mange <[email protected]>

tests/pkg/tests/observability_export_test.go

@@ -5,11 +5,15 @@
 package tests
 
 import (
+	"context"
 	"fmt"
 	"os"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/klog"
 
 	"github.com/stolostron/multicluster-observability-operator/tests/pkg/kustomize"
 	"github.com/stolostron/multicluster-observability-operator/tests/pkg/utils"
@@ -98,9 +102,51 @@ var _ = Describe("Observability:", func() {
 
 	JustAfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
+			klog.V(1).Infof("Test failed, printing debug info. ManagedClusters: %v", clusters)
 			utils.PrintMCOObject(testOptions)
 			utils.PrintAllMCOPodsStatus(testOptions)
 			utils.PrintAllOBAPodsStatus(testOptions)
+			// Print allowList configmaps
+			for _, cluster := range testOptions.ManagedClusters {
+				client := utils.NewKubeClientDynamic(
+					cluster.ClusterServerURL,
+					cluster.KubeConfig,
+					cluster.KubeContext)
+				gvr := schema.GroupVersionResource{
+					Group:    "",
+					Version:  "v1",
+					Resource: "configmaps",
+				}
+				ns := utils.MCO_ADDON_NAMESPACE
+				if cluster.Name == "local-cluster" {
+					ns = utils.MCO_NAMESPACE
+				}
+
+				utils.PrintObject(context.Background(), client, gvr, ns, "observability-metrics-allowlist")
+			}
+
+			// Print logs of pods multicluster-observability-operator in ns open-cluster-management
+			client := utils.NewKubeClient(
+				testOptions.HubCluster.ClusterServerURL,
+				testOptions.KubeConfig,
+				testOptions.HubCluster.KubeContext)
+
+			pods, err := client.CoreV1().Pods("open-cluster-management").List(context.TODO(), metav1.ListOptions{
+				LabelSelector: "name: multicluster-observability-operator",
+			})
+			if err != nil {
+				klog.Errorf("Failed to get pods: %v", err)
+			}
+
+			if len(pods.Items) > 1 {
+				klog.Errorf("More than one pod found, got %d", len(pods.Items))
+			}
+
+			// get logs
+			for _, pod := range pods.Items {
+				utils.LogPodLogs(client, pod.Namespace, pod)
+			}
+
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/utils/mco_deploy.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 )
@@ -171,7 +172,7 @@ func PrintAllMCOPodsStatus(opt TestOptions) {
 	if err != nil {
 		klog.Errorf("Failed to get mch-image-manifest configmap: %s", err.Error())
 	} else {
-		klog.V(1).Infof("mch-image-manifest configmap: \n%v\n", mchImageManifestCM)
+		klog.V(1).Infof("mch-image-manifest configmap: \nmulticluster_observability_operator: %s\n", mchImageManifestCM["multicluster_observability_operator"])
 	}
 
 	klog.V(1).Infof("Pods in namespace %s: \n", MCO_NAMESPACE)
@@ -212,11 +213,6 @@ func LogPodsDebugInfo(hubClient kubernetes.Interface, pods []corev1.Pod, force b
 	klog.V(1).Infof("Checking %d pods in namespace %q", len(podsNames), ns)
 	notRunningPodsCount := 0
 	for _, pod := range pods {
-		// to delete
-		if strings.Contains(pod.GetName(), "multicluster-observability-operator") {
-			logPodLogs(hubClient, ns, pod)
-		}
-
 		if pod.Status.Phase != corev1.PodRunning {
 			notRunningPodsCount++
 		}
@@ -232,9 +228,9 @@ func LogPodsDebugInfo(hubClient kubernetes.Interface, pods []corev1.Pod, force b
 		}
 		klog.V(1).Infof("Pod %q spec: \n%s", pod.Name, string(podSpec))
 
-		logPodStatus(pod)
-		logPodEvents(hubClient, ns, pod.Name)
-		logPodLogs(hubClient, ns, pod)
+		LogPodStatus(pod)
+		LogPodEvents(hubClient, ns, pod.Name)
+		LogPodLogs(hubClient, ns, pod)
 	}
 
 	if notRunningPodsCount == 0 {
@@ -244,7 +240,7 @@ func LogPodsDebugInfo(hubClient kubernetes.Interface, pods []corev1.Pod, force b
 	}
 }
 
-func logPodStatus(pod corev1.Pod) {
+func LogPodStatus(pod corev1.Pod) {
 	var podStatus strings.Builder
 	podStatus.WriteString(">>>>>>>>>> pod status >>>>>>>>>>\n")
 	podStatus.WriteString("Conditions:\n")
@@ -263,7 +259,7 @@ func logPodStatus(pod corev1.Pod) {
 	klog.V(1).Infof("Pod %q is in phase %q and status: \n%s", pod.Name, pod.Status.Phase, podStatus.String())
 }
 
-func logPodEvents(client kubernetes.Interface, ns string, podName string) {
+func LogPodEvents(client kubernetes.Interface, ns string, podName string) {
 	events, err := client.CoreV1().Events(ns).List(context.TODO(), metav1.ListOptions{
 		FieldSelector: "involvedObject.name=" + podName,
 	})
@@ -279,7 +275,7 @@ func logPodEvents(client kubernetes.Interface, ns string, podName string) {
 	klog.V(1).Infof("Pod %q events: \n%s", podName, formattedEvents)
 }
 
-func logPodLogs(client kubernetes.Interface, ns string, pod corev1.Pod) {
+func LogPodLogs(client kubernetes.Interface, ns string, pod corev1.Pod) {
 	for _, container := range pod.Spec.Containers {
 		logsRes := client.CoreV1().Pods(ns).GetLogs(pod.Name, &corev1.PodLogOptions{
 			Container: container.Name,
@@ -336,6 +332,34 @@ func PrintMCOObject(opt TestOptions) {
 	klog.V(1).Infof("MCO status: %+v\n", string(status))
 }
 
+func PrintObject(ctx context.Context, client dynamic.Interface, gvr schema.GroupVersionResource, ns, name string) {
+	if ns == "" || name == "" {
+		klog.V(1).Info("Namespace or name cannot be empty")
+		return
+	}
+
+	obj, err := client.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		klog.V(1).Infof("Failed to get object %s in namespace %s: %v", name, ns, err)
+		return
+	}
+
+	spec, err := json.MarshalIndent(obj.Object["spec"], "", "  ")
+	if err != nil {
+		klog.V(1).Infof("Failed to marshal spec for object %s in namespace %s: %v", name, ns, err)
+		return
+	}
+
+	status, err := json.MarshalIndent(obj.Object["status"], "", "  ")
+	if err != nil {
+		klog.V(1).Infof("Failed to marshal status for object %s in namespace %s: %v", name, ns, err)
+		return
+	}
+
+	klog.V(1).Infof("Object %s in namespace %s spec: %+v\n", name, ns, string(spec))
+	klog.V(1).Infof("Object %s in namespace %s status: %+v\n", name, ns, string(status))
+}
+
 func PrintManagedClusterOBAObject(opt TestOptions) {
 	clientDynamic := GetKubeClientDynamic(opt, false)
 	oba, getErr := clientDynamic.Resource(NewMCOAddonGVR()).
@@ -362,13 +386,16 @@ func GetAllOBAPods(client kubernetes.Interface) ([]corev1.Pod, error) {
 }
 
 func PrintAllOBAPodsStatus(opt TestOptions) {
+	klog.V(1).Infof("Get OBA pods status from managed clusters: %v", opt.ManagedClusters)
+
 	if len(opt.ManagedClusters) == 0 {
 		klog.V(1).Infof("No managedclusters found")
 		return
 	}
 
 	for _, mc := range opt.ManagedClusters {
 		if mc.Name == "local-cluster" {
+			// skip as those pods are already printed in PrintAllMCOPodsStatus
 			continue
 		}