Improve e2e logs

tests/pkg/tests/observability-e2e-test_suite_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 	"gopkg.in/yaml.v2"
@@ -129,7 +130,10 @@ func init() {
 
 func TestObservabilityE2E(t *testing.T) {
 	RegisterFailHandler(Fail)
+	config.DefaultReporterConfig.NoColor = true
+	config.DefaultReporterConfig.Succinct = true
 	junitReporter := reporters.NewJUnitReporter(reportFile)
+	junitReporter.ReporterConfig.NoColor = true
 	RunSpecsWithDefaultAndCustomReporters(t, "Observability E2E Suite", []Reporter{junitReporter})
 }
 
@@ -141,8 +145,6 @@ var _ = BeforeSuite(func() {
 var _ = AfterSuite(func() {
 	if !testFailed {
 		uninstallMCO()
-	} else {
-		utils.PrintAllMCOPodsStatus(testOptions)
 	}
 })
 

tests/pkg/tests/observability_addon_test.go

@@ -105,17 +105,18 @@ var _ = Describe("Observability:", func() {
 		It("[Stable] Waiting for check no metric data in grafana console", func() {
 			Eventually(func() error {
 				for _, cluster := range clusters {
-					err, hasMetric := utils.ContainManagedClusterMetric(
+					res, err := utils.QueryGrafana(
 						testOptions,
 						`timestamp(node_memory_MemAvailable_bytes{cluster="`+cluster+`}) - timestamp(node_memory_MemAvailable_bytes{cluster=`+cluster+`"} offset 1m) > 59`,
-						[]string{`"__name__":"node_memory_MemAvailable_bytes"`},
 					)
-					if err != nil && !hasMetric &&
-						strings.Contains(err.Error(), "failed to find metric name from response") {
-						return nil
+					if err != nil {
+						return err
+					}
+					if len(res.Data.Result) != 0 {
+						return fmt.Errorf("Grafa console still has metric data: %v", res.Data.Result)
 					}
 				}
-				return fmt.Errorf("Check no metric data in grafana console error: %w", err)
+				return nil
 			}, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*5).Should(Succeed())
 		})
 
@@ -219,10 +220,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
-			utils.PrintManagedClusterOBAObject(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/tests/observability_alert_test.go

@@ -191,9 +191,15 @@ var _ = Describe("Observability:", func() {
 
 		By("Checking alert generated")
 		Eventually(func() error {
-			err, _ := utils.ContainManagedClusterMetric(testOptions, `ALERTS{`+labelName+`="`+labelValue+`"}`,
-				[]string{`"__name__":"ALERTS"`, `"` + labelName + `":"` + labelValue + `"`})
-			return err
+			query := fmt.Sprintf(`ALERTS{%s="%s"}`, labelName, labelValue)
+			res, err := utils.QueryGrafana(testOptions, query)
+			if err != nil {
+				return err
+			}
+			if len(res.Data.Result) == 0 {
+				return fmt.Errorf("no data found for %s", query)
+			}
+			return nil
 		}, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed())
 	})
 
@@ -217,6 +223,7 @@ var _ = Describe("Observability:", func() {
 	It("[P2][Sev2][observability][Stable] Should have custom alert updated (alert/g0)", func() {
 		By("Updating custom alert rules")
 
+		// Replace preceding custom alert with new one that cannot fire
 		yamlB, _ := kustomize.Render(
 			kustomize.Options{KustomizationPath: "../../../examples/alerts/custom_rules_invalid"},
 		)
@@ -236,12 +243,21 @@ var _ = Describe("Observability:", func() {
 		By("Checking alert generated")
 		Eventually(
 			func() error {
-				err, _ := utils.ContainManagedClusterMetric(testOptions, `ALERTS{`+labelName+`="`+labelValue+`"}`,
-					[]string{`"__name__":"ALERTS"`, `"` + labelName + `":"` + labelValue + `"`})
-				return err
+				query := fmt.Sprintf(`ALERTS{%s="%s"}`, labelName, labelValue)
+				res, err := utils.QueryGrafana(testOptions, query)
+				if err != nil {
+					return err
+				}
+
+				if len(res.Data.Result) != 0 {
+					// No alert should be generated
+					return fmt.Errorf("alert should not be generated, got %v", res)
+				}
+
+				return nil
 			},
 			EventuallyTimeoutMinute*5,
-			EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response"))
+			EventuallyIntervalSecond*5).Should(Succeed())
 	})
 
 	It("[P2][Sev2][observability][Stable] delete the customized rules (alert/g0)", func() {
@@ -394,9 +410,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/tests/observability_certrenew_test.go

@@ -150,18 +150,19 @@ var _ = Describe("Observability:", func() {
 				namespace,
 				"component=metrics-collector",
 			)
-			if err == nil {
-				for _, pod := range podList.Items {
-					if pod.Name != collectorPodName {
-						if pod.Status.Phase != "Running" {
-							klog.V(1).Infof("<%s> not in Running status yet", pod.Name)
-							return false
-						}
-						return true
+			if err != nil {
+				klog.V(1).Infof("Failed to get pod list: %v", err)
+			}
+			for _, pod := range podList.Items {
+				if pod.Name != collectorPodName {
+					if pod.Status.Phase != "Running" {
+						klog.V(1).Infof("<%s> not in Running status yet", pod.Name)
+						return false
 					}
+					return true
 				}
-
 			}
+
 			// debug code to check label "cert/time-restarted"
 			deployment, err := utils.GetDeployment(
 				testOptions,
@@ -182,9 +183,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 		namespace = MCO_ADDON_NAMESPACE

tests/pkg/tests/observability_config_test.go

@@ -279,9 +279,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/tests/observability_dashboard_test.go

@@ -85,9 +85,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/tests/observability_endpoint_preserve_test.go

@@ -206,9 +206,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		namespace = MCO_ADDON_NAMESPACE
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed

tests/pkg/tests/observability_export_test.go

@@ -5,7 +5,6 @@
 package tests
 
 import (
-	"errors"
 	"fmt"
 	"os"
 
@@ -77,47 +76,42 @@ var _ = Describe("Observability:", func() {
 		By("Waiting for metrics acm_remote_write_requests_total on grafana console")
 		Eventually(func() error {
 			query := fmt.Sprintf("acm_remote_write_requests_total{cluster=\"%s\"} offset 1m", hubClusterName)
-			err, _ := utils.ContainManagedClusterMetric(
+			res, err := utils.QueryGrafana(
 				testOptions,
 				query,
-				[]string{`"__name__":"acm_remote_write_requests_total"`},
 			)
 			if err != nil {
 				return err
 			}
-			err, _ = utils.ContainManagedClusterMetric(
-				testOptions,
-				query,
-				[]string{`"__name__":"acm_remote_write_requests_total"`,
-					`"code":"200`, `"name":"thanos-receiver"`},
-			)
-			if err != nil {
-				return errors.New("metrics not forwarded to thanos-receiver")
+			if len(res.Data.Result) == 0 {
+				return fmt.Errorf("metric %s not found in response", query)
 			}
-			err, _ = utils.ContainManagedClusterMetric(
-				testOptions,
-				query,
-				[]string{`"__name__":"acm_remote_write_requests_total"`,
-					`"code":"204`, `"name":"victoriametrics"`},
-			)
-			if err != nil {
-				return errors.New("metrics not forwarded to victoriametrics")
+
+			// Check if the metric is forwarded to thanos-receiver
+			labelSet := map[string]string{"code": "200", "name": "thanos-receiver"}
+			if !res.ContainsLabelsSet(labelSet) {
+				return fmt.Errorf("labels %v not found in response: %v", labelSet, res)
+			}
+
+			// Check if the metric is forwarded to victoriametrics
+			labelSet = map[string]string{"code": "204", "name": "victoriametrics"}
+			if !res.ContainsLabelsSet(labelSet) {
+				return fmt.Errorf("labels %v not found in response: %v", labelSet, res)
 			}
+
 			return nil
-		}, EventuallyTimeoutMinute*20, EventuallyIntervalSecond*5).Should(Succeed())
+		}, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed())
 	})
 
 	JustAfterEach(func() {
-		Expect(utils.CleanExportResources(testOptions)).NotTo(HaveOccurred())
-		Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred())
-	})
-
-	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})
+
+	AfterEach(func() {
+		Expect(utils.CleanExportResources(testOptions)).NotTo(HaveOccurred())
+		Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred())
+	})
 })

tests/pkg/tests/observability_grafana_dev_test.go

@@ -35,9 +35,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/tests/observability_grafana_test.go

@@ -34,14 +34,17 @@ var _ = Describe("Observability:", func() {
 			}
 			for _, cluster := range clusters {
 				query := fmt.Sprintf("node_memory_MemAvailable_bytes{cluster=\"%s\"}", cluster)
-				err, _ = utils.ContainManagedClusterMetric(
+				res, err := utils.QueryGrafana(
 					testOptions,
 					query,
-					[]string{`"__name__":"node_memory_MemAvailable_bytes"`},
 				)
 				if err != nil {
 					return err
 				}
+
+				if len(res.Data.Result) == 0 {
+					return fmt.Errorf("no data found for %s", query)
+				}
 			}
 			return nil
 		}, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed())
@@ -53,9 +56,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})

tests/pkg/tests/observability_install_test.go

@@ -190,7 +190,7 @@ func installMCO() {
 		mcoLogs, err := utils.GetPodLogs(testOptions, true, mcoNs, mcoPod, "multicluster-observability-operator", false, 1000)
 		Expect(err).NotTo(HaveOccurred())
 		fmt.Fprintf(GinkgoWriter, "[DEBUG] MCO is installed failed, checking MCO operator logs:\n%s\n", mcoLogs)
-		utils.PrintAllMCOPodsStatus(testOptions)
+		utils.LogFailingTestStandardDebugInfo(testOptions)
 
 	}()
 	By("Waiting for MCO ready status")
@@ -214,7 +214,7 @@ func installMCO() {
 		}
 
 		fmt.Fprintf(GinkgoWriter, "[DEBUG] Addon failed, checking pods:\n")
-		utils.PrintAllOBAPodsStatus(testOptions)
+		utils.LogFailingTestStandardDebugInfo(testOptions)
 	}()
 	By("Check endpoint-operator and metrics-collector pods are ready")
 	Eventually(func() error {

tests/pkg/tests/observability_manifestwork_test.go

@@ -7,6 +7,7 @@ package tests
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -106,12 +107,20 @@ var _ = Describe("Observability:", func() {
 
 			It("[Stable] Checking metric to ensure that no data is lost in 1 minute", func() {
 				Eventually(func() error {
-					err, _ = utils.ContainManagedClusterMetric(
+					query := fmt.Sprintf(`timestamp(node_memory_MemAvailable_bytes{cluster="%s"}) - timestamp(node_memory_MemAvailable_bytes{cluster="%s"} offset 1m) > 59`, clusterName, clusterName)
+					res, err := utils.QueryGrafana(
 						testOptions,
-						`timestamp(node_memory_MemAvailable_bytes{cluster="`+clusterName+`}) - timestamp(node_memory_MemAvailable_bytes{cluster=`+clusterName+`"} offset 1m) > 59`,
-						[]string{`"__name__":"node_memory_MemAvailable_bytes"`},
+						query,
 					)
-					return err
+					if err != nil {
+						return err
+					}
+
+					if len(res.Data.Result) == 0 {
+						return fmt.Errorf("no data found for %s", query)
+					}
+
+					return nil
 				}, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*3).Should(Succeed())
 			})
 		}
@@ -123,9 +132,7 @@ var _ = Describe("Observability:", func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			utils.PrintMCOObject(testOptions)
-			utils.PrintAllMCOPodsStatus(testOptions)
-			utils.PrintAllOBAPodsStatus(testOptions)
+			utils.LogFailingTestStandardDebugInfo(testOptions)
 		}
 		testFailed = testFailed || CurrentGinkgoTestDescription().Failed
 	})