From fad0df8be1b9b27cefdaa0a0b0802cd784b394f7 Mon Sep 17 00:00:00 2001 From: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> Date: Wed, 22 May 2024 11:57:13 +0200 Subject: [PATCH] Improve e2e logs (#1435) * change test ContainManagedClusterMetric Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * improve e2e test logging Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * add kube_debug file Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * remove unused functions Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * reduce log lines Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * log statefulsets and daemonsets Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * add copyright Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> * fix pods list, add cm and secrets list Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> --------- Signed-off-by: Thibault Mange <22740367+thibaultmg@users.noreply.github.com> --- .../observability-e2e-test_suite_test.go | 6 +- tests/pkg/tests/observability_addon_test.go | 18 +- tests/pkg/tests/observability_alert_test.go | 34 +- .../pkg/tests/observability_certrenew_test.go | 23 +- tests/pkg/tests/observability_config_test.go | 4 +- .../pkg/tests/observability_dashboard_test.go | 4 +- .../observability_endpoint_preserve_test.go | 4 +- tests/pkg/tests/observability_export_test.go | 50 +- .../tests/observability_grafana_dev_test.go | 4 +- tests/pkg/tests/observability_grafana_test.go | 11 +- tests/pkg/tests/observability_install_test.go | 4 +- .../tests/observability_manifestwork_test.go | 21 +- tests/pkg/tests/observability_metrics_test.go | 66 +-- ...servability_observatorium_preserve_test.go | 34 +- .../pkg/tests/observability_reconcile_test.go | 4 +- .../pkg/tests/observability_retention_test.go | 4 +- tests/pkg/tests/observability_route_test.go | 4 +- .../pkg/tests/observability_uninstall_test.go | 2 +- tests/pkg/utils/cluster_deploy.go | 44 -- tests/pkg/utils/install_config.go | 56 --- tests/pkg/utils/kube_debug.go | 434 ++++++++++++++++++ tests/pkg/utils/mco_configmaps.go | 26 -- tests/pkg/utils/mco_deploy.go | 271 +---------- tests/pkg/utils/mco_metric.go | 80 ++-- tests/pkg/utils/mco_pods.go | 10 - tests/pkg/utils/utils.go | 180 +------- 26 files changed, 658 insertions(+), 740 deletions(-) delete mode 100644 tests/pkg/utils/cluster_deploy.go delete mode 100644 tests/pkg/utils/install_config.go create mode 100644 tests/pkg/utils/kube_debug.go diff --git a/tests/pkg/tests/observability-e2e-test_suite_test.go b/tests/pkg/tests/observability-e2e-test_suite_test.go index ef94ef3639..4575459e7b 100644 --- a/tests/pkg/tests/observability-e2e-test_suite_test.go +++ b/tests/pkg/tests/observability-e2e-test_suite_test.go @@ -13,6 +13,7 @@ import ( "time" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "gopkg.in/yaml.v2" @@ -129,7 +130,10 @@ func init() { func TestObservabilityE2E(t *testing.T) { RegisterFailHandler(Fail) + config.DefaultReporterConfig.NoColor = true + config.DefaultReporterConfig.Succinct = true junitReporter := reporters.NewJUnitReporter(reportFile) + junitReporter.ReporterConfig.NoColor = true RunSpecsWithDefaultAndCustomReporters(t, "Observability E2E Suite", []Reporter{junitReporter}) } @@ -141,8 +145,6 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { if !testFailed { uninstallMCO() - } else { - utils.PrintAllMCOPodsStatus(testOptions) } }) diff --git a/tests/pkg/tests/observability_addon_test.go b/tests/pkg/tests/observability_addon_test.go index 321748964c..f9ce37591a 100644 --- a/tests/pkg/tests/observability_addon_test.go +++ b/tests/pkg/tests/observability_addon_test.go @@ -105,17 +105,18 @@ var _ = Describe("Observability:", func() { It("[Stable] Waiting for check no metric data in grafana console", func() { Eventually(func() error { for _, cluster := range clusters { - err, hasMetric := utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, `timestamp(node_memory_MemAvailable_bytes{cluster="`+cluster+`}) - timestamp(node_memory_MemAvailable_bytes{cluster=`+cluster+`"} offset 1m) > 59`, - []string{`"__name__":"node_memory_MemAvailable_bytes"`}, ) - if err != nil && !hasMetric && - strings.Contains(err.Error(), "failed to find metric name from response") { - return nil + if err != nil { + return err + } + if len(res.Data.Result) != 0 { + return fmt.Errorf("Grafa console still has metric data: %v", res.Data.Result) } } - return fmt.Errorf("Check no metric data in grafana console error: %w", err) + return nil }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*5).Should(Succeed()) }) @@ -219,10 +220,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) - utils.PrintManagedClusterOBAObject(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go index a25df77e52..0d21d1aa1a 100644 --- a/tests/pkg/tests/observability_alert_test.go +++ b/tests/pkg/tests/observability_alert_test.go @@ -191,9 +191,15 @@ var _ = Describe("Observability:", func() { By("Checking alert generated") Eventually(func() error { - err, _ := utils.ContainManagedClusterMetric(testOptions, `ALERTS{`+labelName+`="`+labelValue+`"}`, - []string{`"__name__":"ALERTS"`, `"` + labelName + `":"` + labelValue + `"`}) - return err + query := fmt.Sprintf(`ALERTS{%s="%s"}`, labelName, labelValue) + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } + return nil }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) @@ -217,6 +223,7 @@ var _ = Describe("Observability:", func() { It("[P2][Sev2][observability][Stable] Should have custom alert updated (alert/g0)", func() { By("Updating custom alert rules") + // Replace preceding custom alert with new one that cannot fire yamlB, _ := kustomize.Render( kustomize.Options{KustomizationPath: "../../../examples/alerts/custom_rules_invalid"}, ) @@ -236,12 +243,21 @@ var _ = Describe("Observability:", func() { By("Checking alert generated") Eventually( func() error { - err, _ := utils.ContainManagedClusterMetric(testOptions, `ALERTS{`+labelName+`="`+labelValue+`"}`, - []string{`"__name__":"ALERTS"`, `"` + labelName + `":"` + labelValue + `"`}) - return err + query := fmt.Sprintf(`ALERTS{%s="%s"}`, labelName, labelValue) + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + + if len(res.Data.Result) != 0 { + // No alert should be generated + return fmt.Errorf("alert should not be generated, got %v", res) + } + + return nil }, EventuallyTimeoutMinute*5, - EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Stable] delete the customized rules (alert/g0)", func() { @@ -394,9 +410,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_certrenew_test.go b/tests/pkg/tests/observability_certrenew_test.go index 3e2b303bb3..cdebc9b8c4 100644 --- a/tests/pkg/tests/observability_certrenew_test.go +++ b/tests/pkg/tests/observability_certrenew_test.go @@ -150,18 +150,19 @@ var _ = Describe("Observability:", func() { namespace, "component=metrics-collector", ) - if err == nil { - for _, pod := range podList.Items { - if pod.Name != collectorPodName { - if pod.Status.Phase != "Running" { - klog.V(1).Infof("<%s> not in Running status yet", pod.Name) - return false - } - return true + if err != nil { + klog.V(1).Infof("Failed to get pod list: %v", err) + } + for _, pod := range podList.Items { + if pod.Name != collectorPodName { + if pod.Status.Phase != "Running" { + klog.V(1).Infof("<%s> not in Running status yet", pod.Name) + return false } + return true } - } + // debug code to check label "cert/time-restarted" deployment, err := utils.GetDeployment( testOptions, @@ -182,9 +183,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed namespace = MCO_ADDON_NAMESPACE diff --git a/tests/pkg/tests/observability_config_test.go b/tests/pkg/tests/observability_config_test.go index 00b63d9ecd..10336a7ed6 100644 --- a/tests/pkg/tests/observability_config_test.go +++ b/tests/pkg/tests/observability_config_test.go @@ -279,9 +279,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_dashboard_test.go b/tests/pkg/tests/observability_dashboard_test.go index 80cd3417e4..a2ab147658 100644 --- a/tests/pkg/tests/observability_dashboard_test.go +++ b/tests/pkg/tests/observability_dashboard_test.go @@ -85,9 +85,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_endpoint_preserve_test.go b/tests/pkg/tests/observability_endpoint_preserve_test.go index 556d333d56..50752c6e88 100644 --- a/tests/pkg/tests/observability_endpoint_preserve_test.go +++ b/tests/pkg/tests/observability_endpoint_preserve_test.go @@ -213,9 +213,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } namespace = MCO_ADDON_NAMESPACE testFailed = testFailed || CurrentGinkgoTestDescription().Failed diff --git a/tests/pkg/tests/observability_export_test.go b/tests/pkg/tests/observability_export_test.go index 630cb18bf8..508ca1159b 100644 --- a/tests/pkg/tests/observability_export_test.go +++ b/tests/pkg/tests/observability_export_test.go @@ -5,7 +5,6 @@ package tests import ( - "errors" "fmt" "os" @@ -77,47 +76,42 @@ var _ = Describe("Observability:", func() { By("Waiting for metrics acm_remote_write_requests_total on grafana console") Eventually(func() error { query := fmt.Sprintf("acm_remote_write_requests_total{cluster=\"%s\"} offset 1m", hubClusterName) - err, _ := utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, query, - []string{`"__name__":"acm_remote_write_requests_total"`}, ) if err != nil { return err } - err, _ = utils.ContainManagedClusterMetric( - testOptions, - query, - []string{`"__name__":"acm_remote_write_requests_total"`, - `"code":"200`, `"name":"thanos-receiver"`}, - ) - if err != nil { - return errors.New("metrics not forwarded to thanos-receiver") + if len(res.Data.Result) == 0 { + return fmt.Errorf("metric %s not found in response", query) } - err, _ = utils.ContainManagedClusterMetric( - testOptions, - query, - []string{`"__name__":"acm_remote_write_requests_total"`, - `"code":"204`, `"name":"victoriametrics"`}, - ) - if err != nil { - return errors.New("metrics not forwarded to victoriametrics") + + // Check if the metric is forwarded to thanos-receiver + labelSet := map[string]string{"code": "200", "name": "thanos-receiver"} + if !res.ContainsLabelsSet(labelSet) { + return fmt.Errorf("labels %v not found in response: %v", labelSet, res) + } + + // Check if the metric is forwarded to victoriametrics + labelSet = map[string]string{"code": "204", "name": "victoriametrics"} + if !res.ContainsLabelsSet(labelSet) { + return fmt.Errorf("labels %v not found in response: %v", labelSet, res) } + return nil - }, EventuallyTimeoutMinute*20, EventuallyIntervalSecond*5).Should(Succeed()) + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) JustAfterEach(func() { - Expect(utils.CleanExportResources(testOptions)).NotTo(HaveOccurred()) - Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) - }) - - AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) + + AfterEach(func() { + Expect(utils.CleanExportResources(testOptions)).NotTo(HaveOccurred()) + Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) + }) }) diff --git a/tests/pkg/tests/observability_grafana_dev_test.go b/tests/pkg/tests/observability_grafana_dev_test.go index 09ed3da07b..13fe02af20 100644 --- a/tests/pkg/tests/observability_grafana_dev_test.go +++ b/tests/pkg/tests/observability_grafana_dev_test.go @@ -35,9 +35,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_grafana_test.go b/tests/pkg/tests/observability_grafana_test.go index 8186561609..908351401a 100644 --- a/tests/pkg/tests/observability_grafana_test.go +++ b/tests/pkg/tests/observability_grafana_test.go @@ -34,14 +34,17 @@ var _ = Describe("Observability:", func() { } for _, cluster := range clusters { query := fmt.Sprintf("node_memory_MemAvailable_bytes{cluster=\"%s\"}", cluster) - err, _ = utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, query, - []string{`"__name__":"node_memory_MemAvailable_bytes"`}, ) if err != nil { return err } + + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } } return nil }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) @@ -53,9 +56,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_install_test.go b/tests/pkg/tests/observability_install_test.go index 90340c11b0..485196ba3f 100644 --- a/tests/pkg/tests/observability_install_test.go +++ b/tests/pkg/tests/observability_install_test.go @@ -190,7 +190,7 @@ func installMCO() { mcoLogs, err := utils.GetPodLogs(testOptions, true, mcoNs, mcoPod, "multicluster-observability-operator", false, 1000) Expect(err).NotTo(HaveOccurred()) fmt.Fprintf(GinkgoWriter, "[DEBUG] MCO is installed failed, checking MCO operator logs:\n%s\n", mcoLogs) - utils.PrintAllMCOPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) }() By("Waiting for MCO ready status") @@ -214,7 +214,7 @@ func installMCO() { } fmt.Fprintf(GinkgoWriter, "[DEBUG] Addon failed, checking pods:\n") - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) }() By("Check endpoint-operator and metrics-collector pods are ready") Eventually(func() error { diff --git a/tests/pkg/tests/observability_manifestwork_test.go b/tests/pkg/tests/observability_manifestwork_test.go index f654a6ebc0..193abd6155 100644 --- a/tests/pkg/tests/observability_manifestwork_test.go +++ b/tests/pkg/tests/observability_manifestwork_test.go @@ -7,6 +7,7 @@ package tests import ( "context" "errors" + "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -106,12 +107,20 @@ var _ = Describe("Observability:", func() { It("[Stable] Checking metric to ensure that no data is lost in 1 minute", func() { Eventually(func() error { - err, _ = utils.ContainManagedClusterMetric( + query := fmt.Sprintf(`timestamp(node_memory_MemAvailable_bytes{cluster="%s"}) - timestamp(node_memory_MemAvailable_bytes{cluster="%s"} offset 1m) > 59`, clusterName, clusterName) + res, err := utils.QueryGrafana( testOptions, - `timestamp(node_memory_MemAvailable_bytes{cluster="`+clusterName+`}) - timestamp(node_memory_MemAvailable_bytes{cluster=`+clusterName+`"} offset 1m) > 59`, - []string{`"__name__":"node_memory_MemAvailable_bytes"`}, + query, ) - return err + if err != nil { + return err + } + + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } + + return nil }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*3).Should(Succeed()) }) } @@ -123,9 +132,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_metrics_test.go b/tests/pkg/tests/observability_metrics_test.go index 6e6ed9c381..03f1b59e21 100644 --- a/tests/pkg/tests/observability_metrics_test.go +++ b/tests/pkg/tests/observability_metrics_test.go @@ -11,7 +11,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" "github.com/stolostron/multicluster-observability-operator/tests/pkg/kustomize" "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" @@ -22,9 +21,8 @@ const ( ) var ( - clusters []string - clusterError error - metricslistError error + clusters []string + clusterError error ) var _ = Describe("Observability:", func() { @@ -66,14 +64,17 @@ var _ = Describe("Observability:", func() { Eventually(func() error { for _, cluster := range clusters { query := fmt.Sprintf("node_memory_Active_bytes{cluster=\"%s\"} offset 1m", cluster) - err, _ := utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, query, - []string{`"__name__":"node_memory_Active_bytes"`}, ) if err != nil { return err } + + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } } return nil }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) @@ -88,13 +89,17 @@ var _ = Describe("Observability:", func() { cluster, cluster, ) - metricslistError, _ = utils.ContainManagedClusterMetric(testOptions, query, []string{}) - if metricslistError == nil { - return nil + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + // there should be no data for the deleted metric + if len(res.Data.Result) != 0 { + return fmt.Errorf("metric %s found in response: %v", query, res) } } - return metricslistError - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Integration] Should have no metrics which have been marked for deletion in matches section (metrics/g0)", func() { @@ -106,13 +111,16 @@ var _ = Describe("Observability:", func() { cluster, cluster, ) - metricslistError, _ = utils.ContainManagedClusterMetric(testOptions, query, []string{}) - if metricslistError == nil { - return nil + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + if len(res.Data.Result) != 0 { + return fmt.Errorf("metric %s found in response: %v", query, res) } } - return metricslistError - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Integration] Should have no metrics after custom metrics allowlist deleted (metrics/g0)", func() { @@ -132,13 +140,16 @@ var _ = Describe("Observability:", func() { cluster, cluster, ) - metricslistError, _ = utils.ContainManagedClusterMetric(testOptions, query, []string{}) - if metricslistError == nil { - return nil + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + if len(res.Data.Result) != 0 { + return fmt.Errorf("metric %s found in response: %v", query, res) } } - return metricslistError - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Integration] Should have metrics which used grafana dashboard (ssli/g1)", func() { @@ -162,11 +173,14 @@ var _ = Describe("Observability:", func() { _, ok := ignoreMetricMap[name] if !ok { Eventually(func() error { - err, _ := utils.ContainManagedClusterMetric(testOptions, name, []string{name}) + res, err := utils.QueryGrafana(testOptions, name) if err != nil { - klog.V(1).Infof("failed to get metrics %s", name) + return fmt.Errorf("failed to get metrics %s: %v", name, err) } - return err + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", name) + } + return nil }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*3).Should(Succeed()) } } @@ -178,9 +192,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_observatorium_preserve_test.go b/tests/pkg/tests/observability_observatorium_preserve_test.go index d2786f420b..b1bbd2aabf 100644 --- a/tests/pkg/tests/observability_observatorium_preserve_test.go +++ b/tests/pkg/tests/observability_observatorium_preserve_test.go @@ -6,6 +6,8 @@ package tests import ( "context" + "errors" + "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -70,21 +72,25 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*3, EventuallyIntervalSecond*1).Should(BeTrue()) // ensure the thanos compact is restarted - Eventually(func() bool { + Eventually(func() error { sts, err := utils.GetStatefulSetWithLabel(testOptions, true, THANOS_COMPACT_LABEL, MCO_NAMESPACE) - if err == nil { - if (*sts).Items[0].ResourceVersion != oldCompactResourceVersion { - argList := (*sts).Items[0].Spec.Template.Spec.Containers[0].Args - for _, arg := range argList { - if arg != "--retention.resolution-raw="+updateRetention { - return true - } - } - return false + if err != nil { + return err + } + if sts.Items[0].ResourceVersion != oldCompactResourceVersion { + return errors.New("The thanos compact pod is not restarted. ResourceVersion has not changed.") + } + + argList := sts.Items[0].Spec.Template.Spec.Containers[0].Args + for _, arg := range argList { + // check if the retention resolution is reverted to the original value + if arg == "--retention.resolution-raw="+updateRetention { + return fmt.Errorf("The thanos compact pod is not restarted with the new retention resolution. Args: %v", argList) } } - return false - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(BeTrue()) + + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) By("Wait for thanos compact pods are ready") sts, err := utils.GetStatefulSetWithLabel(testOptions, true, THANOS_COMPACT_LABEL, MCO_NAMESPACE) @@ -107,9 +113,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_reconcile_test.go b/tests/pkg/tests/observability_reconcile_test.go index 8ed55bac45..c37470a648 100644 --- a/tests/pkg/tests/observability_reconcile_test.go +++ b/tests/pkg/tests/observability_reconcile_test.go @@ -201,9 +201,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_retention_test.go b/tests/pkg/tests/observability_retention_test.go index 66b207cf03..acd9ea220c 100644 --- a/tests/pkg/tests/observability_retention_test.go +++ b/tests/pkg/tests/observability_retention_test.go @@ -177,9 +177,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_route_test.go b/tests/pkg/tests/observability_route_test.go index 37c5099f57..9b01930e97 100644 --- a/tests/pkg/tests/observability_route_test.go +++ b/tests/pkg/tests/observability_route_test.go @@ -195,9 +195,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_uninstall_test.go b/tests/pkg/tests/observability_uninstall_test.go index 5dbb86f8f4..0dc0f89b4a 100644 --- a/tests/pkg/tests/observability_uninstall_test.go +++ b/tests/pkg/tests/observability_uninstall_test.go @@ -56,7 +56,7 @@ func uninstallMCO() { Namespace(MCO_ADDON_NAMESPACE). Get(context.TODO(), name, metav1.GetOptions{}) if instance != nil { - utils.PrintManagedClusterOBAObject(testOptions) + utils.PrintObject(context.Background(), clientDynamic, utils.NewMCOAddonGVR(), MCO_ADDON_NAMESPACE, "observability-addon") return errors.New("Failed to delete MCO addon instance") } return nil diff --git a/tests/pkg/utils/cluster_deploy.go b/tests/pkg/utils/cluster_deploy.go deleted file mode 100644 index c16ecf2a3e..0000000000 --- a/tests/pkg/utils/cluster_deploy.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Red Hat, Inc. -// Copyright Contributors to the Open Cluster Management project -// Licensed under the Apache License 2.0 - -package utils - -// ClusterDeploy defines the data passed to Hive. -type ClusterDeploy struct { - Kind string `yaml:"kind"` - APIVersion string `yaml:"apiVersion"` - Items []Items `yaml:"items"` -} - -// Items defines the list of items in the cluster deploy yaml. -type Items struct { - Kind string `yaml:"kind"` - Metadata Metadata `yaml:"metadata"` - StringData StringData `yaml:"stringData,omitempty"` - Spec Spec `yaml:"spec,omitempty"` -} - -// Metadata defines the name. -type Metadata struct { - Name string `yaml:"name,omitempty"` -} - -// StringData defiines the ssh values. -type StringData struct { - Dockerconfigjson string `yaml:".dockerconfigjson,omitempty"` - SSHPrivateKey string `yaml:"ssh-privatekey,omitempty"` -} - -// Spec defines the kube specifications. -type Spec struct { - BaseDomain string `yaml:"baseDomain,omitempty"` - ClusterName string `yaml:"clusterName,omitempty"` - Provisioning Provisioning `yaml:"provisioning,omitempty"` -} - -// Provisioning defines the data related to cluster creation. -type Provisioning struct { - ReleaseImage string `yaml:"releaseImage,omitempty"` - SSHKnownHosts []string `yaml:"sshKnownHosts,omitempty"` -} diff --git a/tests/pkg/utils/install_config.go b/tests/pkg/utils/install_config.go deleted file mode 100644 index 298fefde53..0000000000 --- a/tests/pkg/utils/install_config.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Red Hat, Inc. -// Copyright Contributors to the Open Cluster Management project -// Licensed under the Apache License 2.0 - -package utils - -// InstallConfig definition for install config structure from install-config.yaml. -type InstallConfig struct { - BaseDomain string `yaml:"baseDomain,omitempty"` - Networking Networking `yaml:"networking,omitempty"` - Metadata Metadata `yaml:"metadata"` - Platform Platform `yaml:"platform,omitempty"` - PullSecret string `yaml:"pullSecret,omitempty"` - SSHKey string `yaml:"sshKey,omitempty"` -} - -// Networking definition. -type Networking struct { - NetworkType string `yaml:"networkType"` - MachineCIDR string `yaml:"machineCIDR"` -} - -// Platform definition. -type Platform struct { - Baremetal Baremetal `yaml:"baremetal,omitempty"` -} - -// Baremetal specs for target baremetal provisioning. -type Baremetal struct { - ExternalBridge string `yaml:"externalBridge,omitempty"` - ProvisioningBridge string `yaml:"provisioningBridge,omitempty"` - LibvirtURI string `yaml:"libvirtURI,omitempty"` - ProvisioningNetworkInterface string `yaml:"provisioningNetworkInterface,omitempty"` - ProvisioningNetworkCIDR string `yaml:"provisioningNetworkCIDR,omitempty"` - APIVIP string `yaml:"apiVIP,omitempty"` - DNSVIP string `yaml:"dnsVIP,omitempty"` - IngressVIP string `yaml:"ingressVIP,omitempty"` - Hosts []Host `yaml:"hosts,omitempty"` - SSHKnownHosts string `yaml:"sshKnownHosts,omitempty"` -} - -// Host is an array of baremetal assets. -type Host struct { - Name string `yaml:"name"` - Role string `yaml:"role"` - Bmc Bmc `yaml:"bmc"` - BootMACAddress string `yaml:"bootMACAddress"` - HardwareProfile string `yaml:"hardwareProfile"` -} - -// Bmc definition. -type Bmc struct { - Address string `yaml:"address"` - Username string `yaml:"username"` - Password string `yaml:"password"` -} diff --git a/tests/pkg/utils/kube_debug.go b/tests/pkg/utils/kube_debug.go new file mode 100644 index 0000000000..14471825ce --- /dev/null +++ b/tests/pkg/utils/kube_debug.go @@ -0,0 +1,434 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package utils + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +// LogFailingTestStandardDebugInfo logs standard debug info for failing tests. +// It scans workloads and pods from hub and managed clusters observability namespaces. +// It also prints MCO and OBA objects. +// If a workload or pod is not running, it prints the resource spec, status, events and logs if appropriate. +func LogFailingTestStandardDebugInfo(opt TestOptions) { + klog.V(1).Infof("Test failed, printing debug info. TestOptions: %+v", opt) + + // Print MCO object + hubDynClient := NewKubeClientDynamic( + opt.HubCluster.ClusterServerURL, + opt.KubeConfig, + opt.HubCluster.KubeContext) + PrintObject(context.TODO(), hubDynClient, NewMCOGVRV1BETA2(), MCO_NAMESPACE, MCO_CR_NAME) + + // Check pods in hub + hubClient := NewKubeClient( + opt.HubCluster.ClusterServerURL, + opt.KubeConfig, + opt.HubCluster.KubeContext) + CheckPodsInNamespace(hubClient, "open-cluster-management", []string{"multicluster-observability-operator"}, map[string]string{ + "name": "multicluster-observability-operator", + }) + CheckDeploymentsInNamespace(hubClient, MCO_NAMESPACE) + CheckStatefulSetsInNamespace(hubClient, MCO_NAMESPACE) + CheckDaemonSetsInNamespace(hubClient, MCO_NAMESPACE) + CheckPodsInNamespace(hubClient, MCO_NAMESPACE, []string{}, map[string]string{}) + printConfigMapsInNamespace(hubClient, MCO_NAMESPACE) + printSecretsInNamespace(hubClient, MCO_NAMESPACE) + + for _, mc := range opt.ManagedClusters { + if mc.Name == "local-cluster" { + // Skip local-cluster as same namespace as hub, and already checked + continue + } + + spokeDynClient := NewKubeClientDynamic(mc.ClusterServerURL, opt.KubeConfig, mc.KubeContext) + PrintObject(context.TODO(), spokeDynClient, NewMCOAddonGVR(), MCO_ADDON_NAMESPACE, "observability-addon") + + spokeClient := NewKubeClient(mc.ClusterServerURL, mc.KubeConfig, mc.KubeContext) + CheckDeploymentsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + CheckStatefulSetsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + CheckDaemonSetsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + CheckPodsInNamespace(spokeClient, MCO_ADDON_NAMESPACE, []string{"observability-addon"}, map[string]string{}) + printConfigMapsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + printSecretsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + } +} + +// CheckPodsInNamespace lists pods in a namespace and logs debug info (status, events, logs) for pods not running. +func CheckPodsInNamespace(client kubernetes.Interface, ns string, forcePodNamesLog []string, podLabels map[string]string) { + listOptions := metav1.ListOptions{} + if len(podLabels) > 0 { + listOptions.LabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: podLabels}) + } + pods, err := client.CoreV1().Pods(ns).List(context.TODO(), listOptions) + if err != nil { + klog.Errorf("Failed to get pods in namespace %s: %v", ns, err) + return + } + + if len(pods.Items) == 0 { + klog.V(1).Infof("No pods in namespace %s", ns) + } + + klog.V(1).Infof("Checking %d pods in namespace %q", len(pods.Items), ns) + printPodsStatuses(pods.Items) + + notRunningPodsCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning { + notRunningPodsCount++ + } + + force := false + for _, forcePodName := range forcePodNamesLog { + if strings.Contains(pod.Name, forcePodName) { + force = true + break + } + } + if pod.Status.Phase == corev1.PodRunning && !force { + continue + } + + // print pod spec + podSpec, err := json.MarshalIndent(pod.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal pod %q spec: %s", pod.Name, err.Error()) + } + klog.V(1).Infof("Pod %q spec: \n%s", pod.Name, string(podSpec)) + + LogPodStatus(pod) + LogObjectEvents(client, ns, "Pod", pod.Name) + LogPodLogs(client, ns, pod) + } + + if notRunningPodsCount == 0 { + klog.V(1).Infof("All pods are running in namespace %q", ns) + } else { + klog.Errorf("Found %d pods not running in namespace %q", notRunningPodsCount, ns) + } +} + +func LogPodStatus(podList corev1.Pod) { + var podStatus strings.Builder + podStatus.WriteString(">>>>>>>>>> pod status >>>>>>>>>>\n") + podStatus.WriteString("Conditions:\n") + for _, condition := range podList.Status.Conditions { + podStatus.WriteString(fmt.Sprintf("\t%s: %s %v\n", condition.Type, condition.Status, condition.LastTransitionTime.Time)) + } + podStatus.WriteString("ContainerStatuses:\n") + for _, containerStatus := range podList.Status.ContainerStatuses { + podStatus.WriteString(fmt.Sprintf("\t%s: %t %d %v\n", containerStatus.Name, containerStatus.Ready, containerStatus.RestartCount, containerStatus.State)) + if containerStatus.LastTerminationState.Terminated != nil { + podStatus.WriteString(fmt.Sprintf("\t\tlastTerminated: %v\n", containerStatus.LastTerminationState.Terminated)) + } + } + podStatus.WriteString("<<<<<<<<<< pod status <<<<<<<<<<") + + klog.V(1).Infof("Pod %q is in phase %q and status: \n%s", podList.Name, podList.Status.Phase, podStatus.String()) +} + +func LogPodLogs(client kubernetes.Interface, ns string, pod corev1.Pod) { + for _, container := range pod.Spec.Containers { + logsRes := client.CoreV1().Pods(ns).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: container.Name, + }).Do(context.Background()) + + if logsRes.Error() != nil { + klog.Errorf("Failed to get logs for pod %q: %s", pod.Name, logsRes.Error()) + continue + } + + logs, err := logsRes.Raw() + if err != nil { + klog.Errorf("Failed to get logs for pod %q container %q: %s", pod.Name, container.Name, err.Error()) + continue + } + + // Filter error logs and keep all last 100 lines + maxLines := 100 + cleanedLines := []string{} + lines := strings.Split(string(logs), "\n") + for i, line := range lines { + if strings.Contains(strings.ToLower(line), "error") || i > len(lines)-maxLines { + cleanedLines = append(cleanedLines, line) + } + } + + logs = []byte(strings.Join(cleanedLines, "\n")) + + delimitedLogs := fmt.Sprintf(">>>>>>>>>> container logs >>>>>>>>>>\n%s<<<<<<<<<< container logs <<<<<<<<<<", string(logs)) + klog.V(1).Infof("Pod %q container %q logs (errors and last %d lines): \n%s", pod.Name, container.Name, maxLines, delimitedLogs) + } +} + +func CheckDeploymentsInNamespace(client kubernetes.Interface, ns string) { + deployments, err := client.AppsV1().Deployments(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get deployments in namespace %s: %v", ns, err) + return + } + + if len(deployments.Items) == 0 { + klog.V(1).Infof("No deployments found in namespace %q", ns) + } + + klog.V(1).Infof("Deployments in namespace %s: \n", ns) + printDeploymentsStatuses(client, ns) + + for _, deployment := range deployments.Items { + if deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas { + continue + } + + // print deployment spec + deploymentSpec, err := json.MarshalIndent(deployment.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal deployment %q spec: %s", deployment.Name, err.Error()) + } + klog.V(1).Infof("Deployment %q spec: \n%s", deployment.Name, string(deploymentSpec)) + + LogDeploymentStatus(deployment) + LogObjectEvents(client, ns, "Deployment", deployment.Name) + } +} + +func LogDeploymentStatus(deployment appsv1.Deployment) { + var deploymentStatus strings.Builder + deploymentStatus.WriteString(">>>>>>>>>> deployment status >>>>>>>>>>\n") + deploymentStatus.WriteString(fmt.Sprintf("ReadyReplicas: %d\n", deployment.Status.ReadyReplicas)) + deploymentStatus.WriteString(fmt.Sprintf("UpdatedReplicas: %d\n", deployment.Status.UpdatedReplicas)) + deploymentStatus.WriteString(fmt.Sprintf("AvailableReplicas: %d\n", deployment.Status.AvailableReplicas)) + deploymentStatus.WriteString("Conditions:\n") + for _, condition := range deployment.Status.Conditions { + deploymentStatus.WriteString(fmt.Sprintf("\t%s: %s %v \n\t\t%s %s\n", condition.Type, condition.Status, condition.LastTransitionTime, condition.Message, condition.Reason)) + } + deploymentStatus.WriteString("<<<<<<<<<< deployment status <<<<<<<<<<") + + klog.V(1).Infof("Deployment %q status: \n%s", deployment.Name, deploymentStatus.String()) +} + +func CheckStatefulSetsInNamespace(client kubernetes.Interface, ns string) { + statefulSets, err := client.AppsV1().StatefulSets(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get statefulsets in namespace %s: %v", ns, err) + return + } + + if len(statefulSets.Items) == 0 { + klog.V(1).Infof("No statefulsets found in namespace %q", ns) + return + } + + klog.V(1).Infof("StatefulSets in namespace %s: \n", ns) + printStatefulSetsStatuses(client, ns) + + for _, statefulSet := range statefulSets.Items { + if statefulSet.Status.UpdatedReplicas == *statefulSet.Spec.Replicas { + continue + } + + // Print statefulset spec + statefulSetSpec, err := json.MarshalIndent(statefulSet.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal statefulset %q spec: %s", statefulSet.Name, err.Error()) + } + klog.V(1).Infof("StatefulSet %q spec: \n%s", statefulSet.Name, string(statefulSetSpec)) + + LogObjectEvents(client, ns, "StatefulSet", statefulSet.Name) + } +} + +func CheckDaemonSetsInNamespace(client kubernetes.Interface, ns string) { + daemonSets, err := client.AppsV1().DaemonSets(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get daemonsets in namespace %s: %v", ns, err) + return + } + + if len(daemonSets.Items) == 0 { + klog.V(1).Infof("No daemonsets found in namespace %q", ns) + return + } + + klog.V(1).Infof("DaemonSets in namespace %s: \n", ns) + printDaemonSetsStatuses(client, ns) + + for _, daemonSet := range daemonSets.Items { + if daemonSet.Status.UpdatedNumberScheduled == daemonSet.Status.DesiredNumberScheduled { + continue + } + + // Print daemonset spec + daemonSetSpec, err := json.MarshalIndent(daemonSet.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal daemonset %q spec: %s", daemonSet.Name, err.Error()) + } + klog.V(1).Infof("DaemonSet %q spec: \n%s", daemonSet.Name, string(daemonSetSpec)) + + LogObjectEvents(client, ns, "DaemonSet", daemonSet.Name) + } +} + +func LogObjectEvents(client kubernetes.Interface, ns string, kind string, name string) { + fieldSelector := fmt.Sprintf("involvedObject.kind=%s,involvedObject.name=%s", kind, name) + events, err := client.CoreV1().Events(ns).List(context.TODO(), metav1.ListOptions{ + FieldSelector: fieldSelector, + }) + if err != nil { + klog.Errorf("Failed to get events for %s %s: %s", kind, name, err.Error()) + return + } + + objectEvents := make([]string, 0, len(events.Items)) + for _, event := range events.Items { + objectEvents = append(objectEvents, fmt.Sprintf("%s %s (%d): %s", event.Reason, event.LastTimestamp, event.Count, event.Message)) + } + formattedEvents := fmt.Sprintf(">>>>>>>>>> %s events >>>>>>>>>>\n%s\n<<<<<<<<<< %s events <<<<<<<<<<", kind, strings.Join(objectEvents, "\n"), kind) + klog.V(1).Infof("%s %q events: \n%s", kind, name, formattedEvents) +} + +func printPodsStatuses(pods []corev1.Pod) { + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tSTATUS\tRESTARTS\tAGE") + for _, pod := range pods { + var restartCount int32 + if len(pod.Status.ContainerStatuses) > 0 { + restartCount = pod.Status.ContainerStatuses[0].RestartCount + } + age := time.Since(pod.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%d\t%s\n", + pod.Name, + pod.Status.Phase, + restartCount, + age) + } + writer.Flush() +} + +func printDeploymentsStatuses(clientset kubernetes.Interface, namespace string) { + deploymentsClient := clientset.AppsV1().Deployments(namespace) + deployments, err := deploymentsClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + panic(err.Error()) + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tREADY\tUP-TO-DATE\tAVAILABLE\tAGE") + for _, deployment := range deployments.Items { + ready := fmt.Sprintf("%d/%d", deployment.Status.ReadyReplicas, *deployment.Spec.Replicas) + age := time.Since(deployment.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%d\t%d\t%s\n", + deployment.Name, + ready, + deployment.Status.UpdatedReplicas, + deployment.Status.AvailableReplicas, + age) + } + writer.Flush() +} + +func printStatefulSetsStatuses(clientset kubernetes.Interface, namespace string) { + statefulSetsClient := clientset.AppsV1().StatefulSets(namespace) + statefulSets, err := statefulSetsClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + panic(err.Error()) + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tREADY\tAGE") + for _, statefulSet := range statefulSets.Items { + ready := fmt.Sprintf("%d/%d", statefulSet.Status.ReadyReplicas, *statefulSet.Spec.Replicas) + age := time.Since(statefulSet.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%s\n", + statefulSet.Name, + ready, + age) + } + writer.Flush() +} + +func printDaemonSetsStatuses(clientset kubernetes.Interface, namespace string) { + daemonSetsClient := clientset.AppsV1().DaemonSets(namespace) + daemonSets, err := daemonSetsClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + panic(err.Error()) + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tDESIRED\tCURRENT\tREADY\tAGE") + for _, daemonSet := range daemonSets.Items { + age := time.Since(daemonSet.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%d\t%d\t%d\t%s\n", + daemonSet.Name, + daemonSet.Status.DesiredNumberScheduled, + daemonSet.Status.CurrentNumberScheduled, + daemonSet.Status.NumberReady, + age) + } + writer.Flush() +} + +func printConfigMapsInNamespace(client kubernetes.Interface, ns string) { + configMaps, err := client.CoreV1().ConfigMaps(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get configmaps in namespace %q: %v", ns, err) + return + } + + if len(configMaps.Items) == 0 { + klog.V(1).Infof("No configmaps found in namespace %q", ns) + return + } + + klog.V(1).Infof("ConfigMaps in namespace %s: \n", ns) + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tDATA\tAGE") + for _, configMap := range configMaps.Items { + age := time.Since(configMap.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%d\t%s\n", + configMap.Name, + len(configMap.Data), + age) + } + writer.Flush() +} + +func printSecretsInNamespace(client kubernetes.Interface, ns string) { + secrets, err := client.CoreV1().Secrets(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get secrets in namespace %q: %v", ns, err) + return + } + + if len(secrets.Items) == 0 { + klog.V(1).Infof("No secrets found in namespace %q", ns) + return + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tTYPE\tDATA\tAGE") + for _, secret := range secrets.Items { + age := time.Since(secret.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%d\t%s\n", + secret.Name, + secret.Type, + len(secret.Data), + age) + } + writer.Flush() +} diff --git a/tests/pkg/utils/mco_configmaps.go b/tests/pkg/utils/mco_configmaps.go index cb47c2b9e1..5ec3768483 100644 --- a/tests/pkg/utils/mco_configmaps.go +++ b/tests/pkg/utils/mco_configmaps.go @@ -8,36 +8,10 @@ import ( "context" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" ) -func CreateConfigMap(opt TestOptions, isHub bool, cm *corev1.ConfigMap) error { - clientKube := getKubeClient(opt, isHub) - found, err := clientKube.CoreV1(). - ConfigMaps(cm.ObjectMeta.Namespace). - Get(context.TODO(), cm.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - _, err := clientKube.CoreV1(). - ConfigMaps(cm.ObjectMeta.Namespace). - Create(context.TODO(), cm, metav1.CreateOptions{}) - if err == nil { - klog.V(1).Infof("configmap %s created", cm.ObjectMeta.Name) - } - return err - } - if err != nil { - return err - } - cm.ObjectMeta.ResourceVersion = found.ObjectMeta.ResourceVersion - _, err = clientKube.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) - if err == nil { - klog.V(1).Infof("configmap %s updated", cm.ObjectMeta.Name) - } - return err -} - func GetConfigMap(opt TestOptions, isHub bool, name string, namespace string) (error, *corev1.ConfigMap) { clientKube := getKubeClient(opt, isHub) diff --git a/tests/pkg/utils/mco_deploy.go b/tests/pkg/utils/mco_deploy.go index fdc4719cbb..0b3ddd26f5 100644 --- a/tests/pkg/utils/mco_deploy.go +++ b/tests/pkg/utils/mco_deploy.go @@ -20,7 +20,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" - "k8s.io/client-go/kubernetes" + "k8s.io/client-go/dynamic" "k8s.io/klog" ) @@ -101,26 +101,6 @@ func NewOCMMultiClusterHubGVR() schema.GroupVersionResource { Resource: "multiclusterhubs"} } -func ModifyMCOAvailabilityConfig(opt TestOptions, availabilityConfig string) error { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - return getErr - } - - spec := mco.Object["spec"].(map[string]interface{}) - spec["availabilityConfig"] = availabilityConfig - _, updateErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Update(context.TODO(), mco, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr - } - return nil -} - func GetAllMCOPods(opt TestOptions) ([]corev1.Pod, error) { hubClient := NewKubeClient( opt.HubCluster.ClusterServerURL, @@ -149,183 +129,32 @@ func GetAllMCOPods(opt TestOptions) ([]corev1.Pod, error) { return mcoPods, nil } -func PrintAllMCOPodsStatus(opt TestOptions) { - podList, err := GetAllMCOPods(opt) - if err != nil { - klog.Errorf("Failed to get all MCO pods") - } - - if len(podList) == 0 { - klog.V(1).Infof("Failed to get pod in %q namespace", MCO_NAMESPACE) - } - - hubClient := NewKubeClient( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - - // Print mch-image-manifest configmap - mchImageManifestCM, err := ReadImageManifestConfigMap(hubClient) - if err != nil { - klog.Errorf("Failed to get mch-image-manifest configmap: %s", err.Error()) - } else { - klog.V(1).Infof("mch-image-manifest configmap: %v", mchImageManifestCM) - } - - LogPodsDebugInfo(hubClient, podList, false) -} - -func LogPodsDebugInfo(hubClient kubernetes.Interface, pods []corev1.Pod, force bool) { - if len(pods) == 0 { +func PrintObject(ctx context.Context, client dynamic.Interface, gvr schema.GroupVersionResource, ns, name string) { + if ns == "" || name == "" { + klog.V(1).Info("Namespace or name cannot be empty") return } - ns := pods[0].Namespace - podsNames := make([]string, 0, len(pods)) - for _, pod := range pods { - podsNames = append(podsNames, pod.Name) - } - - klog.V(1).Infof("Checking pods %v in namespace %q", podsNames, ns) - notRunningPodsCount := 0 - for _, pod := range pods { - if pod.Status.Phase != corev1.PodRunning { - notRunningPodsCount++ - } - - if pod.Status.Phase == corev1.PodRunning && !force { - continue - } - - klog.V(1).Infof("Pod %q is in phase %q and status: %s\n", - pod.Name, - pod.Status.Phase, - pod.Status.String()) - - // print pod events - events, err := hubClient.CoreV1().Events(ns).List(context.TODO(), metav1.ListOptions{ - FieldSelector: "involvedObject.name=" + pod.Name, - }) - if err != nil { - klog.Errorf("Failed to get events for pod %s: %s", pod.Name, err.Error()) - } - - podEvents := make([]string, 0, len(events.Items)) - for _, event := range events.Items { - podEvents = append(podEvents, fmt.Sprintf("%s %s (%d): %s", event.Reason, event.LastTimestamp, event.Count, event.Message)) - } - formattedEvents := ">>>>>>>>>> pod events >>>>>>>>>>\n" + strings.Join(podEvents, "\n") + "\n<<<<<<<<<< pod events <<<<<<<<<<" - klog.V(1).Infof("Pod %q events: \n%s", pod.Name, formattedEvents) - - // print pod containers logs - for _, container := range pod.Spec.Containers { - logsRes := hubClient.CoreV1().Pods(ns).GetLogs(pod.Name, &corev1.PodLogOptions{ - Container: container.Name, - }).Do(context.Background()) - - if logsRes.Error() != nil { - klog.Errorf("Failed to get logs for pod %q: %s", pod.Name, logsRes.Error()) - continue - } - - logs, err := logsRes.Raw() - if err != nil { - klog.Errorf("Failed to get logs for pod %q container %q: %s", pod.Name, container.Name, err.Error()) - continue - } - - delimitedLogs := fmt.Sprintf(">>>>>>>>>> container logs >>>>>>>>>>\n%s<<<<<<<<<< container logs <<<<<<<<<<", string(logs)) - klog.V(1).Infof("Pod %q container %q logs: \n%s", pod.Name, container.Name, delimitedLogs) - } - } - - if notRunningPodsCount == 0 { - klog.V(1).Infof("All pods are running in namespace %q", ns) - } -} - -// ReadImageManifestConfigMap reads configmap with the label ocm-configmap-type=image-manifest. -func ReadImageManifestConfigMap(c kubernetes.Interface) (map[string]string, error) { - listOpts := metav1.ListOptions{ - LabelSelector: "ocm-configmap-type=image-manifest", - } - - imageCMList, err := c.CoreV1().ConfigMaps("open-cluster-management").List(context.TODO(), listOpts) + obj, err := client.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to list mch-image-manifest configmaps: %w", err) - } - - if len(imageCMList.Items) != 1 { - return nil, fmt.Errorf("found %d mch-image-manifest configmaps, expected 1", len(imageCMList.Items)) - } - - return imageCMList.Items[0].Data, nil -} - -func PrintMCOObject(opt TestOptions) { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - klog.V(1).Infof("Failed to get mco object") + klog.V(1).Infof("Failed to get object %s in namespace %s: %v", name, ns, err) return } - spec, _ := json.MarshalIndent(mco.Object["spec"], "", " ") - status, _ := json.MarshalIndent(mco.Object["status"], "", " ") - klog.V(1).Infof("MCO spec: %+v\n", string(spec)) - klog.V(1).Infof("MCO status: %+v\n", string(status)) -} - -func PrintManagedClusterOBAObject(opt TestOptions) { - clientDynamic := GetKubeClientDynamic(opt, false) - oba, getErr := clientDynamic.Resource(NewMCOAddonGVR()). - Namespace(MCO_ADDON_NAMESPACE). - Get(context.TODO(), "observability-addon", metav1.GetOptions{}) - if getErr != nil { - klog.V(1).Infof("Failed to get oba object from managedcluster") - return - } - - spec, _ := json.MarshalIndent(oba.Object["spec"], "", " ") - status, _ := json.MarshalIndent(oba.Object["status"], "", " ") - klog.V(1).Infof("OBA spec: %+v\n", string(spec)) - klog.V(1).Infof("OBA status: %+v\n", string(status)) -} - -func GetAllOBAPods(opt TestOptions) ([]corev1.Pod, error) { - clientKube := getKubeClient(opt, false) - obaPods, err := clientKube.CoreV1().Pods(MCO_ADDON_NAMESPACE).List(context.TODO(), metav1.ListOptions{}) + spec, err := json.MarshalIndent(obj.Object["spec"], "", " ") if err != nil { - return []corev1.Pod{}, err - } - - return obaPods.Items, nil -} - -func PrintAllOBAPodsStatus(opt TestOptions) { - if GetManagedClusterName(opt) == "local-cluster" { - klog.V(1).Infof("Skip printing OBA pods status for local-cluster") - return - } - podList, err := GetAllOBAPods(opt) - if err != nil { - klog.Errorf("Failed to get all OBA pods: %v", err) + klog.V(1).Infof("Failed to marshal spec for object %s in namespace %s: %v", name, ns, err) return } - klog.V(1).Infof("Get <%v> pods in <%s> namespace from managedcluster", len(podList), MCO_ADDON_NAMESPACE) - if len(podList) == 0 { + status, err := json.MarshalIndent(obj.Object["status"], "", " ") + if err != nil { + klog.V(1).Infof("Failed to marshal status for object %s in namespace %s: %v", name, ns, err) return } - force := false - if len(podList) == 1 { // only the operator is up - force = true - } - LogPodsDebugInfo(getKubeClient(opt, false), podList, force) + klog.V(1).Infof("Object %s/%s/%s spec: %+v\n", ns, gvr.Resource, name, string(spec)) + klog.V(1).Infof("Object %s/%s/%s status: %+v\n", ns, gvr.Resource, name, string(status)) } func CheckAllPodNodeSelector(opt TestOptions, nodeSelector map[string]interface{}) error { @@ -617,43 +446,6 @@ func RevertMCOCRModification(opt TestOptions) error { return nil } -func CheckMCOAddon(opt TestOptions) error { - client := NewKubeClient( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - if len(opt.ManagedClusters) > 0 { - client = NewKubeClient( - opt.ManagedClusters[0].ClusterServerURL, - opt.ManagedClusters[0].KubeConfig, - "") - } - expectedPodNames := []string{ - "endpoint-observability-operator", - "metrics-collector-deployment", - } - podList, err := client.CoreV1().Pods(MCO_ADDON_NAMESPACE).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - podsn := make(map[string]corev1.PodPhase) - for _, pod := range podList.Items { - podsn[pod.Name] = pod.Status.Phase - } - for _, podName := range expectedPodNames { - exist := false - for key, value := range podsn { - if strings.HasPrefix(key, podName) && value == "Running" { - exist = true - } - } - if !exist { - return errors.New(podName + " not found") - } - } - return nil -} - func CheckMCOAddonResources(opt TestOptions) error { client := NewKubeClient( opt.HubCluster.ClusterServerURL, @@ -696,43 +488,6 @@ func CheckMCOAddonResources(opt TestOptions) error { return nil } -func ModifyMCORetentionResolutionRaw(opt TestOptions) error { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - return getErr - } - - spec := mco.Object["spec"].(map[string]interface{}) - advRetentionCon, _ := CheckAdvRetentionConfig(opt) - if advRetentionCon { - retentionConfig := spec["advanced"].(map[string]interface{})["retentionConfig"].(map[string]interface{}) - retentionConfig["retentionResolutionRaw"] = "3d" - } - _, updateErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Update(context.TODO(), mco, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr - } - return nil -} - -func GetMCOAddonSpecMetrics(opt TestOptions) (bool, error) { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - return false, getErr - } - - enable := mco.Object["spec"].(map[string]interface{})["observabilityAddonSpec"].(map[string]interface{})["enableMetrics"].(bool) - return enable, nil -} - func ModifyMCOAddonSpecMetrics(opt TestOptions, enable bool) error { clientDynamic := NewKubeClientDynamic( opt.HubCluster.ClusterServerURL, diff --git a/tests/pkg/utils/mco_metric.go b/tests/pkg/utils/mco_metric.go index 16318114c8..ac903ca767 100644 --- a/tests/pkg/utils/mco_metric.go +++ b/tests/pkg/utils/mco_metric.go @@ -8,7 +8,6 @@ import ( "bufio" "context" "crypto/tls" - "errors" "fmt" "io" "net/http" @@ -22,17 +21,54 @@ import ( "k8s.io/klog" ) -func ContainManagedClusterMetric(opt TestOptions, query string, matchedLabels []string) (error, bool) { +type GrafanaResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` // Use interface{} because value can be mixed types + } `json:"result"` + } `json:"data"` +} + +func (r GrafanaResponse) ContainsLabelsSet(labels map[string]string) bool { + ret := false +loop: + for _, result := range r.Data.Result { + for key, val := range labels { + if result.Metric[key] != val { + continue loop + } + } + ret = true + break + } + + return ret +} + +func (r GrafanaResponse) String() string { + var ret strings.Builder + ret.WriteString(fmt.Sprintf("Status: %s\n", r.Status)) + ret.WriteString(fmt.Sprintf("ResultType: %s\n", r.Data.ResultType)) + ret.WriteString("Result:\n") + for _, result := range r.Data.Result { + ret.WriteString(fmt.Sprintf("%v %v\n", result.Metric, result.Value)) + } + return ret.String() +} + +func QueryGrafana(opt TestOptions, query string) (*GrafanaResponse, error) { grafanaConsoleURL := GetGrafanaURL(opt) path := "/api/datasources/proxy/1/api/v1/query?" queryParams := url.PathEscape(fmt.Sprintf("query=%s", query)) - klog.V(5).Infof("request url is: %s\n", grafanaConsoleURL+path+queryParams) req, err := http.NewRequest( "GET", grafanaConsoleURL+path+queryParams, nil) if err != nil { - return err, false + return nil, err } client := &http.Client{} @@ -45,7 +81,7 @@ func ContainManagedClusterMetric(opt TestOptions, query string, matchedLabels [] client = &http.Client{Transport: tr} token, err := FetchBearerToken(opt) if err != nil { - return err, false + return nil, err } if token != "" { req.Header.Set("Authorization", "Bearer "+token) @@ -55,41 +91,29 @@ func ContainManagedClusterMetric(opt TestOptions, query string, matchedLabels [] resp, err := client.Do(req) if err != nil { - return err, false + return nil, err } if resp.StatusCode != http.StatusOK { - klog.Errorf("resp: %+v\n", resp) - klog.Errorf("err: %+v\n", err) - return fmt.Errorf("failed to access managed cluster metrics via grafana console: %s", query), false + return nil, fmt.Errorf("failed to access managed cluster metrics via grafana console, status code: %d", resp.StatusCode) } - metricResult, err := io.ReadAll(resp.Body) - klog.V(5).Infof("metricResult: %s\n", metricResult) + respBody, err := io.ReadAll(resp.Body) if err != nil { - return err, false - } - - if !strings.Contains(string(metricResult), `"status":"success"`) { - return errors.New("failed to find valid status from response"), false + return nil, fmt.Errorf("failed to read response body: %v", err) } - if strings.Contains(string(metricResult), `"result":[]`) { - return errors.New("failed to find metric name from response"), false + metricResult := GrafanaResponse{} + err = yaml.Unmarshal(respBody, &metricResult) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response body: %v", err) } - contained := true - for _, label := range matchedLabels { - if !strings.Contains(string(metricResult), label) { - contained = false - break - } - } - if !contained { - return errors.New("failed to find metric name from response"), false + if metricResult.Status != "success" { + return &metricResult, fmt.Errorf("failed to get metric from response, status: %s", metricResult.Status) } - return nil, true + return &metricResult, nil } type MetricsAllowlist struct { diff --git a/tests/pkg/utils/mco_pods.go b/tests/pkg/utils/mco_pods.go index d3a761324a..91b6d721c5 100644 --- a/tests/pkg/utils/mco_pods.go +++ b/tests/pkg/utils/mco_pods.go @@ -37,16 +37,6 @@ func GetPodList(opt TestOptions, isHub bool, namespace string, labelSelector str return nil, podList } -func DeletePod(opt TestOptions, isHub bool, namespace, name string) error { - clientKube := getKubeClient(opt, isHub) - err := clientKube.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) - if err != nil { - klog.Errorf("Failed to delete pod %s in namespace %s due to %v", name, namespace, err) - return err - } - return nil -} - func GetPodLogs( opt TestOptions, isHub bool, diff --git a/tests/pkg/utils/utils.go b/tests/pkg/utils/utils.go index 74748266e7..0ad676d58a 100644 --- a/tests/pkg/utils/utils.go +++ b/tests/pkg/utils/utils.go @@ -6,7 +6,6 @@ package utils import ( "context" - "encoding/json" "errors" "fmt" "os" @@ -25,9 +24,7 @@ import ( k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -36,26 +33,6 @@ import ( "k8s.io/klog" ) -func NewUnversionedRestClient(url, kubeconfig, ctx string) *rest.RESTClient { - klog.V(5).Infof("Create unversionedRestClient for url %s using kubeconfig path %s\n", url, kubeconfig) - config, err := LoadConfig(url, kubeconfig, ctx) - if err != nil { - panic(err) - } - - oldNegotiatedSerializer := config.NegotiatedSerializer - config.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer() - kubeRESTClient, err := rest.UnversionedRESTClientFor(config) - // restore cfg before leaving - defer func(cfg *rest.Config) { cfg.NegotiatedSerializer = oldNegotiatedSerializer }(config) - - if err != nil { - panic(err) - } - - return kubeRESTClient -} - func NewKubeClient(url, kubeconfig, ctx string) kubernetes.Interface { config, err := LoadConfig(url, kubeconfig, ctx) if err != nil { @@ -99,21 +76,6 @@ func NewKubeClientAPIExtension(url, kubeconfig, ctx string) apiextensionsclients return clientset } -// func NewKubeClientDiscovery(url, kubeconfig, ctx string) *discovery.DiscoveryClient { -// klog.V(5).Infof("Create kubeclient discovery for url %s using kubeconfig path %s\n", url, kubeconfig) -// config, err := LoadConfig(url, kubeconfig, ctx) -// if err != nil { -// panic(err) -// } - -// clientset, err := discovery.NewDiscoveryClientForConfig(config) -// if err != nil { -// panic(err) -// } - -// return clientset -// } - func CreateMCOTestingRBAC(opt TestOptions) error { // create new service account and new clusterrolebinding and bind the serviceaccount to cluster-admin clusterrole // then the bearer token can be retrieved from the secret of created serviceaccount @@ -298,13 +260,14 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { apiVersion = v.(string) } + klog.V(5).Infof("Applying kind %q with name %q in namespace %q", kind, obj.GetName(), obj.GetNamespace()) + clientKube := NewKubeClient(url, kubeconfig, ctx) clientAPIExtension := NewKubeClientAPIExtension(url, kubeconfig, ctx) // now use switch over the type of the object // and match each type-case switch kind { case "CustomResourceDefinition": - klog.V(5).Infof("Install CRD: %s\n", f) obj := &apiextensionsv1.CustomResourceDefinition{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -323,7 +286,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientAPIExtension.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), existingObject, metav1.UpdateOptions{}) } case "Namespace": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.Namespace{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -340,7 +302,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().Namespaces().Update(context.TODO(), existingObject, metav1.UpdateOptions{}) } case "ServiceAccount": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.ServiceAccount{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -359,7 +320,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().ServiceAccounts(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "ClusterRoleBinding": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &rbacv1.ClusterRoleBinding{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -376,7 +336,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.RbacV1().ClusterRoleBindings().Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "Secret": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.Secret{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -393,7 +352,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().Secrets(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "ConfigMap": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.ConfigMap{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -412,7 +370,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().ConfigMaps(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "Service": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.Service{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -432,7 +389,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().Services(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "PersistentVolumeClaim": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.PersistentVolumeClaim{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -452,7 +408,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().PersistentVolumeClaims(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "Deployment": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &appsv1.Deployment{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -471,7 +426,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.AppsV1().Deployments(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "LimitRange": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.LimitRange{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -490,7 +444,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().LimitRanges(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "ResourceQuota": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.ResourceQuota{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -509,7 +462,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().ResourceQuotas(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "StorageClass": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &storagev1.StorageClass{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -533,13 +485,11 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { if apiVersion == "observability.open-cluster-management.io/v1beta1" { gvr = NewMCOGVRV1BETA1() } - klog.V(5).Infof("Install MultiClusterObservability: %s\n", f) case "PrometheusRule": gvr = schema.GroupVersionResource{ Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"} - klog.V(5).Infof("Install PrometheusRule: %s\n", f) default: return fmt.Errorf("resource %s not supported", kind) } @@ -612,45 +562,6 @@ func StatusContainsTypeEqualTo(u *unstructured.Unstructured, typeString string) return false } -// GetCluster returns the first cluster with a given tag -func GetCluster(tag string, clusters []Cluster) *Cluster { - for _, cluster := range clusters { - if tag, ok := cluster.Tags[tag]; ok { - if tag { - return &cluster - } - } - } - return nil -} - -// GetClusters returns all clusters with a given tag -func GetClusters(tag string, clusters []Cluster) []*Cluster { - filteredClusters := make([]*Cluster, 0) - for i, cluster := range clusters { - if tag, ok := cluster.Tags[tag]; ok { - if tag { - filteredClusters = append(filteredClusters, &clusters[i]) - } - } - } - return filteredClusters -} - -func HaveServerResources(c Cluster, kubeconfig string, expectedAPIGroups []string) error { - clientAPIExtension := NewKubeClientAPIExtension(c.ClusterServerURL, kubeconfig, c.KubeContext) - clientDiscovery := clientAPIExtension.Discovery() - for _, apiGroup := range expectedAPIGroups { - klog.V(1).Infof("Check if %s exists", apiGroup) - _, err := clientDiscovery.ServerResourcesForGroupVersion(apiGroup) - if err != nil { - klog.V(1).Infof("Error while retrieving server resource %s: %s", apiGroup, err.Error()) - return err - } - } - return nil -} - func HaveCRDs(c Cluster, kubeconfig string, expectedCRDs []string) error { clientAPIExtension := NewKubeClientAPIExtension(c.ClusterServerURL, kubeconfig, c.KubeContext) clientAPIExtensionV1 := clientAPIExtension.ApiextensionsV1() @@ -665,93 +576,6 @@ func HaveCRDs(c Cluster, kubeconfig string, expectedCRDs []string) error { return nil } -func HaveDeploymentsInNamespace( - c Cluster, - kubeconfig string, - namespace string, - expectedDeploymentNames []string, -) error { - - client := NewKubeClient(c.ClusterServerURL, kubeconfig, c.KubeContext) - versionInfo, err := client.Discovery().ServerVersion() - if err != nil { - return err - } - klog.V(1).Infof("Server version info: %v", versionInfo) - - deployments := client.AppsV1().Deployments(namespace) - - for _, deploymentName := range expectedDeploymentNames { - klog.V(1).Infof("Check if deployment %s exists", deploymentName) - deployment, err := deployments.Get(context.TODO(), deploymentName, metav1.GetOptions{}) - if err != nil { - klog.V(1).Infof("Error while retrieving deployment %s: %s", deploymentName, err.Error()) - return err - } - - if deployment.Status.Replicas != deployment.Status.ReadyReplicas { - err = fmt.Errorf("%s: Expect %d but got %d Ready replicas", - deploymentName, - deployment.Status.Replicas, - deployment.Status.ReadyReplicas) - klog.Errorln(err) - return err - } - - for _, condition := range deployment.Status.Conditions { - if condition.Reason == "MinimumReplicasAvailable" { - if condition.Status != corev1.ConditionTrue { - err = fmt.Errorf("%s: Expect %s but got %s", - deploymentName, - condition.Status, - corev1.ConditionTrue) - klog.Errorln(err) - return err - } - } - } - } - - return nil -} - -func GetKubeVersion(client *rest.RESTClient) version.Info { - kubeVersion := version.Info{} - - versionBody, err := client.Get().AbsPath("/version").Do(context.TODO()).Raw() - if err != nil { - klog.Errorf("fail to GET /version with %v", err) - return version.Info{} - } - - err = json.Unmarshal(versionBody, &kubeVersion) - if err != nil { - klog.Errorf("fail to Unmarshal, got '%s': %v", string(versionBody), err) - return version.Info{} - } - - return kubeVersion -} - -func IsOpenshift(client *rest.RESTClient) bool { - //check whether the cluster is openshift or not for openshift version 3.11 and before - _, err := client.Get().AbsPath("/version/openshift").Do(context.TODO()).Raw() - if err == nil { - klog.V(5).Info("Found openshift version from /version/openshift") - return true - } - - //check whether the cluster is openshift or not for openshift version 4.1 - _, err = client.Get().AbsPath("/apis/config.openshift.io/v1/clusterversions").Do(context.TODO()).Raw() - if err == nil { - klog.V(5).Info("Found openshift version from /apis/config.openshift.io/v1/clusterversions") - return true - } - - klog.V(5).Infof("fail to GET openshift version, assuming not OpenShift: %s", err.Error()) - return false -} - // IntegrityChecking checks to ensure all required conditions are met when completing the specs func IntegrityChecking(opt TestOptions) error { var err error