diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml new file mode 100644 index 000000000..a907d312b --- /dev/null +++ b/.github/workflows/integration_tests.yaml @@ -0,0 +1,35 @@ +name: integration + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + name: Integration tests + env: + GOBIN: /tmp/.bin + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.21.x + cache-dependency-path: "**/*.sum" + + - name: Install deps + run: make deps + - name: Build + run: make build + + - name: Run tests + run: | + source <(BIN_DIR=$GOBIN make install-envtest-deps) + make integration-test-operators diff --git a/Makefile b/Makefile index b965c65f6..d2ade19a8 100644 --- a/Makefile +++ b/Makefile @@ -205,3 +205,8 @@ install-integration-test-deps: install-e2e-test-deps: @mkdir -p $(BIN_DIR) @./scripts/install-binaries.sh install_e2e_tests_deps $(BIN_DIR) + +.PHONY: install-envtest-deps +install-envtest-deps: + @mkdir -p $(BIN_DIR) + @./scripts/install-binaries.sh install_envtest_deps $(BIN_DIR) \ No newline at end of file diff --git a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go index 81753ed38..cbade412b 100644 --- a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go +++ b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go @@ -8,6 +8,7 @@ package observabilityendpoint import ( "context" + "os" "path/filepath" "testing" "time" @@ -17,11 +18,15 @@ import ( "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/hypershift" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/util" oav1beta1 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta1" + mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" kubescheme "k8s.io/client-go/kubernetes/scheme" @@ -64,7 +69,8 @@ func TestIntegrationReconcileHypershift(t *testing.T) { } mgr, err := ctrl.NewManager(testEnv.Config, ctrl.Options{ - Scheme: k8sClient.Scheme(), + Scheme: k8sClient.Scheme(), + MetricsBindAddress: "0", // Avoids port conflict with the default port 8080 }) assert.NoError(t, err) @@ -100,8 +106,15 @@ func TestIntegrationReconcileHypershift(t *testing.T) { // setupTestEnv starts the test environment (etcd and kube api-server). func setupTestEnv(t *testing.T) (*envtest.Environment, client.Client) { + rootPath := filepath.Join("..", "..", "..") + crds := readCRDFiles(t, + filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_multiclusterobservabilities.yaml"), + filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "servicemonitor_crd_0_53_1.yaml"), + filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "prometheusrule_crd_0_53_1.yaml"), + ) testEnv := &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("testdata", "crd"), filepath.Join("..", "..", "config", "crd", "bases")}, + CRDs: crds, } cfg, err := testEnv.Start() @@ -114,6 +127,7 @@ func setupTestEnv(t *testing.T) (*envtest.Environment, client.Client) { hyperv1.AddToScheme(scheme) promv1.AddToScheme(scheme) oav1beta1.AddToScheme(scheme) + mcov1beta2.AddToScheme(scheme) k8sClient, err := client.New(cfg, client.Options{Scheme: scheme}) if err != nil { @@ -128,6 +142,28 @@ func setupTestEnv(t *testing.T) (*envtest.Environment, client.Client) { return testEnv, k8sClient } +func readCRDFiles(t *testing.T, crdPaths ...string) []*apiextensionsv1.CustomResourceDefinition { + ret := []*apiextensionsv1.CustomResourceDefinition{} + + for _, crdPath := range crdPaths { + crdYamlData, err := os.ReadFile(crdPath) + if err != nil { + t.Fatalf("Failed to read CRD file: %v", err) + } + + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + var crd apiextensionsv1.CustomResourceDefinition + _, _, err = dec.Decode(crdYamlData, nil, &crd) + if err != nil { + t.Fatalf("Failed to decode CRD: %v", err) + } + + ret = append(ret, &crd) + } + + return ret +} + func makeNamespace(name string) *corev1.Namespace { return &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ diff --git a/operators/endpointmetrics/controllers/observabilityendpoint/testdata/crd/prometheusrule-crd.yaml b/operators/endpointmetrics/controllers/observabilityendpoint/testdata/crd/prometheusrule-crd.yaml deleted file mode 100644 index 1dd024388..000000000 --- a/operators/endpointmetrics/controllers/observabilityendpoint/testdata/crd/prometheusrule-crd.yaml +++ /dev/null @@ -1,100 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.6.2 - creationTimestamp: null - name: prometheusrules.monitoring.coreos.com -spec: - group: monitoring.coreos.com - names: - categories: - - prometheus-operator - kind: PrometheusRule - listKind: PrometheusRuleList - plural: prometheusrules - singular: prometheusrule - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: PrometheusRule defines recording and alerting rules for a Prometheus - instance - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Specification of desired alerting rule definitions for Prometheus. - properties: - groups: - description: Content of Prometheus rule file - items: - description: 'RuleGroup is a list of sequentially evaluated recording - and alerting rules. Note: PartialResponseStrategy is only used - by ThanosRuler and will be ignored by Prometheus instances. Valid - values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response' - properties: - interval: - type: string - name: - type: string - partial_response_strategy: - type: string - rules: - items: - description: 'Rule describes an alerting or recording rule - See Prometheus documentation: [alerting](https://www.prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) - or [recording](https://www.prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) - rule' - properties: - alert: - type: string - annotations: - additionalProperties: - type: string - type: object - expr: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - for: - type: string - labels: - additionalProperties: - type: string - type: object - record: - type: string - required: - - expr - type: object - type: array - required: - - name - - rules - type: object - type: array - type: object - required: - - spec - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/operators/endpointmetrics/controllers/observabilityendpoint/testdata/crd/servicemonitor-crd.json b/operators/endpointmetrics/controllers/observabilityendpoint/testdata/crd/servicemonitor-crd.json deleted file mode 100644 index f8683e129..000000000 --- a/operators/endpointmetrics/controllers/observabilityendpoint/testdata/crd/servicemonitor-crd.json +++ /dev/null @@ -1,780 +0,0 @@ -{ - "apiVersion": "apiextensions.k8s.io/v1", - "kind": "CustomResourceDefinition", - "metadata": { - "annotations": { - "controller-gen.kubebuilder.io/version": "v0.13.0", - "operator.prometheus.io/version": "0.71.2" - }, - "name": "servicemonitors.monitoring.coreos.com" - }, - "spec": { - "group": "monitoring.coreos.com", - "names": { - "categories": [ - "prometheus-operator" - ], - "kind": "ServiceMonitor", - "listKind": "ServiceMonitorList", - "plural": "servicemonitors", - "shortNames": [ - "smon" - ], - "singular": "servicemonitor" - }, - "scope": "Namespaced", - "versions": [ - { - "name": "v1", - "schema": { - "openAPIV3Schema": { - "description": "ServiceMonitor defines monitoring for a set of services.", - "properties": { - "apiVersion": { - "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", - "type": "string" - }, - "kind": { - "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", - "type": "string" - }, - "metadata": { - "type": "object" - }, - "spec": { - "description": "Specification of desired Service selection for target discovery by Prometheus.", - "properties": { - "attachMetadata": { - "description": "`attachMetadata` defines additional metadata which is added to the discovered targets. \n It requires Prometheus >= v2.37.0.", - "properties": { - "node": { - "description": "When set to true, Prometheus must have the `get` permission on the `Nodes` objects.", - "type": "boolean" - } - }, - "type": "object" - }, - "endpoints": { - "description": "List of endpoints part of this ServiceMonitor.", - "items": { - "description": "Endpoint defines an endpoint serving Prometheus metrics to be scraped by Prometheus.", - "properties": { - "authorization": { - "description": "`authorization` configures the Authorization header credentials to use when scraping the target. \n Cannot be set at the same time as `basicAuth`, or `oauth2`.", - "properties": { - "credentials": { - "description": "Selects a key of a Secret in the namespace that contains the credentials for authentication.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "type": { - "description": "Defines the authentication type. The value is case-insensitive. \n \"Basic\" is not a supported value. \n Default: \"Bearer\"", - "type": "string" - } - }, - "type": "object" - }, - "basicAuth": { - "description": "`basicAuth` configures the Basic Authentication credentials to use when scraping the target. \n Cannot be set at the same time as `authorization`, or `oauth2`.", - "properties": { - "password": { - "description": "`password` specifies a key of a Secret containing the password for authentication.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "username": { - "description": "`username` specifies a key of a Secret containing the username for authentication.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - } - }, - "type": "object" - }, - "bearerTokenFile": { - "description": "File to read bearer token for scraping the target. \n Deprecated: use `authorization` instead.", - "type": "string" - }, - "bearerTokenSecret": { - "description": "`bearerTokenSecret` specifies a key of a Secret containing the bearer token for scraping targets. The secret needs to be in the same namespace as the ServiceMonitor object and readable by the Prometheus Operator. \n Deprecated: use `authorization` instead.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "enableHttp2": { - "description": "`enableHttp2` can be used to disable HTTP2 when scraping the target.", - "type": "boolean" - }, - "filterRunning": { - "description": "When true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. \n If unset, the filtering is enabled. \n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase", - "type": "boolean" - }, - "followRedirects": { - "description": "`followRedirects` defines whether the scrape requests should follow HTTP 3xx redirects.", - "type": "boolean" - }, - "honorLabels": { - "description": "When true, `honorLabels` preserves the metric's labels when they collide with the target's labels.", - "type": "boolean" - }, - "honorTimestamps": { - "description": "`honorTimestamps` controls whether Prometheus preserves the timestamps when exposed by the target.", - "type": "boolean" - }, - "interval": { - "description": "Interval at which Prometheus scrapes the metrics from the target. \n If empty, Prometheus uses the global scrape interval.", - "pattern": "^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$", - "type": "string" - }, - "metricRelabelings": { - "description": "`metricRelabelings` configures the relabeling rules to apply to the samples before ingestion.", - "items": { - "description": "RelabelConfig allows dynamic rewriting of the label set for targets, alerts, scraped samples and remote write samples. \n More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config", - "properties": { - "action": { - "default": "replace", - "description": "Action to perform based on the regex matching. \n `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. \n Default: \"Replace\"", - "enum": [ - "replace", - "Replace", - "keep", - "Keep", - "drop", - "Drop", - "hashmod", - "HashMod", - "labelmap", - "LabelMap", - "labeldrop", - "LabelDrop", - "labelkeep", - "LabelKeep", - "lowercase", - "Lowercase", - "uppercase", - "Uppercase", - "keepequal", - "KeepEqual", - "dropequal", - "DropEqual" - ], - "type": "string" - }, - "modulus": { - "description": "Modulus to take of the hash of the source label values. \n Only applicable when the action is `HashMod`.", - "format": "int64", - "type": "integer" - }, - "regex": { - "description": "Regular expression against which the extracted value is matched.", - "type": "string" - }, - "replacement": { - "description": "Replacement value against which a Replace action is performed if the regular expression matches. \n Regex capture groups are available.", - "type": "string" - }, - "separator": { - "description": "Separator is the string between concatenated SourceLabels.", - "type": "string" - }, - "sourceLabels": { - "description": "The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression.", - "items": { - "description": "LabelName is a valid Prometheus label name which may only contain ASCII letters, numbers, as well as underscores.", - "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$", - "type": "string" - }, - "type": "array" - }, - "targetLabel": { - "description": "Label to which the resulting string is written in a replacement. \n It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. \n Regex capture groups are available.", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "oauth2": { - "description": "`oauth2` configures the OAuth2 settings to use when scraping the target. \n It requires Prometheus >= 2.27.0. \n Cannot be set at the same time as `authorization`, or `basicAuth`.", - "properties": { - "clientId": { - "description": "`clientId` specifies a key of a Secret or ConfigMap containing the OAuth2 client's ID.", - "properties": { - "configMap": { - "description": "ConfigMap containing data to use for the targets.", - "properties": { - "key": { - "description": "The key to select.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the ConfigMap or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "secret": { - "description": "Secret containing data to use for the targets.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - } - }, - "type": "object" - }, - "clientSecret": { - "description": "`clientSecret` specifies a key of a Secret containing the OAuth2 client's secret.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "endpointParams": { - "additionalProperties": { - "type": "string" - }, - "description": "`endpointParams` configures the HTTP parameters to append to the token URL.", - "type": "object" - }, - "scopes": { - "description": "`scopes` defines the OAuth2 scopes used for the token request.", - "items": { - "type": "string" - }, - "type": "array" - }, - "tokenUrl": { - "description": "`tokenURL` configures the URL to fetch the token from.", - "minLength": 1, - "type": "string" - } - }, - "required": [ - "clientId", - "clientSecret", - "tokenUrl" - ], - "type": "object" - }, - "params": { - "additionalProperties": { - "items": { - "type": "string" - }, - "type": "array" - }, - "description": "params define optional HTTP URL parameters.", - "type": "object" - }, - "path": { - "description": "HTTP path from which to scrape for metrics. \n If empty, Prometheus uses the default value (e.g. `/metrics`).", - "type": "string" - }, - "port": { - "description": "Name of the Service port which this endpoint refers to. \n It takes precedence over `targetPort`.", - "type": "string" - }, - "proxyUrl": { - "description": "`proxyURL` configures the HTTP Proxy URL (e.g. \"http://proxyserver:2195\") to go through when scraping the target.", - "type": "string" - }, - "relabelings": { - "description": "`relabelings` configures the relabeling rules to apply the target's metadata labels. \n The Operator automatically adds relabelings for a few standard Kubernetes fields. \n The original scrape job's name is available via the `__tmp_prometheus_job_name` label. \n More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config", - "items": { - "description": "RelabelConfig allows dynamic rewriting of the label set for targets, alerts, scraped samples and remote write samples. \n More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config", - "properties": { - "action": { - "default": "replace", - "description": "Action to perform based on the regex matching. \n `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. \n Default: \"Replace\"", - "enum": [ - "replace", - "Replace", - "keep", - "Keep", - "drop", - "Drop", - "hashmod", - "HashMod", - "labelmap", - "LabelMap", - "labeldrop", - "LabelDrop", - "labelkeep", - "LabelKeep", - "lowercase", - "Lowercase", - "uppercase", - "Uppercase", - "keepequal", - "KeepEqual", - "dropequal", - "DropEqual" - ], - "type": "string" - }, - "modulus": { - "description": "Modulus to take of the hash of the source label values. \n Only applicable when the action is `HashMod`.", - "format": "int64", - "type": "integer" - }, - "regex": { - "description": "Regular expression against which the extracted value is matched.", - "type": "string" - }, - "replacement": { - "description": "Replacement value against which a Replace action is performed if the regular expression matches. \n Regex capture groups are available.", - "type": "string" - }, - "separator": { - "description": "Separator is the string between concatenated SourceLabels.", - "type": "string" - }, - "sourceLabels": { - "description": "The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression.", - "items": { - "description": "LabelName is a valid Prometheus label name which may only contain ASCII letters, numbers, as well as underscores.", - "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$", - "type": "string" - }, - "type": "array" - }, - "targetLabel": { - "description": "Label to which the resulting string is written in a replacement. \n It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. \n Regex capture groups are available.", - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - }, - "scheme": { - "description": "HTTP scheme to use for scraping. \n `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. \n If empty, Prometheus uses the default value `http`.", - "enum": [ - "http", - "https" - ], - "type": "string" - }, - "scrapeTimeout": { - "description": "Timeout after which Prometheus considers the scrape to be failed. \n If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used.", - "pattern": "^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$", - "type": "string" - }, - "targetPort": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ], - "description": "Name or number of the target port of the `Pod` object behind the Service. The port must be specified with the container's port property.", - "x-kubernetes-int-or-string": true - }, - "tlsConfig": { - "description": "TLS configuration to use when scraping the target.", - "properties": { - "ca": { - "description": "Certificate authority used when verifying server certificates.", - "properties": { - "configMap": { - "description": "ConfigMap containing data to use for the targets.", - "properties": { - "key": { - "description": "The key to select.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the ConfigMap or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "secret": { - "description": "Secret containing data to use for the targets.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - } - }, - "type": "object" - }, - "caFile": { - "description": "Path to the CA cert in the Prometheus container to use for the targets.", - "type": "string" - }, - "cert": { - "description": "Client certificate to present when doing client-authentication.", - "properties": { - "configMap": { - "description": "ConfigMap containing data to use for the targets.", - "properties": { - "key": { - "description": "The key to select.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the ConfigMap or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "secret": { - "description": "Secret containing data to use for the targets.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - } - }, - "type": "object" - }, - "certFile": { - "description": "Path to the client cert file in the Prometheus container for the targets.", - "type": "string" - }, - "insecureSkipVerify": { - "description": "Disable target certificate validation.", - "type": "boolean" - }, - "keyFile": { - "description": "Path to the client key file in the Prometheus container for the targets.", - "type": "string" - }, - "keySecret": { - "description": "Secret containing the client key file for the targets.", - "properties": { - "key": { - "description": "The key of the secret to select from. Must be a valid secret key.", - "type": "string" - }, - "name": { - "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?", - "type": "string" - }, - "optional": { - "description": "Specify whether the Secret or its key must be defined", - "type": "boolean" - } - }, - "required": [ - "key" - ], - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "serverName": { - "description": "Used to verify the hostname for the targets.", - "type": "string" - } - }, - "type": "object" - }, - "trackTimestampsStaleness": { - "description": "`trackTimestampsStaleness` defines whether Prometheus tracks staleness of the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. \n It requires Prometheus >= v2.48.0.", - "type": "boolean" - } - }, - "type": "object" - }, - "type": "array" - }, - "jobLabel": { - "description": "`jobLabel` selects the label from the associated Kubernetes `Service` object which will be used as the `job` label for all metrics. \n For example if `jobLabel` is set to `foo` and the Kubernetes `Service` object is labeled with `foo: bar`, then Prometheus adds the `job=\"bar\"` label to all ingested metrics. \n If the value of this field is empty or if the label doesn't exist for the given Service, the `job` label of the metrics defaults to the name of the associated Kubernetes `Service`.", - "type": "string" - }, - "keepDroppedTargets": { - "description": "Per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. \n It requires Prometheus >= v2.47.0.", - "format": "int64", - "type": "integer" - }, - "labelLimit": { - "description": "Per-scrape limit on number of labels that will be accepted for a sample. \n It requires Prometheus >= v2.27.0.", - "format": "int64", - "type": "integer" - }, - "labelNameLengthLimit": { - "description": "Per-scrape limit on length of labels name that will be accepted for a sample. \n It requires Prometheus >= v2.27.0.", - "format": "int64", - "type": "integer" - }, - "labelValueLengthLimit": { - "description": "Per-scrape limit on length of labels value that will be accepted for a sample. \n It requires Prometheus >= v2.27.0.", - "format": "int64", - "type": "integer" - }, - "namespaceSelector": { - "description": "Selector to select which namespaces the Kubernetes `Endpoints` objects are discovered from.", - "properties": { - "any": { - "description": "Boolean describing whether all namespaces are selected in contrast to a list restricting them.", - "type": "boolean" - }, - "matchNames": { - "description": "List of namespace names to select from.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "podTargetLabels": { - "description": "`podTargetLabels` defines the labels which are transferred from the associated Kubernetes `Pod` object onto the ingested metrics.", - "items": { - "type": "string" - }, - "type": "array" - }, - "sampleLimit": { - "description": "`sampleLimit` defines a per-scrape limit on the number of scraped samples that will be accepted.", - "format": "int64", - "type": "integer" - }, - "scrapeProtocols": { - "description": "`scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). \n If unset, Prometheus uses its default value. \n It requires Prometheus >= v2.49.0.", - "items": { - "description": "ScrapeProtocol represents a protocol used by Prometheus for scraping metrics. Supported values are: * `OpenMetricsText0.0.1` * `OpenMetricsText1.0.0` * `PrometheusProto` * `PrometheusText0.0.4`", - "enum": [ - "PrometheusProto", - "OpenMetricsText0.0.1", - "OpenMetricsText1.0.0", - "PrometheusText0.0.4" - ], - "type": "string" - }, - "type": "array", - "x-kubernetes-list-type": "set" - }, - "selector": { - "description": "Label selector to select the Kubernetes `Endpoints` objects.", - "properties": { - "matchExpressions": { - "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", - "items": { - "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", - "properties": { - "key": { - "description": "key is the label key that the selector applies to.", - "type": "string" - }, - "operator": { - "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", - "type": "string" - }, - "values": { - "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "key", - "operator" - ], - "type": "object" - }, - "type": "array" - }, - "matchLabels": { - "additionalProperties": { - "type": "string" - }, - "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", - "type": "object" - } - }, - "type": "object", - "x-kubernetes-map-type": "atomic" - }, - "targetLabels": { - "description": "`targetLabels` defines the labels which are transferred from the associated Kubernetes `Service` object onto the ingested metrics.", - "items": { - "type": "string" - }, - "type": "array" - }, - "targetLimit": { - "description": "`targetLimit` defines a limit on the number of scraped targets that will be accepted.", - "format": "int64", - "type": "integer" - } - }, - "required": [ - "selector" - ], - "type": "object" - } - }, - "required": [ - "spec" - ], - "type": "object" - } - }, - "served": true, - "storage": true - } - ] - } - } diff --git a/operators/endpointmetrics/controllers/status/status_controller_integration_test.go b/operators/endpointmetrics/controllers/status/status_controller_integration_test.go index 66f252583..b456792a4 100644 --- a/operators/endpointmetrics/controllers/status/status_controller_integration_test.go +++ b/operators/endpointmetrics/controllers/status/status_controller_integration_test.go @@ -36,6 +36,7 @@ func TestIntegrationReconcileStatus(t *testing.T) { hubNamespace := "hub-namespace" obsAddonName := "observability-addon" + // Setup spoke cluster testEnv, k8sClient := setupTestEnv(t) defer testEnv.Stop() @@ -48,6 +49,7 @@ func TestIntegrationReconcileStatus(t *testing.T) { t.Fatalf("Failed to create resources: %v", err) } + // Setup hub cluster hubTestEnv, hubK8sClient := setupTestEnv(t) defer hubTestEnv.Stop() @@ -59,8 +61,10 @@ func TestIntegrationReconcileStatus(t *testing.T) { t.Fatalf("Failed to create resources: %v", err) } + // Setup controller manager mgr, err := ctrl.NewManager(testEnv.Config, ctrl.Options{ - Scheme: k8sClient.Scheme(), + Scheme: k8sClient.Scheme(), + MetricsBindAddress: "0", // Avoids port conflict with the default port 8080 }) assert.NoError(t, err) @@ -85,6 +89,9 @@ func TestIntegrationReconcileStatus(t *testing.T) { assert.NoError(t, err) }() + // Test: + // Update on the spoke addon status should trigger an update on the hub addon status. + go func() { // Update spoke addon status concurrently to trigger the reconcile loop. addCondition(spokeObsAddon, "Deployed", metav1.ConditionTrue) @@ -102,7 +109,6 @@ func TestIntegrationReconcileStatus(t *testing.T) { assert.NoError(t, err) }() - // Hub addon status should be updated err = wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) { hubObsAddon := &oav1beta1.ObservabilityAddon{} err := hubK8sClient.Get(context.Background(), types.NamespacedName{Name: obsAddonName, Namespace: hubNamespace}, hubObsAddon) diff --git a/operators/multiclusterobservability/api/shared/multiclusterobservability_shared.go b/operators/multiclusterobservability/api/shared/multiclusterobservability_shared.go index 9bbf588cd..9d38df4fa 100644 --- a/operators/multiclusterobservability/api/shared/multiclusterobservability_shared.go +++ b/operators/multiclusterobservability/api/shared/multiclusterobservability_shared.go @@ -20,11 +20,22 @@ import ( // +kubebuilder:validation:MaxLength=2083 type URL string +// Validate validates the underlying URL. func (u URL) Validate() error { _, err := url.Parse(string(u)) return err } +// HostPath returns the URL's host together with its path. +// This also runs a validation of the underlying url. +func (u URL) HostPath() (string, error) { + parsedUrl, err := url.Parse(string(u)) + if err != nil { + return "", err + } + return parsedUrl.Host + parsedUrl.Path, nil +} + // ObservabilityAddonSpec is the spec of observability addon. type ObservabilityAddonSpec struct { // EnableMetrics indicates the observability addon push metrics to hub server. diff --git a/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go b/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go index accea4da1..de03d3b35 100644 --- a/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go +++ b/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go @@ -108,6 +108,10 @@ type QuerySpec struct { // +optional ServiceAccountAnnotations map[string]string `json:"serviceAccountAnnotations,omitempty"` + // Set to true to use the old Prometheus engine for PromQL queries. + // +optional + UsePrometheusEngine bool `json:"usePrometheusEngine,omitempty"` + // WARNING: Use only with guidance from Red Hat Support. Using this feature incorrectly can // lead to an unrecoverable state, data loss, or both, which is not covered by Red Hat Support. // +optional diff --git a/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml b/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml index 2049c7cb1..77508b297 100644 --- a/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml +++ b/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml @@ -520,7 +520,7 @@ spec: resources: limits: cpu: 600m - memory: 1Gi + memory: 3Gi requests: cpu: 100m memory: 128Mi diff --git a/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml b/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml index f77faa0ff..baa1f724d 100644 --- a/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml +++ b/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml @@ -2172,6 +2172,9 @@ spec: type: string description: Annotations is an unstructured key value map stored with a service account type: object + usePrometheusEngine: + description: Set to true to use the old Prometheus engine for PromQL queries. + type: boolean type: object queryFrontend: description: spec for thanos-query-frontend diff --git a/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml b/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml index 90978c997..c6b348139 100644 --- a/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml +++ b/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml @@ -3426,6 +3426,10 @@ spec: description: Annotations is an unstructured key value map stored with a service account type: object + usePrometheusEngine: + description: Set to true to use the old Prometheus engine + for PromQL queries. + type: boolean type: object queryFrontend: description: spec for thanos-query-frontend diff --git a/operators/multiclusterobservability/config/manager/manager.yaml b/operators/multiclusterobservability/config/manager/manager.yaml index c4c0fdfa7..fe4f621c9 100644 --- a/operators/multiclusterobservability/config/manager/manager.yaml +++ b/operators/multiclusterobservability/config/manager/manager.yaml @@ -62,7 +62,7 @@ spec: resources: limits: cpu: 600m - memory: 1024Mi + memory: 3Gi requests: cpu: 100m memory: 128Mi diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/observatorium.go b/operators/multiclusterobservability/controllers/multiclusterobservability/observatorium.go index faf7f9075..912ffb015 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/observatorium.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/observatorium.go @@ -884,6 +884,11 @@ func newQuerySpec(mco *mcov1beta2.MultiClusterObservability) obsv1alpha1.QuerySp mco.Spec.AdvancedConfig.Query.Containers != nil { querySpec.Containers = mco.Spec.AdvancedConfig.Query.Containers } + + if mco.Spec.AdvancedConfig != nil && mco.Spec.AdvancedConfig.Query != nil && + mco.Spec.AdvancedConfig.Query.UsePrometheusEngine { + querySpec.UsePrometheusEngine = true + } return querySpec } diff --git a/operators/multiclusterobservability/controllers/placementrule/hub_info_secret.go b/operators/multiclusterobservability/controllers/placementrule/hub_info_secret.go index 29f862681..a32a1bccf 100644 --- a/operators/multiclusterobservability/controllers/placementrule/hub_info_secret.go +++ b/operators/multiclusterobservability/controllers/placementrule/hub_info_secret.go @@ -23,15 +23,15 @@ import ( func generateHubInfoSecret(client client.Client, obsNamespace string, namespace string, ingressCtlCrdExists bool) (*corev1.Secret, error) { - obsApiRouteHost := "" + obsAPIHost := "" alertmanagerEndpoint := "" alertmanagerRouterCA := "" if ingressCtlCrdExists { var err error - obsApiRouteHost, err = config.GetObsAPIHost(context.TODO(), client, obsNamespace) + obsAPIHost, err = config.GetObsAPIExternalHost(context.TODO(), client, obsNamespace) if err != nil { - log.Error(err, "Failed to get the host for observatorium API route") + log.Error(err, "Failed to get the host for Observatorium API host URL") return nil, err } @@ -56,7 +56,7 @@ func generateHubInfoSecret(client client.Client, obsNamespace string, } else { // for KinD support, the managedcluster and hub cluster are assumed in the same cluster, the observatorium-api // will be accessed through k8s service FQDN + port - obsApiRouteHost = config.GetOperandNamePrefix() + "observatorium-api" + "." + config.GetDefaultNamespace() + ".svc.cluster.local:8080" + obsAPIHost = config.GetOperandNamePrefix() + "observatorium-api" + "." + config.GetDefaultNamespace() + ".svc.cluster.local:8080" // if alerting is disabled, do not set alertmanagerEndpoint if !config.IsAlertingDisabled() { alertmanagerEndpoint = config.AlertmanagerServiceName + "." + config.GetDefaultNamespace() + ".svc.cluster.local:9095" @@ -70,7 +70,7 @@ func generateHubInfoSecret(client client.Client, obsNamespace string, } obsApiURL := url.URL{ - Host: obsApiRouteHost, + Host: obsAPIHost, Path: operatorconfig.ObservatoriumAPIRemoteWritePath, } if !obsApiURL.IsAbs() { diff --git a/operators/multiclusterobservability/controllers/placementrule/hub_info_secret_test.go b/operators/multiclusterobservability/controllers/placementrule/hub_info_secret_test.go index 63ba5d3e0..a144a621b 100644 --- a/operators/multiclusterobservability/controllers/placementrule/hub_info_secret_test.go +++ b/operators/multiclusterobservability/controllers/placementrule/hub_info_secret_test.go @@ -10,6 +10,8 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" + mcoshared "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/shared" + mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -148,10 +150,34 @@ func newTestAmDefaultCA() *corev1.ConfigMap { } } +func newMultiClusterObservability() *mcov1beta2.MultiClusterObservability { + return &mcov1beta2.MultiClusterObservability{ + TypeMeta: metav1.TypeMeta{Kind: "MultiClusterObservability"}, + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: mcov1beta2.MultiClusterObservabilitySpec{ + StorageConfig: &mcov1beta2.StorageConfig{ + MetricObjectStorage: &mcoshared.PreConfiguredStorage{ + Key: "test", + Name: "test", + }, + AlertmanagerStorageSize: "2Gi", + }, + }, + } +} + func TestNewSecret(t *testing.T) { initSchema(t) - objs := []runtime.Object{newTestObsApiRoute(), newTestAlertmanagerRoute(), newTestIngressController(), newTestRouteCASecret()} + mco := newMultiClusterObservability() + config.SetMonitoringCRName(mco.Name) + objs := []runtime.Object{ + newTestObsApiRoute(), + newTestAlertmanagerRoute(), + newTestIngressController(), + newTestRouteCASecret(), + mco, + } c := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() hubInfo, err := generateHubInfoSecret(c, mcoNamespace, namespace, true) @@ -166,6 +192,22 @@ func TestNewSecret(t *testing.T) { if !strings.HasPrefix(hub.ObservatoriumAPIEndpoint, "https://test-host") || hub.AlertmanagerEndpoint != "https://"+routeHost || hub.AlertmanagerRouterCA != routerCA { t.Fatalf("Wrong content in hub info secret: \ngot: "+hub.ObservatoriumAPIEndpoint+" "+hub.AlertmanagerEndpoint+" "+hub.AlertmanagerRouterCA, clusterName+" "+"https://test-host"+" "+"test-host"+" "+routerCA) } + + mco.Spec.AdvancedConfig = &mcov1beta2.AdvancedConfig{CustomObservabilityHubURL: "https://custom-obs", CustomAlertmanagerHubURL: "https://custom-am"} + c = fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + hubInfo, err = generateHubInfoSecret(c, mcoNamespace, namespace, true) + if err != nil { + t.Fatalf("Failed to initial the hub info secret: (%v)", err) + } + hub = &operatorconfig.HubInfo{} + err = yaml.Unmarshal(hubInfo.Data[operatorconfig.HubInfoSecretKey], &hub) + if err != nil { + t.Fatalf("Failed to unmarshal data in hub info secret (%v)", err) + } + if !strings.HasPrefix(hub.ObservatoriumAPIEndpoint, "https://custom-obs") || !strings.HasPrefix(hub.AlertmanagerEndpoint, "https://custom-am") || hub.AlertmanagerRouterCA != routerCA { + t.Fatalf("Wrong content in hub info secret: \ngot: "+hub.ObservatoriumAPIEndpoint+" "+hub.AlertmanagerEndpoint+" "+hub.AlertmanagerRouterCA, clusterName+" "+"https://custom-obs"+" "+"custom-obs"+" "+routerCA) + } + } func TestNewBYOSecret(t *testing.T) { diff --git a/operators/multiclusterobservability/pkg/certificates/certificates.go b/operators/multiclusterobservability/pkg/certificates/certificates.go index d942ae099..ce374f439 100644 --- a/operators/multiclusterobservability/pkg/certificates/certificates.go +++ b/operators/multiclusterobservability/pkg/certificates/certificates.go @@ -461,7 +461,7 @@ func pemEncode(cert []byte, key []byte) (*bytes.Buffer, *bytes.Buffer) { func getHosts(c client.Client, ingressCtlCrdExists bool) ([]string, error) { hosts := []string{config.GetObsAPISvc(config.GetOperandName(config.Observatorium))} if ingressCtlCrdExists { - url, err := config.GetObsAPIHost(context.TODO(), c, config.GetDefaultNamespace()) + url, err := config.GetObsAPIRouteHost(context.TODO(), c, config.GetDefaultNamespace()) if err != nil { log.Error(err, "Failed to get api route address") return nil, err diff --git a/operators/multiclusterobservability/pkg/config/config.go b/operators/multiclusterobservability/pkg/config/config.go index 9ad6ec534..0ce7a5b36 100644 --- a/operators/multiclusterobservability/pkg/config/config.go +++ b/operators/multiclusterobservability/pkg/config/config.go @@ -480,8 +480,23 @@ func GetDefaultTenantName() string { return defaultTenantName } -// GetObsAPIHost is used to get the URL for observartium api gateway. -func GetObsAPIHost(ctx context.Context, client client.Client, namespace string) (string, error) { +// GetObsAPIRouteHost is used to Route's host for Observatorium API. This doesn't take into consideration +// the `advanced.customObservabilityHubURL` configuration. +func GetObsAPIRouteHost(ctx context.Context, client client.Client, namespace string) (string, error) { + mco := &observabilityv1beta2.MultiClusterObservability{} + err := client.Get(ctx, + types.NamespacedName{ + Name: GetMonitoringCRName(), + }, mco) + if err != nil && !errors.IsNotFound(err) { + return "", err + } + return GetRouteHost(client, obsAPIGateway, namespace) +} + +// GetObsAPIExternalHost is used to get the frontend URL that should be used to reach the Observatorium API instance. +// This takes into consideration the `advanced.customObservabilityHubURL` configuration. +func GetObsAPIExternalHost(ctx context.Context, client client.Client, namespace string) (string, error) { mco := &observabilityv1beta2.MultiClusterObservability{} err := client.Get(ctx, types.NamespacedName{ @@ -492,11 +507,16 @@ func GetObsAPIHost(ctx context.Context, client client.Client, namespace string) } advancedConfig := mco.Spec.AdvancedConfig if advancedConfig != nil && advancedConfig.CustomObservabilityHubURL != "" { - err := advancedConfig.CustomObservabilityHubURL.Validate() + hubObsUrl := advancedConfig.CustomObservabilityHubURL + err := hubObsUrl.Validate() + if err != nil { + return "", err + } + obsHostPath, err := hubObsUrl.HostPath() if err != nil { return "", err } - return string(advancedConfig.CustomObservabilityHubURL), nil + return obsHostPath, nil } return GetRouteHost(client, obsAPIGateway, namespace) } diff --git a/operators/multiclusterobservability/pkg/config/config_test.go b/operators/multiclusterobservability/pkg/config/config_test.go index 407919465..353a9bc42 100644 --- a/operators/multiclusterobservability/pkg/config/config_test.go +++ b/operators/multiclusterobservability/pkg/config/config_test.go @@ -15,6 +15,7 @@ import ( routev1 "github.com/openshift/api/route/v1" fakeconfigclient "github.com/openshift/client-go/config/clientset/versioned/fake" observatoriumv1alpha1 "github.com/stolostron/observatorium-operator/api/v1alpha1" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -252,7 +253,7 @@ func TestGetClusterIDFailed(t *testing.T) { } } -func TestGetObsAPIHost(t *testing.T) { +func TestGetObsAPIRouteHost(t *testing.T) { route := &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: obsAPIGateway, @@ -267,12 +268,14 @@ func TestGetObsAPIHost(t *testing.T) { scheme.AddKnownTypes(mcov1beta2.GroupVersion, &mcov1beta2.MultiClusterObservability{}) client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(route).Build() - host, _ := GetObsAPIHost(context.TODO(), client, "default") + host, err := GetObsAPIRouteHost(context.TODO(), client, "default") + assert.NoError(t, err) if host == apiServerURL { t.Errorf("Should not get route host in default namespace") } - host, _ = GetObsAPIHost(context.TODO(), client, "test") + host, err = GetObsAPIRouteHost(context.TODO(), client, "test") + assert.NoError(t, err) if host != apiServerURL { t.Errorf("Observatorium api (%v) is not the expected (%v)", host, apiServerURL) } @@ -289,14 +292,70 @@ func TestGetObsAPIHost(t *testing.T) { }, } client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(route, mco).Build() - host, _ = GetObsAPIHost(context.TODO(), client, "test") - if host != customBaseURL { + host, err = GetObsAPIRouteHost(context.TODO(), client, "test") + assert.NoError(t, err) + if host != apiServerURL { + t.Errorf("Observatorium api (%v) is not the expected (%v)", host, apiServerURL) + } + + mco.Spec.AdvancedConfig.CustomObservabilityHubURL = "httpa://foob ar.c" + client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(route, mco).Build() + host, err = GetObsAPIRouteHost(context.TODO(), client, "test") + assert.NoError(t, err) + if host != apiServerURL { + t.Errorf("Observatorium api (%v) is not the expected (%v)", host, apiServerURL) + } +} + +func TestGetObsAPIExternalHost(t *testing.T) { + route := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: obsAPIGateway, + Namespace: "test", + }, + Spec: routev1.RouteSpec{ + Host: apiServerURL, + }, + } + scheme := runtime.NewScheme() + scheme.AddKnownTypes(routev1.GroupVersion, route) + scheme.AddKnownTypes(mcov1beta2.GroupVersion, &mcov1beta2.MultiClusterObservability{}) + client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(route).Build() + + host, err := GetObsAPIExternalHost(context.TODO(), client, "default") + assert.NoError(t, err) + if host == apiServerURL { + t.Errorf("Should not get route host in default namespace") + } + + host, err = GetObsAPIExternalHost(context.TODO(), client, "test") + assert.NoError(t, err) + if host != apiServerURL { + t.Errorf("Observatorium api (%v) is not the expected (%v)", host, apiServerURL) + } + + customBaseURL := "https://custom.base/url" + expectedHost := "custom.base/url" + mco := &mcov1beta2.MultiClusterObservability{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetMonitoringCRName(), + }, + Spec: mcov1beta2.MultiClusterObservabilitySpec{ + AdvancedConfig: &mcov1beta2.AdvancedConfig{ + CustomObservabilityHubURL: mcoshared.URL(customBaseURL), + }, + }, + } + client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(route, mco).Build() + host, err = GetObsAPIExternalHost(context.TODO(), client, "test") + assert.NoError(t, err) + if host != expectedHost { t.Errorf("Observatorium api (%v) is not the expected (%v)", host, customBaseURL) } mco.Spec.AdvancedConfig.CustomObservabilityHubURL = "httpa://foob ar.c" client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(route, mco).Build() - _, err := GetObsAPIHost(context.TODO(), client, "test") + _, err = GetObsAPIExternalHost(context.TODO(), client, "test") if err == nil { t.Errorf("expected error when parsing URL '%v', but got none", mco.Spec.AdvancedConfig.CustomObservabilityHubURL) } diff --git a/scripts/install-binaries.sh b/scripts/install-binaries.sh index c12472f42..36d15114d 100755 --- a/scripts/install-binaries.sh +++ b/scripts/install-binaries.sh @@ -115,6 +115,12 @@ install_e2e_tests_deps() { install_kustomize ${bin_dir} } +install_envtest_deps() { + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + bin_dir=${1:-${BIN_DIR}} + setup-envtest --bin-dir ${bin_dir} -p env use 1.30.x +} + # check if script is called directly, or sourced (return 0 2>/dev/null) && sourced=1 || sourced=0 # This allows functions within this file to be called individually from Makefile(s). diff --git a/tests/pkg/tests/observability-e2e-test_suite_test.go b/tests/pkg/tests/observability-e2e-test_suite_test.go index ef94ef363..4575459e7 100644 --- a/tests/pkg/tests/observability-e2e-test_suite_test.go +++ b/tests/pkg/tests/observability-e2e-test_suite_test.go @@ -13,6 +13,7 @@ import ( "time" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "gopkg.in/yaml.v2" @@ -129,7 +130,10 @@ func init() { func TestObservabilityE2E(t *testing.T) { RegisterFailHandler(Fail) + config.DefaultReporterConfig.NoColor = true + config.DefaultReporterConfig.Succinct = true junitReporter := reporters.NewJUnitReporter(reportFile) + junitReporter.ReporterConfig.NoColor = true RunSpecsWithDefaultAndCustomReporters(t, "Observability E2E Suite", []Reporter{junitReporter}) } @@ -141,8 +145,6 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { if !testFailed { uninstallMCO() - } else { - utils.PrintAllMCOPodsStatus(testOptions) } }) diff --git a/tests/pkg/tests/observability_addon_test.go b/tests/pkg/tests/observability_addon_test.go index 321748964..f9ce37591 100644 --- a/tests/pkg/tests/observability_addon_test.go +++ b/tests/pkg/tests/observability_addon_test.go @@ -105,17 +105,18 @@ var _ = Describe("Observability:", func() { It("[Stable] Waiting for check no metric data in grafana console", func() { Eventually(func() error { for _, cluster := range clusters { - err, hasMetric := utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, `timestamp(node_memory_MemAvailable_bytes{cluster="`+cluster+`}) - timestamp(node_memory_MemAvailable_bytes{cluster=`+cluster+`"} offset 1m) > 59`, - []string{`"__name__":"node_memory_MemAvailable_bytes"`}, ) - if err != nil && !hasMetric && - strings.Contains(err.Error(), "failed to find metric name from response") { - return nil + if err != nil { + return err + } + if len(res.Data.Result) != 0 { + return fmt.Errorf("Grafa console still has metric data: %v", res.Data.Result) } } - return fmt.Errorf("Check no metric data in grafana console error: %w", err) + return nil }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*5).Should(Succeed()) }) @@ -219,10 +220,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) - utils.PrintManagedClusterOBAObject(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go index a25df77e5..0d21d1aa1 100644 --- a/tests/pkg/tests/observability_alert_test.go +++ b/tests/pkg/tests/observability_alert_test.go @@ -191,9 +191,15 @@ var _ = Describe("Observability:", func() { By("Checking alert generated") Eventually(func() error { - err, _ := utils.ContainManagedClusterMetric(testOptions, `ALERTS{`+labelName+`="`+labelValue+`"}`, - []string{`"__name__":"ALERTS"`, `"` + labelName + `":"` + labelValue + `"`}) - return err + query := fmt.Sprintf(`ALERTS{%s="%s"}`, labelName, labelValue) + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } + return nil }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) @@ -217,6 +223,7 @@ var _ = Describe("Observability:", func() { It("[P2][Sev2][observability][Stable] Should have custom alert updated (alert/g0)", func() { By("Updating custom alert rules") + // Replace preceding custom alert with new one that cannot fire yamlB, _ := kustomize.Render( kustomize.Options{KustomizationPath: "../../../examples/alerts/custom_rules_invalid"}, ) @@ -236,12 +243,21 @@ var _ = Describe("Observability:", func() { By("Checking alert generated") Eventually( func() error { - err, _ := utils.ContainManagedClusterMetric(testOptions, `ALERTS{`+labelName+`="`+labelValue+`"}`, - []string{`"__name__":"ALERTS"`, `"` + labelName + `":"` + labelValue + `"`}) - return err + query := fmt.Sprintf(`ALERTS{%s="%s"}`, labelName, labelValue) + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + + if len(res.Data.Result) != 0 { + // No alert should be generated + return fmt.Errorf("alert should not be generated, got %v", res) + } + + return nil }, EventuallyTimeoutMinute*5, - EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Stable] delete the customized rules (alert/g0)", func() { @@ -394,9 +410,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_certrenew_test.go b/tests/pkg/tests/observability_certrenew_test.go index 3e2b303bb..cdebc9b8c 100644 --- a/tests/pkg/tests/observability_certrenew_test.go +++ b/tests/pkg/tests/observability_certrenew_test.go @@ -150,18 +150,19 @@ var _ = Describe("Observability:", func() { namespace, "component=metrics-collector", ) - if err == nil { - for _, pod := range podList.Items { - if pod.Name != collectorPodName { - if pod.Status.Phase != "Running" { - klog.V(1).Infof("<%s> not in Running status yet", pod.Name) - return false - } - return true + if err != nil { + klog.V(1).Infof("Failed to get pod list: %v", err) + } + for _, pod := range podList.Items { + if pod.Name != collectorPodName { + if pod.Status.Phase != "Running" { + klog.V(1).Infof("<%s> not in Running status yet", pod.Name) + return false } + return true } - } + // debug code to check label "cert/time-restarted" deployment, err := utils.GetDeployment( testOptions, @@ -182,9 +183,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed namespace = MCO_ADDON_NAMESPACE diff --git a/tests/pkg/tests/observability_config_test.go b/tests/pkg/tests/observability_config_test.go index 00b63d9ec..10336a7ed 100644 --- a/tests/pkg/tests/observability_config_test.go +++ b/tests/pkg/tests/observability_config_test.go @@ -279,9 +279,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_dashboard_test.go b/tests/pkg/tests/observability_dashboard_test.go index 80cd3417e..a2ab14765 100644 --- a/tests/pkg/tests/observability_dashboard_test.go +++ b/tests/pkg/tests/observability_dashboard_test.go @@ -85,9 +85,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_endpoint_preserve_test.go b/tests/pkg/tests/observability_endpoint_preserve_test.go index 446432e18..c79d31332 100644 --- a/tests/pkg/tests/observability_endpoint_preserve_test.go +++ b/tests/pkg/tests/observability_endpoint_preserve_test.go @@ -206,9 +206,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } namespace = MCO_ADDON_NAMESPACE testFailed = testFailed || CurrentGinkgoTestDescription().Failed diff --git a/tests/pkg/tests/observability_export_test.go b/tests/pkg/tests/observability_export_test.go index 630cb18bf..508ca1159 100644 --- a/tests/pkg/tests/observability_export_test.go +++ b/tests/pkg/tests/observability_export_test.go @@ -5,7 +5,6 @@ package tests import ( - "errors" "fmt" "os" @@ -77,47 +76,42 @@ var _ = Describe("Observability:", func() { By("Waiting for metrics acm_remote_write_requests_total on grafana console") Eventually(func() error { query := fmt.Sprintf("acm_remote_write_requests_total{cluster=\"%s\"} offset 1m", hubClusterName) - err, _ := utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, query, - []string{`"__name__":"acm_remote_write_requests_total"`}, ) if err != nil { return err } - err, _ = utils.ContainManagedClusterMetric( - testOptions, - query, - []string{`"__name__":"acm_remote_write_requests_total"`, - `"code":"200`, `"name":"thanos-receiver"`}, - ) - if err != nil { - return errors.New("metrics not forwarded to thanos-receiver") + if len(res.Data.Result) == 0 { + return fmt.Errorf("metric %s not found in response", query) } - err, _ = utils.ContainManagedClusterMetric( - testOptions, - query, - []string{`"__name__":"acm_remote_write_requests_total"`, - `"code":"204`, `"name":"victoriametrics"`}, - ) - if err != nil { - return errors.New("metrics not forwarded to victoriametrics") + + // Check if the metric is forwarded to thanos-receiver + labelSet := map[string]string{"code": "200", "name": "thanos-receiver"} + if !res.ContainsLabelsSet(labelSet) { + return fmt.Errorf("labels %v not found in response: %v", labelSet, res) + } + + // Check if the metric is forwarded to victoriametrics + labelSet = map[string]string{"code": "204", "name": "victoriametrics"} + if !res.ContainsLabelsSet(labelSet) { + return fmt.Errorf("labels %v not found in response: %v", labelSet, res) } + return nil - }, EventuallyTimeoutMinute*20, EventuallyIntervalSecond*5).Should(Succeed()) + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) JustAfterEach(func() { - Expect(utils.CleanExportResources(testOptions)).NotTo(HaveOccurred()) - Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) - }) - - AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) + + AfterEach(func() { + Expect(utils.CleanExportResources(testOptions)).NotTo(HaveOccurred()) + Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) + }) }) diff --git a/tests/pkg/tests/observability_grafana_dev_test.go b/tests/pkg/tests/observability_grafana_dev_test.go index 09ed3da07..13fe02af2 100644 --- a/tests/pkg/tests/observability_grafana_dev_test.go +++ b/tests/pkg/tests/observability_grafana_dev_test.go @@ -35,9 +35,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_grafana_test.go b/tests/pkg/tests/observability_grafana_test.go index 818656160..908351401 100644 --- a/tests/pkg/tests/observability_grafana_test.go +++ b/tests/pkg/tests/observability_grafana_test.go @@ -34,14 +34,17 @@ var _ = Describe("Observability:", func() { } for _, cluster := range clusters { query := fmt.Sprintf("node_memory_MemAvailable_bytes{cluster=\"%s\"}", cluster) - err, _ = utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, query, - []string{`"__name__":"node_memory_MemAvailable_bytes"`}, ) if err != nil { return err } + + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } } return nil }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) @@ -53,9 +56,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_install_test.go b/tests/pkg/tests/observability_install_test.go index 90340c11b..485196ba3 100644 --- a/tests/pkg/tests/observability_install_test.go +++ b/tests/pkg/tests/observability_install_test.go @@ -190,7 +190,7 @@ func installMCO() { mcoLogs, err := utils.GetPodLogs(testOptions, true, mcoNs, mcoPod, "multicluster-observability-operator", false, 1000) Expect(err).NotTo(HaveOccurred()) fmt.Fprintf(GinkgoWriter, "[DEBUG] MCO is installed failed, checking MCO operator logs:\n%s\n", mcoLogs) - utils.PrintAllMCOPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) }() By("Waiting for MCO ready status") @@ -214,7 +214,7 @@ func installMCO() { } fmt.Fprintf(GinkgoWriter, "[DEBUG] Addon failed, checking pods:\n") - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) }() By("Check endpoint-operator and metrics-collector pods are ready") Eventually(func() error { diff --git a/tests/pkg/tests/observability_manifestwork_test.go b/tests/pkg/tests/observability_manifestwork_test.go index f654a6ebc..193abd615 100644 --- a/tests/pkg/tests/observability_manifestwork_test.go +++ b/tests/pkg/tests/observability_manifestwork_test.go @@ -7,6 +7,7 @@ package tests import ( "context" "errors" + "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -106,12 +107,20 @@ var _ = Describe("Observability:", func() { It("[Stable] Checking metric to ensure that no data is lost in 1 minute", func() { Eventually(func() error { - err, _ = utils.ContainManagedClusterMetric( + query := fmt.Sprintf(`timestamp(node_memory_MemAvailable_bytes{cluster="%s"}) - timestamp(node_memory_MemAvailable_bytes{cluster="%s"} offset 1m) > 59`, clusterName, clusterName) + res, err := utils.QueryGrafana( testOptions, - `timestamp(node_memory_MemAvailable_bytes{cluster="`+clusterName+`}) - timestamp(node_memory_MemAvailable_bytes{cluster=`+clusterName+`"} offset 1m) > 59`, - []string{`"__name__":"node_memory_MemAvailable_bytes"`}, + query, ) - return err + if err != nil { + return err + } + + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } + + return nil }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*3).Should(Succeed()) }) } @@ -123,9 +132,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_metrics_test.go b/tests/pkg/tests/observability_metrics_test.go index 6e6ed9c38..03f1b59e2 100644 --- a/tests/pkg/tests/observability_metrics_test.go +++ b/tests/pkg/tests/observability_metrics_test.go @@ -11,7 +11,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" "github.com/stolostron/multicluster-observability-operator/tests/pkg/kustomize" "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" @@ -22,9 +21,8 @@ const ( ) var ( - clusters []string - clusterError error - metricslistError error + clusters []string + clusterError error ) var _ = Describe("Observability:", func() { @@ -66,14 +64,17 @@ var _ = Describe("Observability:", func() { Eventually(func() error { for _, cluster := range clusters { query := fmt.Sprintf("node_memory_Active_bytes{cluster=\"%s\"} offset 1m", cluster) - err, _ := utils.ContainManagedClusterMetric( + res, err := utils.QueryGrafana( testOptions, query, - []string{`"__name__":"node_memory_Active_bytes"`}, ) if err != nil { return err } + + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } } return nil }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) @@ -88,13 +89,17 @@ var _ = Describe("Observability:", func() { cluster, cluster, ) - metricslistError, _ = utils.ContainManagedClusterMetric(testOptions, query, []string{}) - if metricslistError == nil { - return nil + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + // there should be no data for the deleted metric + if len(res.Data.Result) != 0 { + return fmt.Errorf("metric %s found in response: %v", query, res) } } - return metricslistError - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Integration] Should have no metrics which have been marked for deletion in matches section (metrics/g0)", func() { @@ -106,13 +111,16 @@ var _ = Describe("Observability:", func() { cluster, cluster, ) - metricslistError, _ = utils.ContainManagedClusterMetric(testOptions, query, []string{}) - if metricslistError == nil { - return nil + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + if len(res.Data.Result) != 0 { + return fmt.Errorf("metric %s found in response: %v", query, res) } } - return metricslistError - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Integration] Should have no metrics after custom metrics allowlist deleted (metrics/g0)", func() { @@ -132,13 +140,16 @@ var _ = Describe("Observability:", func() { cluster, cluster, ) - metricslistError, _ = utils.ContainManagedClusterMetric(testOptions, query, []string{}) - if metricslistError == nil { - return nil + res, err := utils.QueryGrafana(testOptions, query) + if err != nil { + return err + } + if len(res.Data.Result) != 0 { + return fmt.Errorf("metric %s found in response: %v", query, res) } } - return metricslistError - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(MatchError("failed to find metric name from response")) + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) It("[P2][Sev2][observability][Integration] Should have metrics which used grafana dashboard (ssli/g1)", func() { @@ -162,11 +173,14 @@ var _ = Describe("Observability:", func() { _, ok := ignoreMetricMap[name] if !ok { Eventually(func() error { - err, _ := utils.ContainManagedClusterMetric(testOptions, name, []string{name}) + res, err := utils.QueryGrafana(testOptions, name) if err != nil { - klog.V(1).Infof("failed to get metrics %s", name) + return fmt.Errorf("failed to get metrics %s: %v", name, err) } - return err + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", name) + } + return nil }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*3).Should(Succeed()) } } @@ -178,9 +192,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_observatorium_preserve_test.go b/tests/pkg/tests/observability_observatorium_preserve_test.go index d2786f420..b1bbd2aab 100644 --- a/tests/pkg/tests/observability_observatorium_preserve_test.go +++ b/tests/pkg/tests/observability_observatorium_preserve_test.go @@ -6,6 +6,8 @@ package tests import ( "context" + "errors" + "fmt" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -70,21 +72,25 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*3, EventuallyIntervalSecond*1).Should(BeTrue()) // ensure the thanos compact is restarted - Eventually(func() bool { + Eventually(func() error { sts, err := utils.GetStatefulSetWithLabel(testOptions, true, THANOS_COMPACT_LABEL, MCO_NAMESPACE) - if err == nil { - if (*sts).Items[0].ResourceVersion != oldCompactResourceVersion { - argList := (*sts).Items[0].Spec.Template.Spec.Containers[0].Args - for _, arg := range argList { - if arg != "--retention.resolution-raw="+updateRetention { - return true - } - } - return false + if err != nil { + return err + } + if sts.Items[0].ResourceVersion != oldCompactResourceVersion { + return errors.New("The thanos compact pod is not restarted. ResourceVersion has not changed.") + } + + argList := sts.Items[0].Spec.Template.Spec.Containers[0].Args + for _, arg := range argList { + // check if the retention resolution is reverted to the original value + if arg == "--retention.resolution-raw="+updateRetention { + return fmt.Errorf("The thanos compact pod is not restarted with the new retention resolution. Args: %v", argList) } } - return false - }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(BeTrue()) + + return nil + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) By("Wait for thanos compact pods are ready") sts, err := utils.GetStatefulSetWithLabel(testOptions, true, THANOS_COMPACT_LABEL, MCO_NAMESPACE) @@ -107,9 +113,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_reconcile_test.go b/tests/pkg/tests/observability_reconcile_test.go index 8ed55bac4..c37470a64 100644 --- a/tests/pkg/tests/observability_reconcile_test.go +++ b/tests/pkg/tests/observability_reconcile_test.go @@ -201,9 +201,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_retention_test.go b/tests/pkg/tests/observability_retention_test.go index 66b207cf0..acd9ea220 100644 --- a/tests/pkg/tests/observability_retention_test.go +++ b/tests/pkg/tests/observability_retention_test.go @@ -177,9 +177,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_route_test.go b/tests/pkg/tests/observability_route_test.go index 37c5099f5..9b01930e9 100644 --- a/tests/pkg/tests/observability_route_test.go +++ b/tests/pkg/tests/observability_route_test.go @@ -195,9 +195,7 @@ var _ = Describe("Observability:", func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - utils.PrintMCOObject(testOptions) - utils.PrintAllMCOPodsStatus(testOptions) - utils.PrintAllOBAPodsStatus(testOptions) + utils.LogFailingTestStandardDebugInfo(testOptions) } testFailed = testFailed || CurrentGinkgoTestDescription().Failed }) diff --git a/tests/pkg/tests/observability_uninstall_test.go b/tests/pkg/tests/observability_uninstall_test.go index 5dbb86f8f..0dc0f89b4 100644 --- a/tests/pkg/tests/observability_uninstall_test.go +++ b/tests/pkg/tests/observability_uninstall_test.go @@ -56,7 +56,7 @@ func uninstallMCO() { Namespace(MCO_ADDON_NAMESPACE). Get(context.TODO(), name, metav1.GetOptions{}) if instance != nil { - utils.PrintManagedClusterOBAObject(testOptions) + utils.PrintObject(context.Background(), clientDynamic, utils.NewMCOAddonGVR(), MCO_ADDON_NAMESPACE, "observability-addon") return errors.New("Failed to delete MCO addon instance") } return nil diff --git a/tests/pkg/utils/cluster_deploy.go b/tests/pkg/utils/cluster_deploy.go deleted file mode 100644 index c16ecf2a3..000000000 --- a/tests/pkg/utils/cluster_deploy.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Red Hat, Inc. -// Copyright Contributors to the Open Cluster Management project -// Licensed under the Apache License 2.0 - -package utils - -// ClusterDeploy defines the data passed to Hive. -type ClusterDeploy struct { - Kind string `yaml:"kind"` - APIVersion string `yaml:"apiVersion"` - Items []Items `yaml:"items"` -} - -// Items defines the list of items in the cluster deploy yaml. -type Items struct { - Kind string `yaml:"kind"` - Metadata Metadata `yaml:"metadata"` - StringData StringData `yaml:"stringData,omitempty"` - Spec Spec `yaml:"spec,omitempty"` -} - -// Metadata defines the name. -type Metadata struct { - Name string `yaml:"name,omitempty"` -} - -// StringData defiines the ssh values. -type StringData struct { - Dockerconfigjson string `yaml:".dockerconfigjson,omitempty"` - SSHPrivateKey string `yaml:"ssh-privatekey,omitempty"` -} - -// Spec defines the kube specifications. -type Spec struct { - BaseDomain string `yaml:"baseDomain,omitempty"` - ClusterName string `yaml:"clusterName,omitempty"` - Provisioning Provisioning `yaml:"provisioning,omitempty"` -} - -// Provisioning defines the data related to cluster creation. -type Provisioning struct { - ReleaseImage string `yaml:"releaseImage,omitempty"` - SSHKnownHosts []string `yaml:"sshKnownHosts,omitempty"` -} diff --git a/tests/pkg/utils/install_config.go b/tests/pkg/utils/install_config.go deleted file mode 100644 index 298fefde5..000000000 --- a/tests/pkg/utils/install_config.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Red Hat, Inc. -// Copyright Contributors to the Open Cluster Management project -// Licensed under the Apache License 2.0 - -package utils - -// InstallConfig definition for install config structure from install-config.yaml. -type InstallConfig struct { - BaseDomain string `yaml:"baseDomain,omitempty"` - Networking Networking `yaml:"networking,omitempty"` - Metadata Metadata `yaml:"metadata"` - Platform Platform `yaml:"platform,omitempty"` - PullSecret string `yaml:"pullSecret,omitempty"` - SSHKey string `yaml:"sshKey,omitempty"` -} - -// Networking definition. -type Networking struct { - NetworkType string `yaml:"networkType"` - MachineCIDR string `yaml:"machineCIDR"` -} - -// Platform definition. -type Platform struct { - Baremetal Baremetal `yaml:"baremetal,omitempty"` -} - -// Baremetal specs for target baremetal provisioning. -type Baremetal struct { - ExternalBridge string `yaml:"externalBridge,omitempty"` - ProvisioningBridge string `yaml:"provisioningBridge,omitempty"` - LibvirtURI string `yaml:"libvirtURI,omitempty"` - ProvisioningNetworkInterface string `yaml:"provisioningNetworkInterface,omitempty"` - ProvisioningNetworkCIDR string `yaml:"provisioningNetworkCIDR,omitempty"` - APIVIP string `yaml:"apiVIP,omitempty"` - DNSVIP string `yaml:"dnsVIP,omitempty"` - IngressVIP string `yaml:"ingressVIP,omitempty"` - Hosts []Host `yaml:"hosts,omitempty"` - SSHKnownHosts string `yaml:"sshKnownHosts,omitempty"` -} - -// Host is an array of baremetal assets. -type Host struct { - Name string `yaml:"name"` - Role string `yaml:"role"` - Bmc Bmc `yaml:"bmc"` - BootMACAddress string `yaml:"bootMACAddress"` - HardwareProfile string `yaml:"hardwareProfile"` -} - -// Bmc definition. -type Bmc struct { - Address string `yaml:"address"` - Username string `yaml:"username"` - Password string `yaml:"password"` -} diff --git a/tests/pkg/utils/kube_debug.go b/tests/pkg/utils/kube_debug.go new file mode 100644 index 000000000..14471825c --- /dev/null +++ b/tests/pkg/utils/kube_debug.go @@ -0,0 +1,434 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package utils + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +// LogFailingTestStandardDebugInfo logs standard debug info for failing tests. +// It scans workloads and pods from hub and managed clusters observability namespaces. +// It also prints MCO and OBA objects. +// If a workload or pod is not running, it prints the resource spec, status, events and logs if appropriate. +func LogFailingTestStandardDebugInfo(opt TestOptions) { + klog.V(1).Infof("Test failed, printing debug info. TestOptions: %+v", opt) + + // Print MCO object + hubDynClient := NewKubeClientDynamic( + opt.HubCluster.ClusterServerURL, + opt.KubeConfig, + opt.HubCluster.KubeContext) + PrintObject(context.TODO(), hubDynClient, NewMCOGVRV1BETA2(), MCO_NAMESPACE, MCO_CR_NAME) + + // Check pods in hub + hubClient := NewKubeClient( + opt.HubCluster.ClusterServerURL, + opt.KubeConfig, + opt.HubCluster.KubeContext) + CheckPodsInNamespace(hubClient, "open-cluster-management", []string{"multicluster-observability-operator"}, map[string]string{ + "name": "multicluster-observability-operator", + }) + CheckDeploymentsInNamespace(hubClient, MCO_NAMESPACE) + CheckStatefulSetsInNamespace(hubClient, MCO_NAMESPACE) + CheckDaemonSetsInNamespace(hubClient, MCO_NAMESPACE) + CheckPodsInNamespace(hubClient, MCO_NAMESPACE, []string{}, map[string]string{}) + printConfigMapsInNamespace(hubClient, MCO_NAMESPACE) + printSecretsInNamespace(hubClient, MCO_NAMESPACE) + + for _, mc := range opt.ManagedClusters { + if mc.Name == "local-cluster" { + // Skip local-cluster as same namespace as hub, and already checked + continue + } + + spokeDynClient := NewKubeClientDynamic(mc.ClusterServerURL, opt.KubeConfig, mc.KubeContext) + PrintObject(context.TODO(), spokeDynClient, NewMCOAddonGVR(), MCO_ADDON_NAMESPACE, "observability-addon") + + spokeClient := NewKubeClient(mc.ClusterServerURL, mc.KubeConfig, mc.KubeContext) + CheckDeploymentsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + CheckStatefulSetsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + CheckDaemonSetsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + CheckPodsInNamespace(spokeClient, MCO_ADDON_NAMESPACE, []string{"observability-addon"}, map[string]string{}) + printConfigMapsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + printSecretsInNamespace(spokeClient, MCO_ADDON_NAMESPACE) + } +} + +// CheckPodsInNamespace lists pods in a namespace and logs debug info (status, events, logs) for pods not running. +func CheckPodsInNamespace(client kubernetes.Interface, ns string, forcePodNamesLog []string, podLabels map[string]string) { + listOptions := metav1.ListOptions{} + if len(podLabels) > 0 { + listOptions.LabelSelector = metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: podLabels}) + } + pods, err := client.CoreV1().Pods(ns).List(context.TODO(), listOptions) + if err != nil { + klog.Errorf("Failed to get pods in namespace %s: %v", ns, err) + return + } + + if len(pods.Items) == 0 { + klog.V(1).Infof("No pods in namespace %s", ns) + } + + klog.V(1).Infof("Checking %d pods in namespace %q", len(pods.Items), ns) + printPodsStatuses(pods.Items) + + notRunningPodsCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning { + notRunningPodsCount++ + } + + force := false + for _, forcePodName := range forcePodNamesLog { + if strings.Contains(pod.Name, forcePodName) { + force = true + break + } + } + if pod.Status.Phase == corev1.PodRunning && !force { + continue + } + + // print pod spec + podSpec, err := json.MarshalIndent(pod.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal pod %q spec: %s", pod.Name, err.Error()) + } + klog.V(1).Infof("Pod %q spec: \n%s", pod.Name, string(podSpec)) + + LogPodStatus(pod) + LogObjectEvents(client, ns, "Pod", pod.Name) + LogPodLogs(client, ns, pod) + } + + if notRunningPodsCount == 0 { + klog.V(1).Infof("All pods are running in namespace %q", ns) + } else { + klog.Errorf("Found %d pods not running in namespace %q", notRunningPodsCount, ns) + } +} + +func LogPodStatus(podList corev1.Pod) { + var podStatus strings.Builder + podStatus.WriteString(">>>>>>>>>> pod status >>>>>>>>>>\n") + podStatus.WriteString("Conditions:\n") + for _, condition := range podList.Status.Conditions { + podStatus.WriteString(fmt.Sprintf("\t%s: %s %v\n", condition.Type, condition.Status, condition.LastTransitionTime.Time)) + } + podStatus.WriteString("ContainerStatuses:\n") + for _, containerStatus := range podList.Status.ContainerStatuses { + podStatus.WriteString(fmt.Sprintf("\t%s: %t %d %v\n", containerStatus.Name, containerStatus.Ready, containerStatus.RestartCount, containerStatus.State)) + if containerStatus.LastTerminationState.Terminated != nil { + podStatus.WriteString(fmt.Sprintf("\t\tlastTerminated: %v\n", containerStatus.LastTerminationState.Terminated)) + } + } + podStatus.WriteString("<<<<<<<<<< pod status <<<<<<<<<<") + + klog.V(1).Infof("Pod %q is in phase %q and status: \n%s", podList.Name, podList.Status.Phase, podStatus.String()) +} + +func LogPodLogs(client kubernetes.Interface, ns string, pod corev1.Pod) { + for _, container := range pod.Spec.Containers { + logsRes := client.CoreV1().Pods(ns).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: container.Name, + }).Do(context.Background()) + + if logsRes.Error() != nil { + klog.Errorf("Failed to get logs for pod %q: %s", pod.Name, logsRes.Error()) + continue + } + + logs, err := logsRes.Raw() + if err != nil { + klog.Errorf("Failed to get logs for pod %q container %q: %s", pod.Name, container.Name, err.Error()) + continue + } + + // Filter error logs and keep all last 100 lines + maxLines := 100 + cleanedLines := []string{} + lines := strings.Split(string(logs), "\n") + for i, line := range lines { + if strings.Contains(strings.ToLower(line), "error") || i > len(lines)-maxLines { + cleanedLines = append(cleanedLines, line) + } + } + + logs = []byte(strings.Join(cleanedLines, "\n")) + + delimitedLogs := fmt.Sprintf(">>>>>>>>>> container logs >>>>>>>>>>\n%s<<<<<<<<<< container logs <<<<<<<<<<", string(logs)) + klog.V(1).Infof("Pod %q container %q logs (errors and last %d lines): \n%s", pod.Name, container.Name, maxLines, delimitedLogs) + } +} + +func CheckDeploymentsInNamespace(client kubernetes.Interface, ns string) { + deployments, err := client.AppsV1().Deployments(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get deployments in namespace %s: %v", ns, err) + return + } + + if len(deployments.Items) == 0 { + klog.V(1).Infof("No deployments found in namespace %q", ns) + } + + klog.V(1).Infof("Deployments in namespace %s: \n", ns) + printDeploymentsStatuses(client, ns) + + for _, deployment := range deployments.Items { + if deployment.Status.UpdatedReplicas == *deployment.Spec.Replicas { + continue + } + + // print deployment spec + deploymentSpec, err := json.MarshalIndent(deployment.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal deployment %q spec: %s", deployment.Name, err.Error()) + } + klog.V(1).Infof("Deployment %q spec: \n%s", deployment.Name, string(deploymentSpec)) + + LogDeploymentStatus(deployment) + LogObjectEvents(client, ns, "Deployment", deployment.Name) + } +} + +func LogDeploymentStatus(deployment appsv1.Deployment) { + var deploymentStatus strings.Builder + deploymentStatus.WriteString(">>>>>>>>>> deployment status >>>>>>>>>>\n") + deploymentStatus.WriteString(fmt.Sprintf("ReadyReplicas: %d\n", deployment.Status.ReadyReplicas)) + deploymentStatus.WriteString(fmt.Sprintf("UpdatedReplicas: %d\n", deployment.Status.UpdatedReplicas)) + deploymentStatus.WriteString(fmt.Sprintf("AvailableReplicas: %d\n", deployment.Status.AvailableReplicas)) + deploymentStatus.WriteString("Conditions:\n") + for _, condition := range deployment.Status.Conditions { + deploymentStatus.WriteString(fmt.Sprintf("\t%s: %s %v \n\t\t%s %s\n", condition.Type, condition.Status, condition.LastTransitionTime, condition.Message, condition.Reason)) + } + deploymentStatus.WriteString("<<<<<<<<<< deployment status <<<<<<<<<<") + + klog.V(1).Infof("Deployment %q status: \n%s", deployment.Name, deploymentStatus.String()) +} + +func CheckStatefulSetsInNamespace(client kubernetes.Interface, ns string) { + statefulSets, err := client.AppsV1().StatefulSets(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get statefulsets in namespace %s: %v", ns, err) + return + } + + if len(statefulSets.Items) == 0 { + klog.V(1).Infof("No statefulsets found in namespace %q", ns) + return + } + + klog.V(1).Infof("StatefulSets in namespace %s: \n", ns) + printStatefulSetsStatuses(client, ns) + + for _, statefulSet := range statefulSets.Items { + if statefulSet.Status.UpdatedReplicas == *statefulSet.Spec.Replicas { + continue + } + + // Print statefulset spec + statefulSetSpec, err := json.MarshalIndent(statefulSet.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal statefulset %q spec: %s", statefulSet.Name, err.Error()) + } + klog.V(1).Infof("StatefulSet %q spec: \n%s", statefulSet.Name, string(statefulSetSpec)) + + LogObjectEvents(client, ns, "StatefulSet", statefulSet.Name) + } +} + +func CheckDaemonSetsInNamespace(client kubernetes.Interface, ns string) { + daemonSets, err := client.AppsV1().DaemonSets(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get daemonsets in namespace %s: %v", ns, err) + return + } + + if len(daemonSets.Items) == 0 { + klog.V(1).Infof("No daemonsets found in namespace %q", ns) + return + } + + klog.V(1).Infof("DaemonSets in namespace %s: \n", ns) + printDaemonSetsStatuses(client, ns) + + for _, daemonSet := range daemonSets.Items { + if daemonSet.Status.UpdatedNumberScheduled == daemonSet.Status.DesiredNumberScheduled { + continue + } + + // Print daemonset spec + daemonSetSpec, err := json.MarshalIndent(daemonSet.Spec, "", " ") + if err != nil { + klog.Errorf("Failed to marshal daemonset %q spec: %s", daemonSet.Name, err.Error()) + } + klog.V(1).Infof("DaemonSet %q spec: \n%s", daemonSet.Name, string(daemonSetSpec)) + + LogObjectEvents(client, ns, "DaemonSet", daemonSet.Name) + } +} + +func LogObjectEvents(client kubernetes.Interface, ns string, kind string, name string) { + fieldSelector := fmt.Sprintf("involvedObject.kind=%s,involvedObject.name=%s", kind, name) + events, err := client.CoreV1().Events(ns).List(context.TODO(), metav1.ListOptions{ + FieldSelector: fieldSelector, + }) + if err != nil { + klog.Errorf("Failed to get events for %s %s: %s", kind, name, err.Error()) + return + } + + objectEvents := make([]string, 0, len(events.Items)) + for _, event := range events.Items { + objectEvents = append(objectEvents, fmt.Sprintf("%s %s (%d): %s", event.Reason, event.LastTimestamp, event.Count, event.Message)) + } + formattedEvents := fmt.Sprintf(">>>>>>>>>> %s events >>>>>>>>>>\n%s\n<<<<<<<<<< %s events <<<<<<<<<<", kind, strings.Join(objectEvents, "\n"), kind) + klog.V(1).Infof("%s %q events: \n%s", kind, name, formattedEvents) +} + +func printPodsStatuses(pods []corev1.Pod) { + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tSTATUS\tRESTARTS\tAGE") + for _, pod := range pods { + var restartCount int32 + if len(pod.Status.ContainerStatuses) > 0 { + restartCount = pod.Status.ContainerStatuses[0].RestartCount + } + age := time.Since(pod.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%d\t%s\n", + pod.Name, + pod.Status.Phase, + restartCount, + age) + } + writer.Flush() +} + +func printDeploymentsStatuses(clientset kubernetes.Interface, namespace string) { + deploymentsClient := clientset.AppsV1().Deployments(namespace) + deployments, err := deploymentsClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + panic(err.Error()) + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tREADY\tUP-TO-DATE\tAVAILABLE\tAGE") + for _, deployment := range deployments.Items { + ready := fmt.Sprintf("%d/%d", deployment.Status.ReadyReplicas, *deployment.Spec.Replicas) + age := time.Since(deployment.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%d\t%d\t%s\n", + deployment.Name, + ready, + deployment.Status.UpdatedReplicas, + deployment.Status.AvailableReplicas, + age) + } + writer.Flush() +} + +func printStatefulSetsStatuses(clientset kubernetes.Interface, namespace string) { + statefulSetsClient := clientset.AppsV1().StatefulSets(namespace) + statefulSets, err := statefulSetsClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + panic(err.Error()) + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tREADY\tAGE") + for _, statefulSet := range statefulSets.Items { + ready := fmt.Sprintf("%d/%d", statefulSet.Status.ReadyReplicas, *statefulSet.Spec.Replicas) + age := time.Since(statefulSet.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%s\n", + statefulSet.Name, + ready, + age) + } + writer.Flush() +} + +func printDaemonSetsStatuses(clientset kubernetes.Interface, namespace string) { + daemonSetsClient := clientset.AppsV1().DaemonSets(namespace) + daemonSets, err := daemonSetsClient.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + panic(err.Error()) + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tDESIRED\tCURRENT\tREADY\tAGE") + for _, daemonSet := range daemonSets.Items { + age := time.Since(daemonSet.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%d\t%d\t%d\t%s\n", + daemonSet.Name, + daemonSet.Status.DesiredNumberScheduled, + daemonSet.Status.CurrentNumberScheduled, + daemonSet.Status.NumberReady, + age) + } + writer.Flush() +} + +func printConfigMapsInNamespace(client kubernetes.Interface, ns string) { + configMaps, err := client.CoreV1().ConfigMaps(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get configmaps in namespace %q: %v", ns, err) + return + } + + if len(configMaps.Items) == 0 { + klog.V(1).Infof("No configmaps found in namespace %q", ns) + return + } + + klog.V(1).Infof("ConfigMaps in namespace %s: \n", ns) + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tDATA\tAGE") + for _, configMap := range configMaps.Items { + age := time.Since(configMap.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%d\t%s\n", + configMap.Name, + len(configMap.Data), + age) + } + writer.Flush() +} + +func printSecretsInNamespace(client kubernetes.Interface, ns string) { + secrets, err := client.CoreV1().Secrets(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get secrets in namespace %q: %v", ns, err) + return + } + + if len(secrets.Items) == 0 { + klog.V(1).Infof("No secrets found in namespace %q", ns) + return + } + + writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintln(writer, "NAME\tTYPE\tDATA\tAGE") + for _, secret := range secrets.Items { + age := time.Since(secret.CreationTimestamp.Time).Round(time.Second) + fmt.Fprintf(writer, "%s\t%s\t%d\t%s\n", + secret.Name, + secret.Type, + len(secret.Data), + age) + } + writer.Flush() +} diff --git a/tests/pkg/utils/mco_configmaps.go b/tests/pkg/utils/mco_configmaps.go index cb47c2b9e..5ec376848 100644 --- a/tests/pkg/utils/mco_configmaps.go +++ b/tests/pkg/utils/mco_configmaps.go @@ -8,36 +8,10 @@ import ( "context" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" ) -func CreateConfigMap(opt TestOptions, isHub bool, cm *corev1.ConfigMap) error { - clientKube := getKubeClient(opt, isHub) - found, err := clientKube.CoreV1(). - ConfigMaps(cm.ObjectMeta.Namespace). - Get(context.TODO(), cm.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - _, err := clientKube.CoreV1(). - ConfigMaps(cm.ObjectMeta.Namespace). - Create(context.TODO(), cm, metav1.CreateOptions{}) - if err == nil { - klog.V(1).Infof("configmap %s created", cm.ObjectMeta.Name) - } - return err - } - if err != nil { - return err - } - cm.ObjectMeta.ResourceVersion = found.ObjectMeta.ResourceVersion - _, err = clientKube.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) - if err == nil { - klog.V(1).Infof("configmap %s updated", cm.ObjectMeta.Name) - } - return err -} - func GetConfigMap(opt TestOptions, isHub bool, name string, namespace string) (error, *corev1.ConfigMap) { clientKube := getKubeClient(opt, isHub) diff --git a/tests/pkg/utils/mco_deploy.go b/tests/pkg/utils/mco_deploy.go index fdc4719cb..0b3ddd26f 100644 --- a/tests/pkg/utils/mco_deploy.go +++ b/tests/pkg/utils/mco_deploy.go @@ -20,7 +20,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" - "k8s.io/client-go/kubernetes" + "k8s.io/client-go/dynamic" "k8s.io/klog" ) @@ -101,26 +101,6 @@ func NewOCMMultiClusterHubGVR() schema.GroupVersionResource { Resource: "multiclusterhubs"} } -func ModifyMCOAvailabilityConfig(opt TestOptions, availabilityConfig string) error { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - return getErr - } - - spec := mco.Object["spec"].(map[string]interface{}) - spec["availabilityConfig"] = availabilityConfig - _, updateErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Update(context.TODO(), mco, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr - } - return nil -} - func GetAllMCOPods(opt TestOptions) ([]corev1.Pod, error) { hubClient := NewKubeClient( opt.HubCluster.ClusterServerURL, @@ -149,183 +129,32 @@ func GetAllMCOPods(opt TestOptions) ([]corev1.Pod, error) { return mcoPods, nil } -func PrintAllMCOPodsStatus(opt TestOptions) { - podList, err := GetAllMCOPods(opt) - if err != nil { - klog.Errorf("Failed to get all MCO pods") - } - - if len(podList) == 0 { - klog.V(1).Infof("Failed to get pod in %q namespace", MCO_NAMESPACE) - } - - hubClient := NewKubeClient( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - - // Print mch-image-manifest configmap - mchImageManifestCM, err := ReadImageManifestConfigMap(hubClient) - if err != nil { - klog.Errorf("Failed to get mch-image-manifest configmap: %s", err.Error()) - } else { - klog.V(1).Infof("mch-image-manifest configmap: %v", mchImageManifestCM) - } - - LogPodsDebugInfo(hubClient, podList, false) -} - -func LogPodsDebugInfo(hubClient kubernetes.Interface, pods []corev1.Pod, force bool) { - if len(pods) == 0 { +func PrintObject(ctx context.Context, client dynamic.Interface, gvr schema.GroupVersionResource, ns, name string) { + if ns == "" || name == "" { + klog.V(1).Info("Namespace or name cannot be empty") return } - ns := pods[0].Namespace - podsNames := make([]string, 0, len(pods)) - for _, pod := range pods { - podsNames = append(podsNames, pod.Name) - } - - klog.V(1).Infof("Checking pods %v in namespace %q", podsNames, ns) - notRunningPodsCount := 0 - for _, pod := range pods { - if pod.Status.Phase != corev1.PodRunning { - notRunningPodsCount++ - } - - if pod.Status.Phase == corev1.PodRunning && !force { - continue - } - - klog.V(1).Infof("Pod %q is in phase %q and status: %s\n", - pod.Name, - pod.Status.Phase, - pod.Status.String()) - - // print pod events - events, err := hubClient.CoreV1().Events(ns).List(context.TODO(), metav1.ListOptions{ - FieldSelector: "involvedObject.name=" + pod.Name, - }) - if err != nil { - klog.Errorf("Failed to get events for pod %s: %s", pod.Name, err.Error()) - } - - podEvents := make([]string, 0, len(events.Items)) - for _, event := range events.Items { - podEvents = append(podEvents, fmt.Sprintf("%s %s (%d): %s", event.Reason, event.LastTimestamp, event.Count, event.Message)) - } - formattedEvents := ">>>>>>>>>> pod events >>>>>>>>>>\n" + strings.Join(podEvents, "\n") + "\n<<<<<<<<<< pod events <<<<<<<<<<" - klog.V(1).Infof("Pod %q events: \n%s", pod.Name, formattedEvents) - - // print pod containers logs - for _, container := range pod.Spec.Containers { - logsRes := hubClient.CoreV1().Pods(ns).GetLogs(pod.Name, &corev1.PodLogOptions{ - Container: container.Name, - }).Do(context.Background()) - - if logsRes.Error() != nil { - klog.Errorf("Failed to get logs for pod %q: %s", pod.Name, logsRes.Error()) - continue - } - - logs, err := logsRes.Raw() - if err != nil { - klog.Errorf("Failed to get logs for pod %q container %q: %s", pod.Name, container.Name, err.Error()) - continue - } - - delimitedLogs := fmt.Sprintf(">>>>>>>>>> container logs >>>>>>>>>>\n%s<<<<<<<<<< container logs <<<<<<<<<<", string(logs)) - klog.V(1).Infof("Pod %q container %q logs: \n%s", pod.Name, container.Name, delimitedLogs) - } - } - - if notRunningPodsCount == 0 { - klog.V(1).Infof("All pods are running in namespace %q", ns) - } -} - -// ReadImageManifestConfigMap reads configmap with the label ocm-configmap-type=image-manifest. -func ReadImageManifestConfigMap(c kubernetes.Interface) (map[string]string, error) { - listOpts := metav1.ListOptions{ - LabelSelector: "ocm-configmap-type=image-manifest", - } - - imageCMList, err := c.CoreV1().ConfigMaps("open-cluster-management").List(context.TODO(), listOpts) + obj, err := client.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to list mch-image-manifest configmaps: %w", err) - } - - if len(imageCMList.Items) != 1 { - return nil, fmt.Errorf("found %d mch-image-manifest configmaps, expected 1", len(imageCMList.Items)) - } - - return imageCMList.Items[0].Data, nil -} - -func PrintMCOObject(opt TestOptions) { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - klog.V(1).Infof("Failed to get mco object") + klog.V(1).Infof("Failed to get object %s in namespace %s: %v", name, ns, err) return } - spec, _ := json.MarshalIndent(mco.Object["spec"], "", " ") - status, _ := json.MarshalIndent(mco.Object["status"], "", " ") - klog.V(1).Infof("MCO spec: %+v\n", string(spec)) - klog.V(1).Infof("MCO status: %+v\n", string(status)) -} - -func PrintManagedClusterOBAObject(opt TestOptions) { - clientDynamic := GetKubeClientDynamic(opt, false) - oba, getErr := clientDynamic.Resource(NewMCOAddonGVR()). - Namespace(MCO_ADDON_NAMESPACE). - Get(context.TODO(), "observability-addon", metav1.GetOptions{}) - if getErr != nil { - klog.V(1).Infof("Failed to get oba object from managedcluster") - return - } - - spec, _ := json.MarshalIndent(oba.Object["spec"], "", " ") - status, _ := json.MarshalIndent(oba.Object["status"], "", " ") - klog.V(1).Infof("OBA spec: %+v\n", string(spec)) - klog.V(1).Infof("OBA status: %+v\n", string(status)) -} - -func GetAllOBAPods(opt TestOptions) ([]corev1.Pod, error) { - clientKube := getKubeClient(opt, false) - obaPods, err := clientKube.CoreV1().Pods(MCO_ADDON_NAMESPACE).List(context.TODO(), metav1.ListOptions{}) + spec, err := json.MarshalIndent(obj.Object["spec"], "", " ") if err != nil { - return []corev1.Pod{}, err - } - - return obaPods.Items, nil -} - -func PrintAllOBAPodsStatus(opt TestOptions) { - if GetManagedClusterName(opt) == "local-cluster" { - klog.V(1).Infof("Skip printing OBA pods status for local-cluster") - return - } - podList, err := GetAllOBAPods(opt) - if err != nil { - klog.Errorf("Failed to get all OBA pods: %v", err) + klog.V(1).Infof("Failed to marshal spec for object %s in namespace %s: %v", name, ns, err) return } - klog.V(1).Infof("Get <%v> pods in <%s> namespace from managedcluster", len(podList), MCO_ADDON_NAMESPACE) - if len(podList) == 0 { + status, err := json.MarshalIndent(obj.Object["status"], "", " ") + if err != nil { + klog.V(1).Infof("Failed to marshal status for object %s in namespace %s: %v", name, ns, err) return } - force := false - if len(podList) == 1 { // only the operator is up - force = true - } - LogPodsDebugInfo(getKubeClient(opt, false), podList, force) + klog.V(1).Infof("Object %s/%s/%s spec: %+v\n", ns, gvr.Resource, name, string(spec)) + klog.V(1).Infof("Object %s/%s/%s status: %+v\n", ns, gvr.Resource, name, string(status)) } func CheckAllPodNodeSelector(opt TestOptions, nodeSelector map[string]interface{}) error { @@ -617,43 +446,6 @@ func RevertMCOCRModification(opt TestOptions) error { return nil } -func CheckMCOAddon(opt TestOptions) error { - client := NewKubeClient( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - if len(opt.ManagedClusters) > 0 { - client = NewKubeClient( - opt.ManagedClusters[0].ClusterServerURL, - opt.ManagedClusters[0].KubeConfig, - "") - } - expectedPodNames := []string{ - "endpoint-observability-operator", - "metrics-collector-deployment", - } - podList, err := client.CoreV1().Pods(MCO_ADDON_NAMESPACE).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - podsn := make(map[string]corev1.PodPhase) - for _, pod := range podList.Items { - podsn[pod.Name] = pod.Status.Phase - } - for _, podName := range expectedPodNames { - exist := false - for key, value := range podsn { - if strings.HasPrefix(key, podName) && value == "Running" { - exist = true - } - } - if !exist { - return errors.New(podName + " not found") - } - } - return nil -} - func CheckMCOAddonResources(opt TestOptions) error { client := NewKubeClient( opt.HubCluster.ClusterServerURL, @@ -696,43 +488,6 @@ func CheckMCOAddonResources(opt TestOptions) error { return nil } -func ModifyMCORetentionResolutionRaw(opt TestOptions) error { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - return getErr - } - - spec := mco.Object["spec"].(map[string]interface{}) - advRetentionCon, _ := CheckAdvRetentionConfig(opt) - if advRetentionCon { - retentionConfig := spec["advanced"].(map[string]interface{})["retentionConfig"].(map[string]interface{}) - retentionConfig["retentionResolutionRaw"] = "3d" - } - _, updateErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Update(context.TODO(), mco, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr - } - return nil -} - -func GetMCOAddonSpecMetrics(opt TestOptions) (bool, error) { - clientDynamic := NewKubeClientDynamic( - opt.HubCluster.ClusterServerURL, - opt.KubeConfig, - opt.HubCluster.KubeContext) - mco, getErr := clientDynamic.Resource(NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) - if getErr != nil { - return false, getErr - } - - enable := mco.Object["spec"].(map[string]interface{})["observabilityAddonSpec"].(map[string]interface{})["enableMetrics"].(bool) - return enable, nil -} - func ModifyMCOAddonSpecMetrics(opt TestOptions, enable bool) error { clientDynamic := NewKubeClientDynamic( opt.HubCluster.ClusterServerURL, diff --git a/tests/pkg/utils/mco_metric.go b/tests/pkg/utils/mco_metric.go index 16318114c..ac903ca76 100644 --- a/tests/pkg/utils/mco_metric.go +++ b/tests/pkg/utils/mco_metric.go @@ -8,7 +8,6 @@ import ( "bufio" "context" "crypto/tls" - "errors" "fmt" "io" "net/http" @@ -22,17 +21,54 @@ import ( "k8s.io/klog" ) -func ContainManagedClusterMetric(opt TestOptions, query string, matchedLabels []string) (error, bool) { +type GrafanaResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` // Use interface{} because value can be mixed types + } `json:"result"` + } `json:"data"` +} + +func (r GrafanaResponse) ContainsLabelsSet(labels map[string]string) bool { + ret := false +loop: + for _, result := range r.Data.Result { + for key, val := range labels { + if result.Metric[key] != val { + continue loop + } + } + ret = true + break + } + + return ret +} + +func (r GrafanaResponse) String() string { + var ret strings.Builder + ret.WriteString(fmt.Sprintf("Status: %s\n", r.Status)) + ret.WriteString(fmt.Sprintf("ResultType: %s\n", r.Data.ResultType)) + ret.WriteString("Result:\n") + for _, result := range r.Data.Result { + ret.WriteString(fmt.Sprintf("%v %v\n", result.Metric, result.Value)) + } + return ret.String() +} + +func QueryGrafana(opt TestOptions, query string) (*GrafanaResponse, error) { grafanaConsoleURL := GetGrafanaURL(opt) path := "/api/datasources/proxy/1/api/v1/query?" queryParams := url.PathEscape(fmt.Sprintf("query=%s", query)) - klog.V(5).Infof("request url is: %s\n", grafanaConsoleURL+path+queryParams) req, err := http.NewRequest( "GET", grafanaConsoleURL+path+queryParams, nil) if err != nil { - return err, false + return nil, err } client := &http.Client{} @@ -45,7 +81,7 @@ func ContainManagedClusterMetric(opt TestOptions, query string, matchedLabels [] client = &http.Client{Transport: tr} token, err := FetchBearerToken(opt) if err != nil { - return err, false + return nil, err } if token != "" { req.Header.Set("Authorization", "Bearer "+token) @@ -55,41 +91,29 @@ func ContainManagedClusterMetric(opt TestOptions, query string, matchedLabels [] resp, err := client.Do(req) if err != nil { - return err, false + return nil, err } if resp.StatusCode != http.StatusOK { - klog.Errorf("resp: %+v\n", resp) - klog.Errorf("err: %+v\n", err) - return fmt.Errorf("failed to access managed cluster metrics via grafana console: %s", query), false + return nil, fmt.Errorf("failed to access managed cluster metrics via grafana console, status code: %d", resp.StatusCode) } - metricResult, err := io.ReadAll(resp.Body) - klog.V(5).Infof("metricResult: %s\n", metricResult) + respBody, err := io.ReadAll(resp.Body) if err != nil { - return err, false - } - - if !strings.Contains(string(metricResult), `"status":"success"`) { - return errors.New("failed to find valid status from response"), false + return nil, fmt.Errorf("failed to read response body: %v", err) } - if strings.Contains(string(metricResult), `"result":[]`) { - return errors.New("failed to find metric name from response"), false + metricResult := GrafanaResponse{} + err = yaml.Unmarshal(respBody, &metricResult) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal response body: %v", err) } - contained := true - for _, label := range matchedLabels { - if !strings.Contains(string(metricResult), label) { - contained = false - break - } - } - if !contained { - return errors.New("failed to find metric name from response"), false + if metricResult.Status != "success" { + return &metricResult, fmt.Errorf("failed to get metric from response, status: %s", metricResult.Status) } - return nil, true + return &metricResult, nil } type MetricsAllowlist struct { diff --git a/tests/pkg/utils/mco_pods.go b/tests/pkg/utils/mco_pods.go index d3a761324..91b6d721c 100644 --- a/tests/pkg/utils/mco_pods.go +++ b/tests/pkg/utils/mco_pods.go @@ -37,16 +37,6 @@ func GetPodList(opt TestOptions, isHub bool, namespace string, labelSelector str return nil, podList } -func DeletePod(opt TestOptions, isHub bool, namespace, name string) error { - clientKube := getKubeClient(opt, isHub) - err := clientKube.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) - if err != nil { - klog.Errorf("Failed to delete pod %s in namespace %s due to %v", name, namespace, err) - return err - } - return nil -} - func GetPodLogs( opt TestOptions, isHub bool, diff --git a/tests/pkg/utils/utils.go b/tests/pkg/utils/utils.go index 74748266e..0ad676d58 100644 --- a/tests/pkg/utils/utils.go +++ b/tests/pkg/utils/utils.go @@ -6,7 +6,6 @@ package utils import ( "context" - "encoding/json" "errors" "fmt" "os" @@ -25,9 +24,7 @@ import ( k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -36,26 +33,6 @@ import ( "k8s.io/klog" ) -func NewUnversionedRestClient(url, kubeconfig, ctx string) *rest.RESTClient { - klog.V(5).Infof("Create unversionedRestClient for url %s using kubeconfig path %s\n", url, kubeconfig) - config, err := LoadConfig(url, kubeconfig, ctx) - if err != nil { - panic(err) - } - - oldNegotiatedSerializer := config.NegotiatedSerializer - config.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer() - kubeRESTClient, err := rest.UnversionedRESTClientFor(config) - // restore cfg before leaving - defer func(cfg *rest.Config) { cfg.NegotiatedSerializer = oldNegotiatedSerializer }(config) - - if err != nil { - panic(err) - } - - return kubeRESTClient -} - func NewKubeClient(url, kubeconfig, ctx string) kubernetes.Interface { config, err := LoadConfig(url, kubeconfig, ctx) if err != nil { @@ -99,21 +76,6 @@ func NewKubeClientAPIExtension(url, kubeconfig, ctx string) apiextensionsclients return clientset } -// func NewKubeClientDiscovery(url, kubeconfig, ctx string) *discovery.DiscoveryClient { -// klog.V(5).Infof("Create kubeclient discovery for url %s using kubeconfig path %s\n", url, kubeconfig) -// config, err := LoadConfig(url, kubeconfig, ctx) -// if err != nil { -// panic(err) -// } - -// clientset, err := discovery.NewDiscoveryClientForConfig(config) -// if err != nil { -// panic(err) -// } - -// return clientset -// } - func CreateMCOTestingRBAC(opt TestOptions) error { // create new service account and new clusterrolebinding and bind the serviceaccount to cluster-admin clusterrole // then the bearer token can be retrieved from the secret of created serviceaccount @@ -298,13 +260,14 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { apiVersion = v.(string) } + klog.V(5).Infof("Applying kind %q with name %q in namespace %q", kind, obj.GetName(), obj.GetNamespace()) + clientKube := NewKubeClient(url, kubeconfig, ctx) clientAPIExtension := NewKubeClientAPIExtension(url, kubeconfig, ctx) // now use switch over the type of the object // and match each type-case switch kind { case "CustomResourceDefinition": - klog.V(5).Infof("Install CRD: %s\n", f) obj := &apiextensionsv1.CustomResourceDefinition{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -323,7 +286,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientAPIExtension.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), existingObject, metav1.UpdateOptions{}) } case "Namespace": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.Namespace{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -340,7 +302,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().Namespaces().Update(context.TODO(), existingObject, metav1.UpdateOptions{}) } case "ServiceAccount": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.ServiceAccount{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -359,7 +320,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().ServiceAccounts(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "ClusterRoleBinding": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &rbacv1.ClusterRoleBinding{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -376,7 +336,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.RbacV1().ClusterRoleBindings().Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "Secret": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.Secret{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -393,7 +352,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().Secrets(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "ConfigMap": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.ConfigMap{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -412,7 +370,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().ConfigMaps(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "Service": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.Service{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -432,7 +389,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().Services(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "PersistentVolumeClaim": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.PersistentVolumeClaim{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -452,7 +408,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().PersistentVolumeClaims(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "Deployment": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &appsv1.Deployment{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -471,7 +426,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.AppsV1().Deployments(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "LimitRange": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.LimitRange{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -490,7 +444,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().LimitRanges(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "ResourceQuota": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &corev1.ResourceQuota{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -509,7 +462,6 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { _, err = clientKube.CoreV1().ResourceQuotas(obj.Namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) } case "StorageClass": - klog.V(5).Infof("Install %s: %s\n", kind, f) obj := &storagev1.StorageClass{} err = yaml.Unmarshal([]byte(f), obj) if err != nil { @@ -533,13 +485,11 @@ func Apply(url string, kubeconfig string, ctx string, yamlB []byte) error { if apiVersion == "observability.open-cluster-management.io/v1beta1" { gvr = NewMCOGVRV1BETA1() } - klog.V(5).Infof("Install MultiClusterObservability: %s\n", f) case "PrometheusRule": gvr = schema.GroupVersionResource{ Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"} - klog.V(5).Infof("Install PrometheusRule: %s\n", f) default: return fmt.Errorf("resource %s not supported", kind) } @@ -612,45 +562,6 @@ func StatusContainsTypeEqualTo(u *unstructured.Unstructured, typeString string) return false } -// GetCluster returns the first cluster with a given tag -func GetCluster(tag string, clusters []Cluster) *Cluster { - for _, cluster := range clusters { - if tag, ok := cluster.Tags[tag]; ok { - if tag { - return &cluster - } - } - } - return nil -} - -// GetClusters returns all clusters with a given tag -func GetClusters(tag string, clusters []Cluster) []*Cluster { - filteredClusters := make([]*Cluster, 0) - for i, cluster := range clusters { - if tag, ok := cluster.Tags[tag]; ok { - if tag { - filteredClusters = append(filteredClusters, &clusters[i]) - } - } - } - return filteredClusters -} - -func HaveServerResources(c Cluster, kubeconfig string, expectedAPIGroups []string) error { - clientAPIExtension := NewKubeClientAPIExtension(c.ClusterServerURL, kubeconfig, c.KubeContext) - clientDiscovery := clientAPIExtension.Discovery() - for _, apiGroup := range expectedAPIGroups { - klog.V(1).Infof("Check if %s exists", apiGroup) - _, err := clientDiscovery.ServerResourcesForGroupVersion(apiGroup) - if err != nil { - klog.V(1).Infof("Error while retrieving server resource %s: %s", apiGroup, err.Error()) - return err - } - } - return nil -} - func HaveCRDs(c Cluster, kubeconfig string, expectedCRDs []string) error { clientAPIExtension := NewKubeClientAPIExtension(c.ClusterServerURL, kubeconfig, c.KubeContext) clientAPIExtensionV1 := clientAPIExtension.ApiextensionsV1() @@ -665,93 +576,6 @@ func HaveCRDs(c Cluster, kubeconfig string, expectedCRDs []string) error { return nil } -func HaveDeploymentsInNamespace( - c Cluster, - kubeconfig string, - namespace string, - expectedDeploymentNames []string, -) error { - - client := NewKubeClient(c.ClusterServerURL, kubeconfig, c.KubeContext) - versionInfo, err := client.Discovery().ServerVersion() - if err != nil { - return err - } - klog.V(1).Infof("Server version info: %v", versionInfo) - - deployments := client.AppsV1().Deployments(namespace) - - for _, deploymentName := range expectedDeploymentNames { - klog.V(1).Infof("Check if deployment %s exists", deploymentName) - deployment, err := deployments.Get(context.TODO(), deploymentName, metav1.GetOptions{}) - if err != nil { - klog.V(1).Infof("Error while retrieving deployment %s: %s", deploymentName, err.Error()) - return err - } - - if deployment.Status.Replicas != deployment.Status.ReadyReplicas { - err = fmt.Errorf("%s: Expect %d but got %d Ready replicas", - deploymentName, - deployment.Status.Replicas, - deployment.Status.ReadyReplicas) - klog.Errorln(err) - return err - } - - for _, condition := range deployment.Status.Conditions { - if condition.Reason == "MinimumReplicasAvailable" { - if condition.Status != corev1.ConditionTrue { - err = fmt.Errorf("%s: Expect %s but got %s", - deploymentName, - condition.Status, - corev1.ConditionTrue) - klog.Errorln(err) - return err - } - } - } - } - - return nil -} - -func GetKubeVersion(client *rest.RESTClient) version.Info { - kubeVersion := version.Info{} - - versionBody, err := client.Get().AbsPath("/version").Do(context.TODO()).Raw() - if err != nil { - klog.Errorf("fail to GET /version with %v", err) - return version.Info{} - } - - err = json.Unmarshal(versionBody, &kubeVersion) - if err != nil { - klog.Errorf("fail to Unmarshal, got '%s': %v", string(versionBody), err) - return version.Info{} - } - - return kubeVersion -} - -func IsOpenshift(client *rest.RESTClient) bool { - //check whether the cluster is openshift or not for openshift version 3.11 and before - _, err := client.Get().AbsPath("/version/openshift").Do(context.TODO()).Raw() - if err == nil { - klog.V(5).Info("Found openshift version from /version/openshift") - return true - } - - //check whether the cluster is openshift or not for openshift version 4.1 - _, err = client.Get().AbsPath("/apis/config.openshift.io/v1/clusterversions").Do(context.TODO()).Raw() - if err == nil { - klog.V(5).Info("Found openshift version from /apis/config.openshift.io/v1/clusterversions") - return true - } - - klog.V(5).Infof("fail to GET openshift version, assuming not OpenShift: %s", err.Error()) - return false -} - // IntegrityChecking checks to ensure all required conditions are met when completing the specs func IntegrityChecking(opt TestOptions) error { var err error