diff --git a/.github/workflows/run-vuln-check.yaml b/.github/workflows/run-vuln-check.yaml index af671ffe..df826356 100644 --- a/.github/workflows/run-vuln-check.yaml +++ b/.github/workflows/run-vuln-check.yaml @@ -15,5 +15,5 @@ jobs: - name: vulncheck uses: golang/govulncheck-action@v1 with: - go-version-input: 1.21.5 + go-version-input: 1.22.1 go-package: ./... diff --git a/Dockerfile b/Dockerfile index b22e4ec3..2c987475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.21.5 as builder +FROM golang:1.22.1 as builder ARG TARGETOS ARG TARGETARCH diff --git a/Makefile b/Makefile index 1e6039b4..119c2617 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ k3d-import-img: .PHONY: apply-sample-cr apply-sample-cr: - kubectl apply -f config/samples/clusterinventory_v1_gardenercluster.yaml + kubectl apply -f config/samples/infrastructuremanager_v1_gardenercluster.yaml .PHONE: local-build-and-deploy local-build-and-deploy: docker-build k3d-import-img deploy gardener-secret-deploy apply-sample-cr diff --git a/cmd/main.go b/cmd/main.go index 86cc8ff9..322fa5be 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -26,6 +26,7 @@ import ( gardener_apis "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/internal/controller" + "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "github.com/kyma-project/infrastructure-manager/internal/gardener" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" @@ -120,7 +121,8 @@ func main() { } rotationPeriod := time.Duration(minimalRotationTimeRatio*expirationTime.Minutes()) * time.Minute - if err = (controller.NewGardenerClusterController(mgr, kubeconfigProvider, logger, rotationPeriod)).SetupWithManager(mgr); err != nil { + metrics := metrics.NewMetrics() + if err = (controller.NewGardenerClusterController(mgr, kubeconfigProvider, logger, rotationPeriod, metrics)).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GardenerCluster") os.Exit(1) } diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index b429a7ab..a2ce9894 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -52,15 +52,17 @@ type GardenerClusterController struct { KubeconfigProvider KubeconfigProvider log logr.Logger rotationPeriod time.Duration + metrics metrics.Metrics } -func NewGardenerClusterController(mgr ctrl.Manager, kubeconfigProvider KubeconfigProvider, logger logr.Logger, rotationPeriod time.Duration) *GardenerClusterController { +func NewGardenerClusterController(mgr ctrl.Manager, kubeconfigProvider KubeconfigProvider, logger logr.Logger, rotationPeriod time.Duration, metrics metrics.Metrics) *GardenerClusterController { return &GardenerClusterController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), KubeconfigProvider: kubeconfigProvider, log: logger, rotationPeriod: rotationPeriod, + metrics: metrics, } } @@ -86,11 +88,12 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req controller.log.Info("Starting reconciliation.", loggingContext(req)...) var cluster imv1.GardenerCluster - metrics.IncrementReconciliationLoopsStarted() err := controller.Get(ctx, req.NamespacedName, &cluster) + if err != nil { if k8serrors.IsNotFound(err) { + controller.unsetStateMetric(req) err = controller.deleteKubeconfigSecret(ctx, req.Name) } @@ -148,6 +151,10 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req return controller.resultWithRequeue(&cluster, requeueAfter), nil } +func (controller *GardenerClusterController) unsetStateMetric(req ctrl.Request) { + controller.metrics.CleanUpGardenerClusterGauge(req.NamespacedName.Name) +} + func loggingContextFromCluster(cluster *imv1.GardenerCluster) []any { return []any{"GardenerCluster", cluster.Name, "Namespace", cluster.Namespace} } @@ -159,7 +166,7 @@ func loggingContext(req ctrl.Request) []any { func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.GardenerCluster, requeueAfter time.Duration) ctrl.Result { controller.log.Info("result with requeue", "RequeueAfter", requeueAfter.String()) - metrics.SetGardenerClusterStates(*cluster) + controller.metrics.SetGardenerClusterStates(*cluster) return ctrl.Result{ Requeue: true, @@ -169,7 +176,7 @@ func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.Gar func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { //nolint:unparam controller.log.Info("result without requeue") - metrics.SetGardenerClusterStates(*cluster) + controller.metrics.SetGardenerClusterStates(*cluster) return ctrl.Result{} } diff --git a/internal/controller/gardener_cluster_controller_test.go b/internal/controller/gardener_cluster_controller_test.go index 854fd91a..5abbbd2e 100644 --- a/internal/controller/gardener_cluster_controller_test.go +++ b/internal/controller/gardener_cluster_controller_test.go @@ -2,6 +2,10 @@ package controller import ( "context" + "fmt" + "io" + "net/http" + "regexp" "time" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -53,6 +57,10 @@ var _ = Describe("Gardener Cluster controller", func() { lastSyncTime := kubeconfigSecret.Annotations[lastKubeconfigSyncAnnotation] Expect(lastSyncTime).ToNot(BeEmpty()) + By("Metrics should be appended after creation") + metricReason, metricState := getMetricsAttributes(kymaName) + Expect(metricReason).To(Equal(string(imv1.ConditionReasonKubeconfigSecretCreated))) + Expect(metricState).To(Equal(string(imv1.ReadyState))) }) It("Should delete secret", func() { @@ -80,8 +88,16 @@ var _ = Describe("Gardener Cluster controller", func() { By("Wait for secret deletion") Eventually(func() bool { err := k8sClient.Get(context.Background(), secretKey, &kubeconfigSecret) - return err != nil && k8serrors.IsNotFound(err) - }, time.Second*30, time.Second*3).Should(BeTrue()) + secretNotFound := err != nil && k8serrors.IsNotFound(err) + return secretNotFound + }, time.Minute*30, time.Second*3).Should(BeTrue()) + + By("Metrics should be cleared after deletion") + Eventually(func() string { + metricReason, _ := getMetricsAttributes(kymaName) + return metricReason + }, time.Second*30, time.Second*3).Should(BeEmpty()) + }) It("Should set Error status on CR if failed to fetch kubeconfig", func() { @@ -103,12 +119,17 @@ var _ = Describe("Gardener Cluster controller", func() { return newGardenerCluster.Status.State == imv1.ErrorState }, time.Second*30, time.Second*3).Should(BeTrue()) + + By("Metrics should contain error label") + metricReason, metricState := getMetricsAttributes(kymaName) + Expect(metricReason).To(Equal(string(imv1.ConditionReasonFailedToGetKubeconfig))) + Expect(metricState).To(Equal(string(imv1.ErrorState))) + }) }) Context("Secret with kubeconfig exists", func() { namespace := "default" - DescribeTable("Should update secret", func(gardenerClusterCR imv1.GardenerCluster, secret corev1.Secret, expectedKubeconfig string) { By("Create kubeconfig secret") Expect(k8sClient.Create(context.Background(), &secret)).To(Succeed()) @@ -152,7 +173,6 @@ var _ = Describe("Gardener Cluster controller", func() { Expect(string(kubeconfigSecret.Data["config"])).To(Equal(expectedKubeconfig)) lastSyncTime := kubeconfigSecret.Annotations[lastKubeconfigSyncAnnotation] Expect(lastSyncTime).ToNot(BeEmpty()) - }, Entry("Rotate kubeconfig when rotation time passed", fixGardenerClusterCR("kymaname4", namespace, "shootName4", "secret-name4"), @@ -192,6 +212,47 @@ var _ = Describe("Gardener Cluster controller", func() { }) }) +func getMetricsAttributes(runtimeID string) (outputReason, outputState string) { + stringBody, _ := getMetricsBody() + clusterStateMetricRegex := getGardenerClusterStateMetricRegex(runtimeID) + matches := clusterStateMetricRegex.FindStringSubmatch(stringBody) + if len(matches) > 0 { + outputReason = matches[1] + outputState = matches[2] + } + + return outputReason, outputState +} + +// getGardenerClusterStateMetricRegex returns regex that will find matches of gardener_cluster_state metrics +// and capture two groups for given `runtimeId` label value: +// 1) `reason` label value +// 2) `state` label value +func getGardenerClusterStateMetricRegex(runtimeID string) *regexp.Regexp { + regexString := fmt.Sprintf("infrastructure_manager_im_gardener_clusters_state.*reason=\"(.*?)\",runtimeId=\"%v\",state=\"(.*?)\"", runtimeID) + return regexp.MustCompile(regexString) +} + +func getMetricsBody() (string, error) { + clnt := &http.Client{} + request, err := http.NewRequestWithContext(suiteCtx, http.MethodGet, "http://localhost:8080/metrics", nil) + if err != nil { + return "", fmt.Errorf("request to metrics endpoint :%w", err) + } + response, err := clnt.Do(request) + if err != nil { + return "", fmt.Errorf("response from metrics endpoint :%w", err) + } + defer response.Body.Close() + bodyBytes, err := io.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("response body:%w", err) + } + bodyString := string(bodyBytes) + + return bodyString, nil +} + func fixNewSecret(name, namespace, kymaName, shootName, data string, lastSyncTime string) corev1.Secret { labels := fixSecretLabels(kymaName, shootName) annotations := map[string]string{lastKubeconfigSyncAnnotation: lastSyncTime} @@ -246,7 +307,8 @@ func fixSecretLabels(kymaName, shootName string) map[string]string { func fixGardenerClusterCR(kymaName, namespace, shootName, secretName string) imv1.GardenerCluster { return newTestGardenerClusterCR(kymaName, namespace, shootName, secretName). - WithLabels(fixGardenerClusterLabels(kymaName, shootName)).ToCluster() + WithLabels(fixGardenerClusterLabels(kymaName, shootName)). + ToCluster() } func fixGardenerClusterCRWithForceRotationAnnotation(kymaName, namespace, shootName, secretName string) imv1.GardenerCluster { @@ -305,7 +367,7 @@ func fixGardenerClusterLabels(kymaName, shootName string) map[string]string { labels := map[string]string{} labels["kyma-project.io/instance-id"] = "instanceID" - labels["kyma-project.io/runtime-id"] = "runtimeID" + labels["kyma-project.io/runtime-id"] = kymaName labels["kyma-project.io/broker-plan-id"] = "planID" labels["kyma-project.io/broker-plan-name"] = "planName" labels["kyma-project.io/global-account-id"] = "globalAccountID" diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index c6f7a99a..d1755731 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -7,36 +7,47 @@ import ( ) const ( - shootName = "shootName" - state = "state" + runtimeIDKeyName = "runtimeId" + state = "state" + reason = "reason" + componentName = "infrastructure_manager" + RuntimeIDLabel = "kyma-project.io/runtime-id" + GardenerClusterStateMetricName = "im_gardener_clusters_state" ) -var ( - - //nolint:godox //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 - playgroundTotalReconciliationLoopsStarted = prometheus.NewCounter( //nolint:gochecknoglobals - prometheus.CounterOpts{ - Name: "im_playground_reconciliation_loops_started_total", - Help: "Number of times reconciliation loop was started", - }, - ) - - metricGardenerClustersState = prometheus.NewGaugeVec( //nolint:gochecknoglobals - prometheus.GaugeOpts{ //nolint:gochecknoglobals - Subsystem: "infrastructure_manager", - Name: "im_gardener_clusters_state", - Help: "Indicates the Status.state for GardenerCluster CRs", - }, []string{shootName, state}) -) +type Metrics struct { + gardenerClustersStateGaugeVec *prometheus.GaugeVec +} -func init() { - ctrlMetrics.Registry.MustRegister(playgroundTotalReconciliationLoopsStarted, metricGardenerClustersState) +func NewMetrics() Metrics { + m := Metrics{ + gardenerClustersStateGaugeVec: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: componentName, + Name: GardenerClusterStateMetricName, + Help: "Indicates the Status.state for GardenerCluster CRs", + }, []string{runtimeIDKeyName, state, reason}), + } + ctrlMetrics.Registry.MustRegister(m.gardenerClustersStateGaugeVec) + return m } -func IncrementReconciliationLoopsStarted() { - playgroundTotalReconciliationLoopsStarted.Inc() +func (m Metrics) SetGardenerClusterStates(cluster v1.GardenerCluster) { + var runtimeID = cluster.GetLabels()[RuntimeIDLabel] + + if runtimeID != "" { + if len(cluster.Status.Conditions) != 0 { + var reason = cluster.Status.Conditions[0].Reason + + // first clean the old metric + m.CleanUpGardenerClusterGauge(runtimeID) + m.gardenerClustersStateGaugeVec.WithLabelValues(runtimeID, string(cluster.Status.State), reason).Set(1) + } + } } -func SetGardenerClusterStates(cluster v1.GardenerCluster) { - metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Set(1) +func (m Metrics) CleanUpGardenerClusterGauge(runtimeID string) { + m.gardenerClustersStateGaugeVec.DeletePartialMatch(prometheus.Labels{ + runtimeIDKeyName: runtimeID, + }) } diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 81f8a669..041be3b2 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -23,6 +23,7 @@ import ( "time" infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1" + metrics "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "github.com/kyma-project/infrastructure-manager/internal/controller/mocks" . "github.com/onsi/ginkgo/v2" //nolint:revive . "github.com/onsi/gomega" //nolint:revive @@ -81,8 +82,9 @@ var _ = BeforeSuite(func() { kubeconfigProviderMock := &mocks.KubeconfigProvider{} setupKubeconfigProviderMock(kubeconfigProviderMock) + metrics := metrics.NewMetrics() - controller := NewGardenerClusterController(mgr, kubeconfigProviderMock, logger, TestKubeconfigValidityTime) + controller := NewGardenerClusterController(mgr, kubeconfigProviderMock, logger, TestKubeconfigValidityTime, metrics) Expect(controller).NotTo(BeNil()) err = controller.SetupWithManager(mgr)