infrastructure manager metrics (2/n)

.github/workflows/run-vuln-check.yaml

@@ -15,5 +15,5 @@ jobs:
     - name: vulncheck
       uses: golang/govulncheck-action@v1
       with:
-        go-version-input: 1.21.5
+        go-version-input: 1.22.1
         go-package: ./...

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.21.5 as builder
+FROM golang:1.22.1 as builder
 ARG TARGETOS
 ARG TARGETARCH
 

Makefile

@@ -135,7 +135,7 @@ k3d-import-img:
 
 .PHONY: apply-sample-cr
 apply-sample-cr: 
-	kubectl apply -f config/samples/clusterinventory_v1_gardenercluster.yaml
+	kubectl apply -f config/samples/infrastructuremanager_v1_gardenercluster.yaml
 
 .PHONE: local-build-and-deploy
 local-build-and-deploy: docker-build k3d-import-img deploy gardener-secret-deploy apply-sample-cr

cmd/main.go

@@ -26,6 +26,7 @@ import (
 	gardener_apis "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1"
 	infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 	"github.com/kyma-project/infrastructure-manager/internal/controller"
+	"github.com/kyma-project/infrastructure-manager/internal/controller/metrics"
 	"github.com/kyma-project/infrastructure-manager/internal/gardener"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -120,7 +121,8 @@ func main() {
 	}
 
 	rotationPeriod := time.Duration(minimalRotationTimeRatio*expirationTime.Minutes()) * time.Minute
-	if err = (controller.NewGardenerClusterController(mgr, kubeconfigProvider, logger, rotationPeriod)).SetupWithManager(mgr); err != nil {
+	metrics := metrics.NewMetrics()
+	if err = (controller.NewGardenerClusterController(mgr, kubeconfigProvider, logger, rotationPeriod, metrics)).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "GardenerCluster")
 		os.Exit(1)
 	}

internal/controller/gardener_cluster_controller.go

@@ -52,15 +52,17 @@ type GardenerClusterController struct {
 	KubeconfigProvider KubeconfigProvider
 	log                logr.Logger
 	rotationPeriod     time.Duration
+	metrics            metrics.Metrics
 }
 
-func NewGardenerClusterController(mgr ctrl.Manager, kubeconfigProvider KubeconfigProvider, logger logr.Logger, rotationPeriod time.Duration) *GardenerClusterController {
+func NewGardenerClusterController(mgr ctrl.Manager, kubeconfigProvider KubeconfigProvider, logger logr.Logger, rotationPeriod time.Duration, metrics metrics.Metrics) *GardenerClusterController {
 	return &GardenerClusterController{
 		Client:             mgr.GetClient(),
 		Scheme:             mgr.GetScheme(),
 		KubeconfigProvider: kubeconfigProvider,
 		log:                logger,
 		rotationPeriod:     rotationPeriod,
+		metrics:            metrics,
 	}
 }
 
@@ -86,11 +88,12 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req
 	controller.log.Info("Starting reconciliation.", loggingContext(req)...)
 
 	var cluster imv1.GardenerCluster
-	metrics.IncrementReconciliationLoopsStarted()
 
 	err := controller.Get(ctx, req.NamespacedName, &cluster)
+
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
+			controller.unsetStateMetric(req)
 			err = controller.deleteKubeconfigSecret(ctx, req.Name)
 		}
 
@@ -148,6 +151,10 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req
 	return controller.resultWithRequeue(&cluster, requeueAfter), nil
 }
 
+func (controller *GardenerClusterController) unsetStateMetric(req ctrl.Request) {
+	controller.metrics.UnSetGardenerClusterStates(req.NamespacedName.Name)
+}
+
 func loggingContextFromCluster(cluster *imv1.GardenerCluster) []any {
 	return []any{"GardenerCluster", cluster.Name, "Namespace", cluster.Namespace}
 }
@@ -159,7 +166,7 @@ func loggingContext(req ctrl.Request) []any {
 func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.GardenerCluster, requeueAfter time.Duration) ctrl.Result {
 	controller.log.Info("result with requeue", "RequeueAfter", requeueAfter.String())
 
-	metrics.SetGardenerClusterStates(*cluster)
+	controller.metrics.SetGardenerClusterStates(*cluster)
 
 	return ctrl.Result{
 		Requeue:      true,
@@ -169,7 +176,7 @@ func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.Gar
 
 func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { //nolint:unparam
 	controller.log.Info("result without requeue")
-	metrics.SetGardenerClusterStates(*cluster)
+	controller.metrics.SetGardenerClusterStates(*cluster)
 	return ctrl.Result{}
 }
 

internal/controller/gardener_cluster_controller_test.go

@@ -2,6 +2,10 @@ package controller
 
 import (
 	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
 	"time"
 
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
@@ -53,6 +57,10 @@ var _ = Describe("Gardener Cluster controller", func() {
 			lastSyncTime := kubeconfigSecret.Annotations[lastKubeconfigSyncAnnotation]
 			Expect(lastSyncTime).ToNot(BeEmpty())
 
+			By("Metrics should be appended after creation")
+			metricReason, metricState := getMetricsAttributes(kymaName)
+			Expect(metricReason).To(Equal(string(imv1.ConditionReasonKubeconfigSecretCreated)))
+			Expect(metricState).To(Equal(string(imv1.ReadyState)))
 		})
 
 		It("Should delete secret", func() {
@@ -80,8 +88,16 @@ var _ = Describe("Gardener Cluster controller", func() {
 			By("Wait for secret deletion")
 			Eventually(func() bool {
 				err := k8sClient.Get(context.Background(), secretKey, &kubeconfigSecret)
-				return err != nil && k8serrors.IsNotFound(err)
-			}, time.Second*30, time.Second*3).Should(BeTrue())
+				secretNotFound := err != nil && k8serrors.IsNotFound(err)
+				return secretNotFound
+			}, time.Minute*30, time.Second*3).Should(BeTrue())
+
+			By("Metrics should be cleared after deletion")
+			Eventually(func() string {
+				metricReason, _ := getMetricsAttributes(kymaName)
+				return metricReason
+			}, time.Second*30, time.Second*3).Should(BeEmpty())
+
 		})
 
 		It("Should set Error status on CR if failed to fetch kubeconfig", func() {
@@ -103,12 +119,17 @@ var _ = Describe("Gardener Cluster controller", func() {
 
 				return newGardenerCluster.Status.State == imv1.ErrorState
 			}, time.Second*30, time.Second*3).Should(BeTrue())
+
+			By("Metrics should contain error label")
+			metricReason, metricState := getMetricsAttributes(kymaName)
+			Expect(metricReason).To(Equal(string(imv1.ConditionReasonFailedToGetKubeconfig)))
+			Expect(metricState).To(Equal(string(imv1.ErrorState)))
+
 		})
 	})
 
 	Context("Secret with kubeconfig exists", func() {
 		namespace := "default"
-
 		DescribeTable("Should update secret", func(gardenerClusterCR imv1.GardenerCluster, secret corev1.Secret, expectedKubeconfig string) {
 			By("Create kubeconfig secret")
 			Expect(k8sClient.Create(context.Background(), &secret)).To(Succeed())
@@ -152,7 +173,6 @@ var _ = Describe("Gardener Cluster controller", func() {
 			Expect(string(kubeconfigSecret.Data["config"])).To(Equal(expectedKubeconfig))
 			lastSyncTime := kubeconfigSecret.Annotations[lastKubeconfigSyncAnnotation]
 			Expect(lastSyncTime).ToNot(BeEmpty())
-
 		},
 			Entry("Rotate kubeconfig when rotation time passed",
 				fixGardenerClusterCR("kymaname4", namespace, "shootName4", "secret-name4"),
@@ -192,6 +212,47 @@ var _ = Describe("Gardener Cluster controller", func() {
 	})
 })
 
+func getMetricsAttributes(runtimeID string) (outputReason, outputState string) {
+	stringBody, _ := getMetricsBody()
+	clusterStateMetricRegex := getGardenerClusterStateMetricRegex(runtimeID)
+	matches := clusterStateMetricRegex.FindStringSubmatch(stringBody)
+	if len(matches) > 0 {
+		outputReason = matches[1]
+		outputState = matches[2]
+	}
+
+	return outputReason, outputState
+}
+
+// getGardenerClusterStateMetricRegex returns regex that will find matches of gardener_cluster_state metrics
+// and capture two groups for given `runtimeId` label value:
+// 1) `reason` label value
+// 2) `state` label value
+func getGardenerClusterStateMetricRegex(runtimeID string) *regexp.Regexp {
+	regexString := fmt.Sprintf("infrastructure_manager_im_gardener_clusters_state.*reason=\"(.*?)\",runtimeId=\"%v\",state=\"(.*?)\"", runtimeID)
+	return regexp.MustCompile(regexString)
+}
+
+func getMetricsBody() (string, error) {
+	clnt := &http.Client{}
+	request, err := http.NewRequestWithContext(suiteCtx, http.MethodGet, "http://localhost:8080/metrics", nil)
+	if err != nil {
+		return "", fmt.Errorf("request to metrics endpoint :%w", err)
+	}
+	response, err := clnt.Do(request)
+	if err != nil {
+		return "", fmt.Errorf("response from metrics endpoint :%w", err)
+	}
+	defer response.Body.Close()
+	bodyBytes, err := io.ReadAll(response.Body)
+	if err != nil {
+		return "", fmt.Errorf("response body:%w", err)
+	}
+	bodyString := string(bodyBytes)
+
+	return bodyString, nil
+}
+
 func fixNewSecret(name, namespace, kymaName, shootName, data string, lastSyncTime string) corev1.Secret {
 	labels := fixSecretLabels(kymaName, shootName)
 	annotations := map[string]string{lastKubeconfigSyncAnnotation: lastSyncTime}
@@ -246,7 +307,8 @@ func fixSecretLabels(kymaName, shootName string) map[string]string {
 
 func fixGardenerClusterCR(kymaName, namespace, shootName, secretName string) imv1.GardenerCluster {
 	return newTestGardenerClusterCR(kymaName, namespace, shootName, secretName).
-		WithLabels(fixGardenerClusterLabels(kymaName, shootName)).ToCluster()
+		WithLabels(fixGardenerClusterLabels(kymaName, shootName)).
+		ToCluster()
 }
 
 func fixGardenerClusterCRWithForceRotationAnnotation(kymaName, namespace, shootName, secretName string) imv1.GardenerCluster {
@@ -305,7 +367,7 @@ func fixGardenerClusterLabels(kymaName, shootName string) map[string]string {
 	labels := map[string]string{}
 
 	labels["kyma-project.io/instance-id"] = "instanceID"
-	labels["kyma-project.io/runtime-id"] = "runtimeID"
+	labels["kyma-project.io/runtime-id"] = kymaName
 	labels["kyma-project.io/broker-plan-id"] = "planID"
 	labels["kyma-project.io/broker-plan-name"] = "planName"
 	labels["kyma-project.io/global-account-id"] = "globalAccountID"

internal/controller/metrics/metrics.go

@@ -7,36 +7,51 @@ import (
 )
 
 const (
-	shootName = "shootName"
-	state     = "state"
+	runtimeIDKeyName               = "runtimeId"
+	state                          = "state"
+	reason                         = "reason"
+	componentName                  = "infrastructure_manager"
+	RuntimeIDLabel                 = "kyma-project.io/runtime-id"
+	GardenerClusterStateMetricName = "im_gardener_clusters_state"
 )
 
-var (
-
-	//nolint:godox //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11
-	playgroundTotalReconciliationLoopsStarted = prometheus.NewCounter( //nolint:gochecknoglobals
-		prometheus.CounterOpts{
-			Name: "im_playground_reconciliation_loops_started_total",
-			Help: "Number of times reconciliation loop was started",
-		},
-	)
-
-	metricGardenerClustersState = prometheus.NewGaugeVec( //nolint:gochecknoglobals
-		prometheus.GaugeOpts{ //nolint:gochecknoglobals
-			Subsystem: "infrastructure_manager",
-			Name:      "im_gardener_clusters_state",
-			Help:      "Indicates the Status.state for GardenerCluster CRs",
-		}, []string{shootName, state})
-)
+type Metrics struct {
+	gardenerClustersStateGaugeVec *prometheus.GaugeVec
+}
+
+func NewMetrics() Metrics {
+	m := Metrics{
+		gardenerClustersStateGaugeVec: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Subsystem: componentName,
+				Name:      GardenerClusterStateMetricName,
+				Help:      "Indicates the Status.state for GardenerCluster CRs",
+			}, []string{runtimeIDKeyName, state, reason}),
+	}
+	ctrlMetrics.Registry.MustRegister(m.gardenerClustersStateGaugeVec)
+	return m
+}
+
+func (m Metrics) SetGardenerClusterStates(cluster v1.GardenerCluster) {
+	var runtimeID = cluster.GetLabels()[RuntimeIDLabel]
+
+	if runtimeID != "" {
+		if len(cluster.Status.Conditions) != 0 {
+			var reason = cluster.Status.Conditions[0].Reason
 
-func init() {
-	ctrlMetrics.Registry.MustRegister(playgroundTotalReconciliationLoopsStarted, metricGardenerClustersState)
+			// first clean the old metric
+			m.cleanUpGardenerClusterGauge(runtimeID)
+			m.gardenerClustersStateGaugeVec.WithLabelValues(runtimeID, string(cluster.Status.State), reason).Set(1)
+		}
+	}
 }
 
-func IncrementReconciliationLoopsStarted() {
-	playgroundTotalReconciliationLoopsStarted.Inc()
+func (m Metrics) UnSetGardenerClusterStates(runtimeID string) {
+	m.cleanUpGardenerClusterGauge(runtimeID)
 }
 
-func SetGardenerClusterStates(cluster v1.GardenerCluster) {
-	metricGardenerClustersState.WithLabelValues(cluster.Spec.Shoot.Name, string(cluster.Status.State)).Set(1)
+func (m Metrics) cleanUpGardenerClusterGauge(runtimeID string) {
+	m.gardenerClustersStateGaugeVec.DeletePartialMatch(prometheus.Labels{
+		runtimeIDKeyName: runtimeID,
+	})
 }

internal/controller/suite_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	metrics "github.com/kyma-project/infrastructure-manager/internal/controller/metrics"
 	"github.com/kyma-project/infrastructure-manager/internal/controller/mocks"
 	. "github.com/onsi/ginkgo/v2" //nolint:revive
 	. "github.com/onsi/gomega"    //nolint:revive
@@ -81,8 +82,9 @@ var _ = BeforeSuite(func() {
 
 	kubeconfigProviderMock := &mocks.KubeconfigProvider{}
 	setupKubeconfigProviderMock(kubeconfigProviderMock)
+	metrics := metrics.NewMetrics()
 
-	controller := NewGardenerClusterController(mgr, kubeconfigProviderMock, logger, TestKubeconfigValidityTime)
+	controller := NewGardenerClusterController(mgr, kubeconfigProviderMock, logger, TestKubeconfigValidityTime, metrics)
 	Expect(controller).NotTo(BeNil())
 
 	err = controller.SetupWithManager(mgr)