diff --git a/cmd/main.go b/cmd/main.go index 86cc8ff9..1edddcec 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "fmt" + "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "os" "time" @@ -120,7 +121,8 @@ func main() { } rotationPeriod := time.Duration(minimalRotationTimeRatio*expirationTime.Minutes()) * time.Minute - if err = (controller.NewGardenerClusterController(mgr, kubeconfigProvider, logger, rotationPeriod)).SetupWithManager(mgr); err != nil { + metrics := metrics.NewMetrics() + if err = (controller.NewGardenerClusterController(mgr, kubeconfigProvider, logger, rotationPeriod, metrics)).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GardenerCluster") os.Exit(1) } diff --git a/internal/controller/gardener_cluster_controller.go b/internal/controller/gardener_cluster_controller.go index 9ffba95e..1135f49d 100644 --- a/internal/controller/gardener_cluster_controller.go +++ b/internal/controller/gardener_cluster_controller.go @@ -52,15 +52,17 @@ type GardenerClusterController struct { KubeconfigProvider KubeconfigProvider log logr.Logger rotationPeriod time.Duration + metrics metrics.Metrics } -func NewGardenerClusterController(mgr ctrl.Manager, kubeconfigProvider KubeconfigProvider, logger logr.Logger, rotationPeriod time.Duration) *GardenerClusterController { +func NewGardenerClusterController(mgr ctrl.Manager, kubeconfigProvider KubeconfigProvider, logger logr.Logger, rotationPeriod time.Duration, metrics metrics.Metrics) *GardenerClusterController { return &GardenerClusterController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), KubeconfigProvider: kubeconfigProvider, log: logger, rotationPeriod: rotationPeriod, + metrics: metrics, } } @@ -86,7 +88,6 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req controller.log.Info("Starting reconciliation.", loggingContext(req)...) var cluster imv1.GardenerCluster - metrics.IncrementReconciliationLoopsStarted() err := controller.Get(ctx, req.NamespacedName, &cluster) @@ -152,12 +153,7 @@ func (controller *GardenerClusterController) Reconcile(ctx context.Context, req } func (controller *GardenerClusterController) unsetStateMetric(ctx context.Context, req ctrl.Request) { - var secretKey = "kubeconfig-" + req.NamespacedName.Name - var secretNamespacedName = types.NamespacedName{Name: secretKey, Namespace: "kcp-system"} - var kubeconfigSecret corev1.Secret - _ = controller.Get(ctx, secretNamespacedName, &kubeconfigSecret) - - metrics.UnSetGardenerClusterStates(kubeconfigSecret) + controller.metrics.UnSetGardenerClusterStates(req.NamespacedName.Name) } func loggingContextFromCluster(cluster *imv1.GardenerCluster) []any { @@ -171,7 +167,7 @@ func loggingContext(req ctrl.Request) []any { func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.GardenerCluster, requeueAfter time.Duration) ctrl.Result { controller.log.Info("result with requeue", "RequeueAfter", requeueAfter.String()) - metrics.SetGardenerClusterStates(*cluster) + controller.metrics.SetGardenerClusterStates(*cluster) return ctrl.Result{ Requeue: true, @@ -181,7 +177,7 @@ func (controller *GardenerClusterController) resultWithRequeue(cluster *imv1.Gar func (controller *GardenerClusterController) resultWithoutRequeue(cluster *imv1.GardenerCluster) ctrl.Result { //nolint:unparam controller.log.Info("result without requeue") - metrics.SetGardenerClusterStates(*cluster) + controller.metrics.SetGardenerClusterStates(*cluster) return ctrl.Result{} } diff --git a/internal/controller/metrics/metrics.go b/internal/controller/metrics/metrics.go index ad997a86..22b4baeb 100644 --- a/internal/controller/metrics/metrics.go +++ b/internal/controller/metrics/metrics.go @@ -4,51 +4,67 @@ import ( "fmt" v1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/prometheus/client_golang/prometheus" - corev1 "k8s.io/api/core/v1" ctrlMetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) const ( - runtimeId = "runtimeId" - state = "state" + runtimeIdKeyName = "runtimeId" + state = "state" + reason = "reason" + runtimeIdLabel = "kyma-project.io/runtime-id" + componentName = "infrastructure_manager" ) -var ( - - //nolint:godox //TODO: test custom metric, remove when done with https://github.com/kyma-project/infrastructure-manager/issues/11 - playgroundTotalReconciliationLoopsStarted = prometheus.NewCounter( //nolint:gochecknoglobals - prometheus.CounterOpts{ - Name: "im_playground_reconciliation_loops_started_total", - Help: "Number of times reconciliation loop was started", - }, - ) - - metricGardenerClustersState = prometheus.NewGaugeVec( //nolint:gochecknoglobals - prometheus.GaugeOpts{ //nolint:gochecknoglobals - Subsystem: "infrastructure_manager", - Name: "im_gardener_clusters_state", - Help: "Indicates the Status.state for GardenerCluster CRs", - }, []string{runtimeId, state}) -) +type Metrics struct { + gardenerClustersStateGaugeVec *prometheus.GaugeVec +} -func init() { - ctrlMetrics.Registry.MustRegister(playgroundTotalReconciliationLoopsStarted, metricGardenerClustersState) +func NewMetrics() Metrics { + m := Metrics{ + gardenerClustersStateGaugeVec: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: componentName, + Name: "im_gardener_clusters_state", + Help: "Indicates the Status.state for GardenerCluster CRs", + }, []string{runtimeIdKeyName, state, reason}), + } + ctrlMetrics.Registry.MustRegister(m.gardenerClustersStateGaugeVec) + return m } -func IncrementReconciliationLoopsStarted() { - playgroundTotalReconciliationLoopsStarted.Inc() +func (m Metrics) SetGardenerClusterStates(cluster v1.GardenerCluster) { + var runtimeId = cluster.GetLabels()[runtimeIdLabel] + + if runtimeId != "" { + var reason = cluster.Status.Conditions[0].Reason + + //first clean the old metric + m.cleanUpGardenerClusterGauge(runtimeId) + m.gardenerClustersStateGaugeVec.WithLabelValues(runtimeId, string(cluster.Status.State), reason).Set(1) + } } -func SetGardenerClusterStates(cluster v1.GardenerCluster) { - metricGardenerClustersState.WithLabelValues(cluster.Name, string(cluster.Status.State)).Set(1) +func (m Metrics) UnSetGardenerClusterStates(runtimeId string) { + m.cleanUpGardenerClusterGauge(runtimeId) } -func UnSetGardenerClusterStates(secret corev1.Secret) { - var runtimeId = secret.GetLabels()["kyma-project.io/runtime-id"] - var deletedReady = metricGardenerClustersState.DeleteLabelValues(runtimeId, "Ready") - var deletedError = metricGardenerClustersState.DeleteLabelValues(runtimeId, "Error") +func (m Metrics) cleanUpGardenerClusterGauge(runtimeId string) { + + var readyMetric, _ = m.gardenerClustersStateGaugeVec.GetMetricWithLabelValues(runtimeId, "Ready") + if readyMetric != nil { + readyMetric.Set(0) + } + var errorMetric, _ = m.gardenerClustersStateGaugeVec.GetMetricWithLabelValues(runtimeId, "Error") + if errorMetric != nil { + errorMetric.Set(0) + } + fmt.Printf("GardenerClusterStates set value to 0 for %v", runtimeId) + + metricsDeleted := m.gardenerClustersStateGaugeVec.DeletePartialMatch(prometheus.Labels{ + runtimeIdKeyName: runtimeId, + }) - if deletedReady || deletedError { - fmt.Printf("GardenerClusterStates deleted value for %v", runtimeId) + if metricsDeleted > 0 { + fmt.Printf("gardenerClusterStateGauge deleted %d metrics for runtimeId %v", metricsDeleted, runtimeId) } } diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index d4c08cf4..1e6044b6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -18,6 +18,7 @@ package controller import ( "context" + metrics "github.com/kyma-project/infrastructure-manager/internal/controller/metrics" "path/filepath" "testing" "time" @@ -81,8 +82,9 @@ var _ = BeforeSuite(func() { kubeconfigProviderMock := &mocks.KubeconfigProvider{} setupKubeconfigProviderMock(kubeconfigProviderMock) + metrics := metrics.NewMetrics() - controller := NewGardenerClusterController(mgr, kubeconfigProviderMock, logger, TestKubeconfigValidityTime) + controller := NewGardenerClusterController(mgr, kubeconfigProviderMock, logger, TestKubeconfigValidityTime, metrics) Expect(controller).NotTo(BeNil()) err = controller.SetupWithManager(mgr)