[chore] improve metrics for integration (apache#5154)

* [chore] improve metrics for integration

improve metrics for integrations, record a vector with integration_name and state

* [chore] add tests

* chore: change test dependency

* solve go.mod conflict

* chore: change test library and fix lint

pkg/controller/integration/integration_controller.go

@@ -20,6 +20,7 @@ package integration
 import (
 	"context"
 	"fmt"
+
 	"reflect"
 	"time"
 
@@ -95,6 +96,7 @@ func integrationUpdateFunc(old *v1.Integration, it *v1.Integration) bool {
 		timeToFirstReadiness.Observe(duration.Seconds())
 	}
 
+	updateIntegrationPhase(it.Name, string(it.Status.Phase))
 	// If traits have changed, the reconciliation loop must kick in as
 	// traits may have impact
 	sameTraits, err := trait.IntegrationsHaveSameTraits(old, it)

pkg/controller/integration/metrics.go

@@ -18,28 +18,53 @@ limitations under the License.
 package integration
 
 import (
+	"strings"
 	"time"
 
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-var timeToFirstReadiness = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Name: "camel_k_integration_first_readiness_seconds",
-		Help: "Camel K integration time to first readiness",
-		Buckets: []float64{
-			5 * time.Second.Seconds(),
-			10 * time.Second.Seconds(),
-			30 * time.Second.Seconds(),
-			1 * time.Minute.Seconds(),
-			2 * time.Minute.Seconds(),
+var (
+	timeToFirstReadiness = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name: "camel_k_integration_first_readiness_seconds",
+			Help: "Camel K integration time to first readiness",
+			Buckets: []float64{
+				5 * time.Second.Seconds(),
+				10 * time.Second.Seconds(),
+				30 * time.Second.Seconds(),
+				1 * time.Minute.Seconds(),
+				2 * time.Minute.Seconds(),
+			},
 		},
-	},
+	)
+
+	integration = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "camel_k_integration_phase_total",
+			Help: "Number of integration processed",
+		}, []string{
+			"phase",
+			"id",
+		},
+	)
 )
 
 func init() {
 	// Register custom metrics with the global prometheus registry
-	metrics.Registry.MustRegister(timeToFirstReadiness)
+	metrics.Registry.MustRegister(timeToFirstReadiness, integration)
+}
+
+func updateIntegrationPhase(iID string, p string) {
+	phase := strings.ReplaceAll(strings.ToLower(p), " ", "_")
+
+	if phase != "" && iID != "" {
+		labels := prometheus.Labels{
+			"id":    iID,
+			"phase": phase,
+		}
+		integration.With(labels).Inc()
+	}
 }

pkg/controller/integration/metrics_test.go

@@ -0,0 +1,82 @@
+package integration
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+// getMetricValue returns the sum of the Counter metrics associated with the Collector
+// e.g. the metric for a non-vector, or the sum of the metrics for vector labels.
+// If the metric is a Histogram then number of samples is used.
+func getMetricValue(col prometheus.Collector) float64 {
+	var total float64
+	collect(col, func(m *dto.Metric) {
+		if h := m.GetHistogram(); h != nil {
+			total += float64(h.GetSampleCount())
+		} else {
+			total += m.GetCounter().GetValue()
+		}
+	})
+	return total
+}
+
+// collect calls the function for each metric associated with the Collector
+func collect(col prometheus.Collector, do func(*dto.Metric)) {
+	c := make(chan prometheus.Metric)
+	go func(c chan prometheus.Metric) {
+		col.Collect(c)
+		close(c)
+	}(c)
+	for x := range c { // eg range across distinct label vector values
+		m := dto.Metric{}
+		_ = x.Write(&m)
+		do(&m)
+	}
+}
+
+func Test_updateIntegrationPhase(t *testing.T) {
+	type args struct {
+		iId      string
+		p        string
+		expected float64
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "test should fail with empty id", args: args{
+				iId:      "",
+				p:        "running",
+				expected: 0,
+			},
+		},
+		{
+			name: "test should fail with empty phase", args: args{
+				iId:      "int-1",
+				p:        "",
+				expected: 0,
+			},
+		},
+		{
+			name: "test should pass and increase the counter", args: args{
+				iId:      "int-1",
+				p:        "running",
+				expected: 1,
+			},
+		},
+	}
+	for _, tt := range tests {
+		//		integration.Reset()
+		t.Run(tt.name, func(t *testing.T) {
+			updateIntegrationPhase(tt.args.iId, tt.args.p)
+			labels := map[string]string{"phase": tt.args.p, "id": tt.args.iId}
+			if i, err := integration.GetMetricWith(labels); err == nil {
+				val := getMetricValue(i)
+				assert.Equal(t, val, tt.args.expected)
+			}
+		})
+	}
+}