[WIP-do not review] [issue-464]Create a Prometheus ServiceMonitor object that can capture/collect metrics from deployed SonataFlow instances

Makefile

@@ -235,6 +235,8 @@ CONTROLLER_TOOLS_VERSION ?= v0.9.2
 KIND_VERSION ?= v0.20.0
 KNATIVE_VERSION ?= v1.13.2
 TIMEOUT_SECS ?= 180s
+PROMETHEUS_VERSION ?= v0.70.0
+GRAFANA_VERSION ?= v5.13.0
 
 KNATIVE_SERVING_PREFIX ?= "https://github.com/knative/serving/releases/download/knative-$(KNATIVE_VERSION)"
 KNATIVE_EVENTING_PREFIX ?= "https://github.com/knative/eventing/releases/download/knative-$(KNATIVE_VERSION)"
@@ -365,6 +367,16 @@ deploy-knative: create-cluster
 	kubectl apply -f ./test/testdata/knative_serving_eventing.yaml
 	kubectl wait  --for=condition=Ready=True KnativeServing/knative-serving -n knative-serving --timeout=$(TIMEOUT_SECS)
 	kubectl wait  --for=condition=Ready=True KnativeEventing/knative-eventing -n knative-eventing --timeout=$(TIMEOUT_SECS)
+
+.PHONY: deploy-monitoring
+deploy-monitoring: create-cluster
+	kubectl create -f https://github.com/prometheus-operator/prometheus-operator/releases/download/$(PROMETHEUS_VERSION)/bundle.yaml
+	kubectl wait  --for=condition=Available=True deploy/prometheus-operator -n default --timeout=$(TIMEOUT_SECS)
+	kubectl create -f https://github.com/grafana/grafana-operator/releases/download/$(GRAFANA_VERSION)/kustomize-cluster_scoped.yaml
+	kubectl wait  --for=condition=Available=True deploy/grafana-operator-controller-manager -n grafana --timeout=$(TIMEOUT_SECS)
+	kubectl apply -f ./test/testdata/monitoring.yaml
+	kubectl wait  --for=condition=Available=True prometheus/prometheus -n default --timeout=$(TIMEOUT_SECS)
+	kubectl wait  --for=condition=Available=True deploy/grafana-deployment -n default --timeout=$(TIMEOUT_SECS)
 
 .PHONY: delete-cluster
 delete-cluster: install-kind

api/v1alpha08/sonataflowplatform_types.go

@@ -63,6 +63,10 @@ type SonataFlowPlatformSpec struct {
 	// These properties MAY NOT be propagated to a SonataFlowClusterPlatform since PropertyVarSource can only refer local context sources.
 	// +optional
 	Properties *PropertyPlatformSpec `json:"properties,omitempty"`
+	// MonitoringEnabled indicates whether prometheus service monitors should be created for metrics scraping
+	// +optional
+	// +default: false
+	MonitoringEnabled bool `json:"monitoringEnabled"`
 }
 
 // PlatformEventingSpec specifies the Knative Eventing integration details in the platform.

bundle/manifests/sonataflow-operator.clusterserviceversion.yaml

@@ -487,6 +487,17 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - update
+          - watch
         - apiGroups:
           - sonataflow.org
           resources:

bundle/manifests/sonataflow.org_sonataflowplatforms.yaml

@@ -474,6 +474,10 @@ spec:
                         type: string
                     type: object
                 type: object
+              monitoringEnabled:
+                description: MonitoringEnabled indicates whether prometheus service
+                  monitors should be created for metrics scraping
+                type: boolean
               persistence:
                 description: Persistence defines the platform persistence configuration.
                   When this field is set, the configuration is used as the persistence

config/crd/bases/sonataflow.org_sonataflowplatforms.yaml

@@ -475,6 +475,10 @@ spec:
                         type: string
                     type: object
                 type: object
+              monitoringEnabled:
+                description: MonitoringEnabled indicates whether prometheus service
+                  monitors should be created for metrics scraping
+                type: boolean
               persistence:
                 description: Persistence defines the platform persistence configuration.
                   When this field is set, the configuration is used as the persistence

config/rbac/role.yaml

@@ -5,6 +5,17 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - sonataflow.org
   resources:

controllers/discovery/knative_catalog.go

@@ -23,8 +23,8 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/log"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 

controllers/platform/k8s.go

@@ -25,12 +25,12 @@ import (
 
 	operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/container-builder/client"
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/variables"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/log"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/utils"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 	kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj"
 	"github.com/imdario/mergo"

controllers/platform/services/properties.go

@@ -29,10 +29,10 @@ import (
 	"k8s.io/klog/v2"
 
 	operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08"
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/workflowdef"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 
 	"github.com/magiconair/properties"
 )

controllers/platform/services/services.go

@@ -23,8 +23,8 @@ import (
 	"fmt"
 
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/cfg"
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes"
 	"github.com/imdario/mergo"
 	"github.com/magiconair/properties"

controllers/profiles/common/knative_eventing.go

@@ -18,8 +18,8 @@ import (
 	"context"
 
 	operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08"
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/log"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )

controllers/profiles/common/monitoring.go

@@ -0,0 +1,65 @@
+// Copyright 2024 Apache Software Foundation (ASF)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"context"
+
+	operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/log"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/monitoring"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ MonitoringEventingHandler = &monitoringObjectManager{}
+
+type monitoringObjectManager struct {
+	serviceMonitor ObjectEnsurer
+	*StateSupport
+}
+
+func NewMonitoringEventingHandler(support *StateSupport) MonitoringEventingHandler {
+	return &monitoringObjectManager{
+		serviceMonitor: NewObjectEnsurer(support.C, ServiceMonitorCreator),
+		StateSupport:   support,
+	}
+}
+
+type MonitoringEventingHandler interface {
+	Ensure(ctx context.Context, workflow *operatorapi.SonataFlow) ([]client.Object, error)
+}
+
+func (k monitoringObjectManager) Ensure(ctx context.Context, workflow *operatorapi.SonataFlow) ([]client.Object, error) {
+	var objs []client.Object
+
+	monitoringAvail, err := monitoring.GetPrometheusAvailability(k.Cfg)
+	if err != nil {
+		klog.V(log.I).InfoS("Error checking Prometheus availability: %v", err)
+		return nil, err
+	}
+	if !monitoringAvail {
+		klog.V(log.I).InfoS("Prometheus is not installed")
+	} else {
+		// create serviceMonitor
+		serviceMonitor, _, err := k.serviceMonitor.Ensure(ctx, workflow)
+		if err != nil {
+			return objs, err
+		} else if serviceMonitor != nil {
+			objs = append(objs, serviceMonitor)
+		}
+	}
+	return objs, nil
+}

controllers/profiles/common/mutate_visitors.go

@@ -27,8 +27,8 @@ import (
 
 	operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/discovery"
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/properties"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 	kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj"
 	"github.com/imdario/mergo"

controllers/profiles/common/object_creators.go

@@ -29,6 +29,7 @@ import (
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/workflowdef"
 	servingv1 "knative.dev/serving/pkg/apis/serving/v1"
 
+	prometheus "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	cncfmodel "github.com/serverlessworkflow/sdk-go/v2/model"
 
 	"github.com/imdario/mergo"
@@ -45,13 +46,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08"
-	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/persistence"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/properties"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/variables"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/utils"
+	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/knative"
 	kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/utils/openshift"
 	"github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj"
@@ -64,6 +65,10 @@ const (
 	deploymentKind           = "Deployment"
 	k8sServiceAPIVersion     = "v1"
 	k8sServiceKind           = "Service"
+	k8sServicePortName       = "web"
+	knativeServicePortName   = "user-port"
+	metricsServicePortPath   = "/q/metrics"
+	knativeServiceRouteLabel = "serving.knative.dev/route"
 )
 
 // ObjectCreator is the func that creates the initial reference object, if the object doesn't exist in the cluster, this one is created.
@@ -252,23 +257,42 @@ func defaultContainer(workflow *operatorapi.SonataFlow, plf *operatorapi.SonataF
 // It maps the default HTTP port (80) to the target Java application webserver on port 8080.
 func ServiceCreator(workflow *operatorapi.SonataFlow) (client.Object, error) {
 	lbl := workflowproj.GetMergedLabels(workflow)
-
-	service := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      workflow.Name,
-			Namespace: workflow.Namespace,
-			Labels:    lbl,
-		},
-		Spec: corev1.ServiceSpec{
-			Selector: lbl,
-			Ports: []corev1.ServicePort{{
-				Protocol:   corev1.ProtocolTCP,
-				Port:       defaultHTTPServicePort,
-				TargetPort: variables.DefaultHTTPWorkflowPortIntStr,
-			}},
-		},
+	var service *corev1.Service
+	if workflow.IsKnativeDeployment() {
+		service = &corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-metrics", workflow.Name), // default service name is already used by knaitve service
+				Namespace: workflow.Namespace,
+				Labels:    lbl,
+			},
+			Spec: corev1.ServiceSpec{
+				Selector: lbl,
+				Ports: []corev1.ServicePort{{
+					Name:       knativeServicePortName,
+					Protocol:   corev1.ProtocolTCP,
+					Port:       defaultHTTPServicePort,
+					TargetPort: variables.DefaultHTTPWorkflowPortIntStr,
+				}},
+			},
+		}
+	} else {
+		service = &corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      workflow.Name,
+				Namespace: workflow.Namespace,
+				Labels:    lbl,
+			},
+			Spec: corev1.ServiceSpec{
+				Selector: lbl,
+				Ports: []corev1.ServicePort{{
+					Name:       k8sServicePortName,
+					Protocol:   corev1.ProtocolTCP,
+					Port:       defaultHTTPServicePort,
+					TargetPort: variables.DefaultHTTPWorkflowPortIntStr,
+				}},
+			},
+		}
 	}
-
 	return service, nil
 }
 
@@ -439,10 +463,55 @@ func UserPropsConfigMapCreator(workflow *operatorapi.SonataFlow) (client.Object,
 
 // ManagedPropsConfigMapCreator creates an empty ConfigMap to hold the external application properties
 func ManagedPropsConfigMapCreator(workflow *operatorapi.SonataFlow, platform *operatorapi.SonataFlowPlatform) (client.Object, error) {
-
 	props, err := properties.ApplicationManagedProperties(workflow, platform)
 	if err != nil {
 		return nil, err
 	}
 	return workflowproj.CreateNewManagedPropsConfigMap(workflow, props), nil
 }
+
+// ServiceMonitorCreator is an ObjectsCreator for Service Monitor for the workflow service.
+func ServiceMonitorCreator(workflow *operatorapi.SonataFlow) (client.Object, error) {
+	lbl := workflowproj.GetMergedLabels(workflow)
+	var spec *prometheus.ServiceMonitorSpec
+	if workflow.IsKnativeDeployment() {
+		spec = &prometheus.ServiceMonitorSpec{
+			Selector: metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					workflowproj.LabelWorkflow:          workflow.Name,
+					workflowproj.LabelWorkflowNamespace: workflow.Namespace,
+				},
+			},
+			Endpoints: []prometheus.Endpoint{
+				{
+					Port: knativeServicePortName,
+					Path: metricsServicePortPath,
+				},
+			},
+		}
+	} else {
+		spec = &prometheus.ServiceMonitorSpec{
+			Selector: metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					workflowproj.LabelWorkflow:          workflow.Name,
+					workflowproj.LabelWorkflowNamespace: workflow.Namespace,
+				},
+			},
+			Endpoints: []prometheus.Endpoint{
+				{
+					Port: k8sServicePortName,
+					Path: metricsServicePortPath,
+				},
+			},
+		}
+	}
+	serviceMonitor := &prometheus.ServiceMonitor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      workflow.Name,
+			Namespace: workflow.Namespace,
+			Labels:    lbl,
+		},
+		Spec: *spec,
+	}
+	return serviceMonitor, nil
+}