First drop: Hub metrics collector feature (#1314)

operators/endpointmetrics/pkg/util/client.go

@@ -28,11 +28,8 @@ var (
 )
 
 var (
-	log = ctrl.Log.WithName("util")
-)
-
-const (
-	hubKubeConfigPath = "/spoke/hub-kubeconfig/kubeconfig"
+	log               = ctrl.Log.WithName("util")
+	hubKubeConfigPath = os.Getenv("HUB_KUBECONFIG")
 )
 
 // GetOrCreateOCPClient get an existing hub client or create new one if it doesn't exist.
@@ -45,6 +42,7 @@ func GetOrCreateHubClient(renew bool) (client.Client, error) {
 		return hubClient, nil
 	}
 	// create the config from the path
+	log.Info("Coleen: hubKubeConfigPath", "hubKubeConfigPath", hubKubeConfigPath)
 	config, err := clientcmd.BuildConfigFromFlags("", hubKubeConfigPath)
 	if err != nil {
 		log.Error(err, "Failed to create the config")

operators/multiclusterobservability/controllers/placementrule/manifestwork.go

@@ -16,6 +16,7 @@ import (
 	"gopkg.in/yaml.v2"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -199,7 +200,8 @@ func createManifestwork(c client.Client, work *workv1.ManifestWork) error {
 // generateGlobalManifestResources generates global resources, eg. manifestwork,
 // endpoint-metrics-operator deploy and hubInfo Secret...
 // this function is expensive and should not be called for each reconcile loop.
-func generateGlobalManifestResources(c client.Client, mco *mcov1beta2.MultiClusterObservability) (
+func generateGlobalManifestResources(c client.Client, mco *mcov1beta2.MultiClusterObservability,
+) (
 	[]workv1.Manifest, *workv1.Manifest, *workv1.Manifest, error) {
 
 	works := []workv1.Manifest{}
@@ -384,6 +386,16 @@ func createManifestWorks(
 
 	log.Info(fmt.Sprintf("Cluster: %+v, Spec.NodeSelector (after): %+v", clusterName, spec.NodeSelector))
 	log.Info(fmt.Sprintf("Cluster: %+v, Spec.Tolerations (after): %+v", clusterName, spec.Tolerations))
+	if clusterName != clusterNamespace {
+		spec.Volumes = []corev1.Volume{}
+		spec.Containers[0].VolumeMounts = []corev1.VolumeMount{}
+		for i, env := range spec.Containers[0].Env {
+			if env.Name == "HUB_KUBECONFIG" {
+				spec.Containers[0].Env[i].Value = ""
+				break
+			}
+		}
+	}
 	dep.Spec.Template.Spec = spec
 	manifests = injectIntoWork(manifests, dep)
 	// replace the pull secret and addon components image
@@ -423,10 +435,36 @@ func createManifestWorks(
 
 	work.Spec.Workload.Manifests = manifests
 
-	err = createManifestwork(c, work)
+	if clusterName != clusterNamespace {
+		// install the endpoint operator into open-cluster-management-observability namespace
+		createUpdateResources(c, manifests)
+	} else {
+		err = createManifestwork(c, work)
+	}
+
 	return err
 }
 
+func createUpdateResources(c client.Client, manifests []workv1.Manifest) error {
+	for _, manifest := range manifests {
+		obj := manifest.RawExtension.Object.(client.Object)
+		if obj.GetObjectKind().GroupVersionKind().Kind == "ObservabilityAddon" {
+			continue
+		}
+		obj.SetNamespace(config.GetDefaultNamespace())
+		if obj.GetObjectKind().GroupVersionKind().Kind == "ClusterRoleBinding" {
+			role := obj.(*rbacv1.ClusterRoleBinding)
+			role.Subjects[0].Namespace = config.GetDefaultNamespace()
+		}
+		err := c.Create(context.TODO(), obj)
+		if err != nil && !k8serrors.IsAlreadyExists(err) {
+			log.Error(err, "Failed to create resource", "kind", obj.GetObjectKind().GroupVersionKind().Kind)
+		}
+	}
+
+	return nil
+}
+
 // generateAmAccessorTokenSecret generates the secret that contains the access_token
 // for the Alertmanager in the Hub cluster
 func generateAmAccessorTokenSecret(cl client.Client) (*corev1.Secret, error) {

operators/multiclusterobservability/controllers/placementrule/placementrule_controller.go

@@ -65,6 +65,7 @@ var (
 	isplacementControllerRunnning = false
 	managedClusterList            = map[string]string{}
 	managedClusterListMutex       = &sync.RWMutex{}
+	installMetricsWithoutAddon    = false
 )
 
 // PlacementRuleReconciler reconciles a PlacementRule object
@@ -93,6 +94,21 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	reqLogger := log.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name)
 	reqLogger.Info("Reconciling PlacementRule")
 
+	// TODO
+	if _, ok := managedClusterList["local-cluster"]; !ok {
+		obj := &clusterv1.ManagedCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "local-cluster",
+				Namespace: "open-cluster-management-observability",
+				Labels: map[string]string{
+					"openshiftVersion": "mimical",
+				},
+			},
+		}
+		installMetricsWithoutAddon = true
+		updateManagedClusterList(obj)
+	}
+
 	if config.GetMonitoringCRName() == "" {
 		reqLogger.Info("multicluster observability resource is not available")
 		return ctrl.Result{}, nil
@@ -120,14 +136,22 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, nil
 	}
 
+	//Check for MulticlusterGlobalHub CRD
+	mcghCrdExists := r.CRDMap[config.MCGHCrdName]
+	log.Info("Coleen MulticlusterGlobalHub CRD exists", "exists", mcghCrdExists)
+	//if Multicluster Global hub exists we block metrics-collector creation in spokes
+	if mcghCrdExists {
+		mco.Spec.ObservabilityAddonSpec.EnableMetrics = false
+	}
+
 	if !deleteAll && !mco.Spec.ObservabilityAddonSpec.EnableMetrics {
 		reqLogger.Info("EnableMetrics is set to false. Delete Observability addons")
 		deleteAll = true
 	}
 
 	// check if the MCH CRD exists
 	mchCrdExists := r.CRDMap[config.MCHCrdName]
-	// requeue after 10 seconds if the mch crd exists and image image manifests map is empty
+	// requeue after 10 seconds if the mch crd exists and image  manifests map is empty
 	if mchCrdExists && len(config.GetImageManifests()) == 0 {
 		// if the mch CR is not ready, then requeue the request after 10s
 		return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
@@ -158,6 +182,14 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		reqLogger.Error(err, "Failed to list observabilityaddon resource")
 		return ctrl.Result{}, err
 	}
+	if installMetricsWithoutAddon {
+		obsAddonList.Items = append(obsAddonList.Items, mcov1beta1.ObservabilityAddon{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      localClusterName,
+				Namespace: config.GetDefaultNamespace(),
+			},
+		})
+	}
 
 	if !deleteAll {
 		if err := createAllRelatedRes(
@@ -181,6 +213,14 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		reqLogger.Error(err, "Failed to list observabilityaddon resource")
 		return ctrl.Result{}, err
 	}
+	if installMetricsWithoutAddon {
+		obsAddonList.Items = append(obsAddonList.Items, mcov1beta1.ObservabilityAddon{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "local-cluster",
+				Namespace: "local-cluster",
+			},
+		})
+	}
 	workList := &workv1.ManifestWorkList{}
 	err = r.Client.List(context.TODO(), workList, opts)
 	if err != nil {
@@ -358,6 +398,10 @@ func createAllRelatedRes(
 				err = createManagedClusterRes(c, mco,
 					managedCluster, managedCluster,
 					works, metricsAllowlistConfigMap, crdv1Work, endpointMetricsOperatorDeploy, hubInfoSecret, true)
+			} else if openshiftVersion == "mimical" {
+				err = createManagedClusterRes(c, mco,
+					managedCluster, config.GetDefaultNamespace(),
+					works, metricsAllowlistConfigMap, crdv1Work, endpointMetricsOperatorDeploy, hubInfoSecret, false)
 			} else {
 				err = createManagedClusterRes(c, mco,
 					managedCluster, managedCluster,
@@ -377,10 +421,12 @@ func createAllRelatedRes(
 	failedDeleteOba := false
 	for _, cluster := range currentClusters {
 		log.Info("To delete observabilityAddon", "namespace", cluster)
-		err = deleteObsAddon(c, cluster)
-		if err != nil {
-			failedDeleteOba = true
-			log.Error(err, "Failed to delete observabilityaddon", "namespace", cluster)
+		if cluster != config.GetDefaultNamespace() {
+			err = deleteObsAddon(c, cluster)
+			if err != nil {
+				failedDeleteOba = true
+				log.Error(err, "Failed to delete observabilityaddon", "namespace", cluster)
+			}
 		}
 	}
 

operators/multiclusterobservability/controllers/placementrule/predicate_func.go

@@ -17,6 +17,10 @@ func getClusterPreds() predicate.Funcs {
 	createFunc := func(e event.CreateEvent) bool {
 		log.Info("CreateFunc", "managedCluster", e.Object.GetName())
 
+		if e.Object.GetName() == "local-cluster" {
+			delete(managedClusterList, "local-cluster")
+		}
+
 		if isAutomaticAddonInstallationDisabled(e.Object) {
 			return false
 		}

operators/multiclusterobservability/main.go

@@ -7,6 +7,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/cloudflare/cfssl/log"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -260,10 +261,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	mcghCrdExists, err := operatorsutil.CheckCRDExist(crdClient, config.MCGHCrdName)
+	log.Info("Coleen Checking if MCGH CRD exists", "exists", mcghCrdExists)
+	if err != nil {
+		setupLog.Error(err, "Coleen")
+		os.Exit(1)
+	}
+
 	crdMaps := map[string]bool{
 		config.MCHCrdName:                     mchCrdExists,
 		config.StorageVersionMigrationCrdName: svmCrdExists,
 		config.IngressControllerCRD:           ingressCtlCrdExists,
+		config.MCGHCrdName:                    mcghCrdExists,
 	}
 
 	if err = (&mcoctrl.MultiClusterObservabilityReconciler{

operators/multiclusterobservability/pkg/config/config.go

@@ -249,6 +249,7 @@ const (
 	IngressControllerCRD           = "ingresscontrollers.operator.openshift.io"
 	MCHCrdName                     = "multiclusterhubs.operator.open-cluster-management.io"
 	MCOCrdName                     = "multiclusterobservabilities.observability.open-cluster-management.io"
+	MCGHCrdName                    = "multiclusterglobalhubs.operator.open-cluster-management.io"
 	StorageVersionMigrationCrdName = "storageversionmigrations.migration.k8s.io"
 )