diff --git a/operators/endpointmetrics/pkg/util/client.go b/operators/endpointmetrics/pkg/util/client.go index 42eb84a23..8c7859321 100644 --- a/operators/endpointmetrics/pkg/util/client.go +++ b/operators/endpointmetrics/pkg/util/client.go @@ -28,11 +28,8 @@ var ( ) var ( - log = ctrl.Log.WithName("util") -) - -const ( - hubKubeConfigPath = "/spoke/hub-kubeconfig/kubeconfig" + log = ctrl.Log.WithName("util") + hubKubeConfigPath = os.Getenv("HUB_KUBECONFIG") ) // GetOrCreateOCPClient get an existing hub client or create new one if it doesn't exist. @@ -45,6 +42,7 @@ func GetOrCreateHubClient(renew bool) (client.Client, error) { return hubClient, nil } // create the config from the path + log.Info("Coleen: hubKubeConfigPath", "hubKubeConfigPath", hubKubeConfigPath) config, err := clientcmd.BuildConfigFromFlags("", hubKubeConfigPath) if err != nil { log.Error(err, "Failed to create the config") diff --git a/operators/multiclusterobservability/controllers/placementrule/manifestwork.go b/operators/multiclusterobservability/controllers/placementrule/manifestwork.go index 8fc6f8de4..366410f6d 100644 --- a/operators/multiclusterobservability/controllers/placementrule/manifestwork.go +++ b/operators/multiclusterobservability/controllers/placementrule/manifestwork.go @@ -16,6 +16,7 @@ import ( "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -199,7 +200,8 @@ func createManifestwork(c client.Client, work *workv1.ManifestWork) error { // generateGlobalManifestResources generates global resources, eg. manifestwork, // endpoint-metrics-operator deploy and hubInfo Secret... // this function is expensive and should not be called for each reconcile loop. -func generateGlobalManifestResources(c client.Client, mco *mcov1beta2.MultiClusterObservability) ( +func generateGlobalManifestResources(c client.Client, mco *mcov1beta2.MultiClusterObservability, +) ( []workv1.Manifest, *workv1.Manifest, *workv1.Manifest, error) { works := []workv1.Manifest{} @@ -384,6 +386,16 @@ func createManifestWorks( log.Info(fmt.Sprintf("Cluster: %+v, Spec.NodeSelector (after): %+v", clusterName, spec.NodeSelector)) log.Info(fmt.Sprintf("Cluster: %+v, Spec.Tolerations (after): %+v", clusterName, spec.Tolerations)) + if clusterName != clusterNamespace { + spec.Volumes = []corev1.Volume{} + spec.Containers[0].VolumeMounts = []corev1.VolumeMount{} + for i, env := range spec.Containers[0].Env { + if env.Name == "HUB_KUBECONFIG" { + spec.Containers[0].Env[i].Value = "" + break + } + } + } dep.Spec.Template.Spec = spec manifests = injectIntoWork(manifests, dep) // replace the pull secret and addon components image @@ -423,10 +435,36 @@ func createManifestWorks( work.Spec.Workload.Manifests = manifests - err = createManifestwork(c, work) + if clusterName != clusterNamespace { + // install the endpoint operator into open-cluster-management-observability namespace + createUpdateResources(c, manifests) + } else { + err = createManifestwork(c, work) + } + return err } +func createUpdateResources(c client.Client, manifests []workv1.Manifest) error { + for _, manifest := range manifests { + obj := manifest.RawExtension.Object.(client.Object) + if obj.GetObjectKind().GroupVersionKind().Kind == "ObservabilityAddon" { + continue + } + obj.SetNamespace(config.GetDefaultNamespace()) + if obj.GetObjectKind().GroupVersionKind().Kind == "ClusterRoleBinding" { + role := obj.(*rbacv1.ClusterRoleBinding) + role.Subjects[0].Namespace = config.GetDefaultNamespace() + } + err := c.Create(context.TODO(), obj) + if err != nil && !k8serrors.IsAlreadyExists(err) { + log.Error(err, "Failed to create resource", "kind", obj.GetObjectKind().GroupVersionKind().Kind) + } + } + + return nil +} + // generateAmAccessorTokenSecret generates the secret that contains the access_token // for the Alertmanager in the Hub cluster func generateAmAccessorTokenSecret(cl client.Client) (*corev1.Secret, error) { diff --git a/operators/multiclusterobservability/controllers/placementrule/placementrule_controller.go b/operators/multiclusterobservability/controllers/placementrule/placementrule_controller.go index 318f015f7..3fe08af5f 100644 --- a/operators/multiclusterobservability/controllers/placementrule/placementrule_controller.go +++ b/operators/multiclusterobservability/controllers/placementrule/placementrule_controller.go @@ -65,6 +65,7 @@ var ( isplacementControllerRunnning = false managedClusterList = map[string]string{} managedClusterListMutex = &sync.RWMutex{} + installMetricsWithoutAddon = false ) // PlacementRuleReconciler reconciles a PlacementRule object @@ -93,6 +94,21 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques reqLogger := log.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name) reqLogger.Info("Reconciling PlacementRule") + // TODO + if _, ok := managedClusterList["local-cluster"]; !ok { + obj := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-cluster", + Namespace: "open-cluster-management-observability", + Labels: map[string]string{ + "openshiftVersion": "mimical", + }, + }, + } + installMetricsWithoutAddon = true + updateManagedClusterList(obj) + } + if config.GetMonitoringCRName() == "" { reqLogger.Info("multicluster observability resource is not available") return ctrl.Result{}, nil @@ -120,6 +136,14 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } + //Check for MulticlusterGlobalHub CRD + mcghCrdExists := r.CRDMap[config.MCGHCrdName] + log.Info("Coleen MulticlusterGlobalHub CRD exists", "exists", mcghCrdExists) + //if Multicluster Global hub exists we block metrics-collector creation in spokes + if mcghCrdExists { + mco.Spec.ObservabilityAddonSpec.EnableMetrics = false + } + if !deleteAll && !mco.Spec.ObservabilityAddonSpec.EnableMetrics { reqLogger.Info("EnableMetrics is set to false. Delete Observability addons") deleteAll = true @@ -127,7 +151,7 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques // check if the MCH CRD exists mchCrdExists := r.CRDMap[config.MCHCrdName] - // requeue after 10 seconds if the mch crd exists and image image manifests map is empty + // requeue after 10 seconds if the mch crd exists and image manifests map is empty if mchCrdExists && len(config.GetImageManifests()) == 0 { // if the mch CR is not ready, then requeue the request after 10s return ctrl.Result{RequeueAfter: 10 * time.Second}, nil @@ -158,6 +182,14 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques reqLogger.Error(err, "Failed to list observabilityaddon resource") return ctrl.Result{}, err } + if installMetricsWithoutAddon { + obsAddonList.Items = append(obsAddonList.Items, mcov1beta1.ObservabilityAddon{ + ObjectMeta: metav1.ObjectMeta{ + Name: localClusterName, + Namespace: config.GetDefaultNamespace(), + }, + }) + } if !deleteAll { if err := createAllRelatedRes( @@ -181,6 +213,14 @@ func (r *PlacementRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reques reqLogger.Error(err, "Failed to list observabilityaddon resource") return ctrl.Result{}, err } + if installMetricsWithoutAddon { + obsAddonList.Items = append(obsAddonList.Items, mcov1beta1.ObservabilityAddon{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-cluster", + Namespace: "local-cluster", + }, + }) + } workList := &workv1.ManifestWorkList{} err = r.Client.List(context.TODO(), workList, opts) if err != nil { @@ -358,6 +398,10 @@ func createAllRelatedRes( err = createManagedClusterRes(c, mco, managedCluster, managedCluster, works, metricsAllowlistConfigMap, crdv1Work, endpointMetricsOperatorDeploy, hubInfoSecret, true) + } else if openshiftVersion == "mimical" { + err = createManagedClusterRes(c, mco, + managedCluster, config.GetDefaultNamespace(), + works, metricsAllowlistConfigMap, crdv1Work, endpointMetricsOperatorDeploy, hubInfoSecret, false) } else { err = createManagedClusterRes(c, mco, managedCluster, managedCluster, @@ -377,10 +421,12 @@ func createAllRelatedRes( failedDeleteOba := false for _, cluster := range currentClusters { log.Info("To delete observabilityAddon", "namespace", cluster) - err = deleteObsAddon(c, cluster) - if err != nil { - failedDeleteOba = true - log.Error(err, "Failed to delete observabilityaddon", "namespace", cluster) + if cluster != config.GetDefaultNamespace() { + err = deleteObsAddon(c, cluster) + if err != nil { + failedDeleteOba = true + log.Error(err, "Failed to delete observabilityaddon", "namespace", cluster) + } } } diff --git a/operators/multiclusterobservability/controllers/placementrule/predicate_func.go b/operators/multiclusterobservability/controllers/placementrule/predicate_func.go index a19af143f..6c5c16398 100644 --- a/operators/multiclusterobservability/controllers/placementrule/predicate_func.go +++ b/operators/multiclusterobservability/controllers/placementrule/predicate_func.go @@ -17,6 +17,10 @@ func getClusterPreds() predicate.Funcs { createFunc := func(e event.CreateEvent) bool { log.Info("CreateFunc", "managedCluster", e.Object.GetName()) + if e.Object.GetName() == "local-cluster" { + delete(managedClusterList, "local-cluster") + } + if isAutomaticAddonInstallationDisabled(e.Object) { return false } diff --git a/operators/multiclusterobservability/main.go b/operators/multiclusterobservability/main.go index 423e89cfd..018511f76 100644 --- a/operators/multiclusterobservability/main.go +++ b/operators/multiclusterobservability/main.go @@ -7,6 +7,7 @@ package main import ( "flag" "fmt" + "github.com/cloudflare/cfssl/log" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -260,10 +261,18 @@ func main() { os.Exit(1) } + mcghCrdExists, err := operatorsutil.CheckCRDExist(crdClient, config.MCGHCrdName) + log.Info("Coleen Checking if MCGH CRD exists", "exists", mcghCrdExists) + if err != nil { + setupLog.Error(err, "Coleen") + os.Exit(1) + } + crdMaps := map[string]bool{ config.MCHCrdName: mchCrdExists, config.StorageVersionMigrationCrdName: svmCrdExists, config.IngressControllerCRD: ingressCtlCrdExists, + config.MCGHCrdName: mcghCrdExists, } if err = (&mcoctrl.MultiClusterObservabilityReconciler{ diff --git a/operators/multiclusterobservability/pkg/config/config.go b/operators/multiclusterobservability/pkg/config/config.go index d1440bc7e..b6be44704 100644 --- a/operators/multiclusterobservability/pkg/config/config.go +++ b/operators/multiclusterobservability/pkg/config/config.go @@ -249,6 +249,7 @@ const ( IngressControllerCRD = "ingresscontrollers.operator.openshift.io" MCHCrdName = "multiclusterhubs.operator.open-cluster-management.io" MCOCrdName = "multiclusterobservabilities.observability.open-cluster-management.io" + MCGHCrdName = "multiclusterglobalhubs.operator.open-cluster-management.io" StorageVersionMigrationCrdName = "storageversionmigrations.migration.k8s.io" )