Merge pull request #477 from furkatgofurov7/bump-capi-1-8

Bump CAPI to v1.8.5

Makefile

@@ -67,7 +67,11 @@ KUSTOMIZE_BIN := kustomize
 KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER))
 KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v4
 
-SETUP_ENVTEST_VER := v0.0.0-20211110210527-619e6b92dab9
+# This is a commit from CR main (22.05.2024).
+# Intentionally using a commit from main to use a setup-envtest version
+# that uses binaries from controller-tools, not GCS.
+# CR PR: https://github.com/kubernetes-sigs/controller-runtime/pull/2811
+SETUP_ENVTEST_VER := v0.0.0-20240522175850-2e9781e9fc60
 SETUP_ENVTEST_BIN := setup-envtest
 SETUP_ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(SETUP_ENVTEST_BIN)-$(SETUP_ENVTEST_VER))
 SETUP_ENVTEST_PKG := sigs.k8s.io/controller-runtime/tools/setup-envtest
@@ -101,7 +105,7 @@ GOLANGCI_LINT_VER := v1.55.1
 GOLANGCI_LINT_BIN := golangci-lint
 GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN))
 
-GINKGO_VER := v2.17.1
+GINKGO_VER := v2.20.1
 GINKGO_BIN := ginkgo
 GINKGO := $(abspath $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER))
 GINKGO_PKG := github.com/onsi/ginkgo/v2/ginkgo

bootstrap/config/crd/patches/webhook_in_rke2configs.yaml

@@ -8,9 +8,6 @@ spec:
     strategy: Webhook
     webhook:
       clientConfig:
-        # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
-        # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
-        caBundle: Cg==
         service:
           namespace: system
           name: webhook-service

bootstrap/config/crd/patches/webhook_in_rke2configtemplates.yaml

@@ -8,9 +8,6 @@ spec:
     strategy: Webhook
     webhook:
       clientConfig:
-        # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
-        # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
-        caBundle: Cg==
         service:
           namespace: system
           name: webhook-service

bootstrap/config/manager/manager.yaml

@@ -49,6 +49,7 @@ spec:
           privileged: false
           runAsUser: 65532
           runAsGroup: 65532
+        terminationMessagePolicy: FallbackToLogsOnError
       terminationGracePeriodSeconds: 10
       serviceAccountName: manager
       tolerations:

bootstrap/internal/controllers/rke2config_controller.go

@@ -37,6 +37,7 @@ import (
 	kubeyaml "sigs.k8s.io/yaml"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/annotations"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -63,6 +64,7 @@ const (
 // RKE2ConfigReconciler reconciles a Rke2Config object.
 type RKE2ConfigReconciler struct {
 	RKE2InitLock RKE2InitLock
+	Tracker      *remote.ClusterCacheTracker
 	client.Client
 	Scheme *runtime.Scheme
 }

bootstrap/main.go

@@ -19,7 +19,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"os"
 	"time"
 
@@ -39,6 +38,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/remote"
 	"sigs.k8s.io/cluster-api/util/flags"
 
 	bootstrapv1alpha1 "github.com/rancher/cluster-api-provider-rke2/bootstrap/api/v1alpha1"
@@ -47,27 +47,29 @@ import (
 	controlplanev1alpha1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1alpha1"
 	controlplanev1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta1"
 	"github.com/rancher/cluster-api-provider-rke2/pkg/consts"
+	"github.com/rancher/cluster-api-provider-rke2/version"
 )
 
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
 
 	// flags.
-	enableLeaderElection        bool
-	leaderElectionLeaseDuration time.Duration
-	leaderElectionRenewDeadline time.Duration
-	leaderElectionRetryPeriod   time.Duration
-	watchFilterValue            string
-	profilerAddress             string
-	concurrencyNumber           int
-	syncPeriod                  time.Duration
-	watchNamespace              string
-	webhookPort                 int
-	webhookCertDir              string
-	healthAddr                  string
-
-	diagnosticsOptions = flags.DiagnosticsOptions{}
+	enableLeaderElection           bool
+	leaderElectionLeaseDuration    time.Duration
+	leaderElectionRenewDeadline    time.Duration
+	leaderElectionRetryPeriod      time.Duration
+	watchFilterValue               string
+	profilerAddress                string
+	concurrencyNumber              int
+	syncPeriod                     time.Duration
+	clusterCacheTrackerClientQPS   float32
+	clusterCacheTrackerClientBurst int
+	watchNamespace                 string
+	webhookPort                    int
+	webhookCertDir                 string
+	healthAddr                     string
+	managerOptions                 = flags.ManagerOptions{}
 )
 
 func init() {
@@ -108,6 +110,12 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&syncPeriod, "sync-period", consts.DefaultSyncPeriod,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
+	fs.Float32Var(&clusterCacheTrackerClientQPS, "clustercachetracker-client-qps", 20,
+		"Maximum queries per second from the cluster cache tracker clients to the Kubernetes API server of workload clusters.")
+
+	fs.IntVar(&clusterCacheTrackerClientBurst, "clustercachetracker-client-burst", 30,
+		"Maximum number of queries that should be allowed in one burst from the cluster cache tracker clients to the Kubernetes API server of workload clusters.")
+
 	fs.StringVar(&watchNamespace, "namespace", "",
 		"Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") //nolint:lll
 
@@ -119,7 +127,7 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&healthAddr, "health-addr", ":9440",
 		"The address the health endpoint binds to.")
 
-	flags.AddDiagnosticsOptions(fs, &diagnosticsOptions)
+	flags.AddManagerOptions(fs, &managerOptions)
 }
 
 // Add RBAC for the authorized diagnostics endpoint.
@@ -133,7 +141,13 @@ func main() {
 
 	ctrl.SetLogger(klog.Background())
 
-	diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions)
+	restConfig := ctrl.GetConfigOrDie()
+
+	tlsOptions, metricsOptions, err := flags.GetManagerOptions(managerOptions)
+	if err != nil {
+		setupLog.Error(err, "Unable to start manager: invalid flags")
+		os.Exit(1)
+	}
 
 	var watchNamespaces map[string]cache.Config
 
@@ -144,23 +158,16 @@ func main() {
 		}
 	}
 
-	if profilerAddress != "" {
-		klog.Infof("Profiler listening for requests at %s", profilerAddress)
-
-		go func() {
-			klog.Info(http.ListenAndServe(profilerAddress, nil))
-		}()
-	}
-
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:           scheme,
-		LeaderElection:   enableLeaderElection,
-		LeaderElectionID: "rke2-bootstrap-manager-leader-election-capi",
-		PprofBindAddress: profilerAddress,
-		LeaseDuration:    &leaderElectionLeaseDuration,
-		RenewDeadline:    &leaderElectionRenewDeadline,
-		RetryPeriod:      &leaderElectionRetryPeriod,
-		Metrics:          diagnosticsOpts,
+	ctrlOptions := ctrl.Options{
+		Scheme:                 scheme,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "rke2-bootstrap-manager-leader-election-capi",
+		PprofBindAddress:       profilerAddress,
+		LeaseDuration:          &leaderElectionLeaseDuration,
+		RenewDeadline:          &leaderElectionRenewDeadline,
+		RetryPeriod:            &leaderElectionRetryPeriod,
+		HealthProbeBindAddress: healthAddr,
+		Metrics:                *metricsOptions,
 		Cache: cache.Options{
 			DefaultNamespaces: watchNamespaces,
 			SyncPeriod:        &syncPeriod,
@@ -177,23 +184,27 @@ func main() {
 			webhook.Options{
 				Port:    webhookPort,
 				CertDir: webhookCertDir,
+				TLSOpts: tlsOptions,
 			},
 		),
-		HealthProbeBindAddress: healthAddr,
-	})
+	}
+
+	mgr, err := ctrl.NewManager(restConfig, ctrlOptions)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
 
+	// Setup the context that's going to be used in controllers and for the manager.
+	ctx := ctrl.SetupSignalHandler()
+
 	setupChecks(mgr)
 	setupReconcilers(mgr)
 	setupWebhooks(mgr)
-	//+kubebuilder:scaffold:builder
 
-	setupLog.Info("starting manager")
+	setupLog.Info("Starting manager", "version", version.Get().String())
 
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
@@ -212,9 +223,36 @@ func setupChecks(mgr ctrl.Manager) {
 }
 
 func setupReconcilers(mgr ctrl.Manager) {
+	secretCachingClient, err := client.New(mgr.GetConfig(), client.Options{
+		HTTPClient: mgr.GetHTTPClient(),
+		Cache: &client.CacheOptions{
+			Reader: mgr.GetCache(),
+		},
+	})
+	if err != nil {
+		setupLog.Error(err, "Unable to create secret caching client")
+		os.Exit(1)
+	}
+
+	tracker, err := remote.NewClusterCacheTracker(
+		mgr,
+		remote.ClusterCacheTrackerOptions{
+			SecretCachingClient: secretCachingClient,
+			ControllerName:      "rke2-bootstrap-controller",
+			Log:                 &ctrl.Log,
+			ClientQPS:           clusterCacheTrackerClientQPS,
+			ClientBurst:         clusterCacheTrackerClientBurst,
+		},
+	)
+	if err != nil {
+		setupLog.Error(err, "Unable to create cluster cache tracker")
+		os.Exit(1)
+	}
+
 	if err := (&controllers.RKE2ConfigReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Tracker: tracker,
+		Client:  mgr.GetClient(),
+		Scheme:  mgr.GetScheme(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Rke2Config")
 		os.Exit(1)

controlplane/config/crd/patches/webhook_in_rke2controlplanes.yaml

@@ -8,9 +8,6 @@ spec:
     strategy: Webhook
     webhook:
       clientConfig:
-        # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
-        # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
-        caBundle: Cg==
         service:
           namespace: system
           name: webhook-service

controlplane/config/crd/patches/webhook_in_rke2controlplanetemplates.yaml

@@ -8,9 +8,6 @@ spec:
     strategy: Webhook
     webhook:
       clientConfig:
-        # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
-        # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
-        caBundle: Cg==
         service:
           namespace: system
           name: webhook-service

controlplane/config/manager/manager.yaml

@@ -62,6 +62,7 @@ spec:
           privileged: false
           runAsUser: 65532
           runAsGroup: 65532
+        terminationMessagePolicy: FallbackToLogsOnError
         # TODO(user): Configure the resources accordingly based on the project requirements.
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:

controlplane/internal/controllers/rke2controlplane_controller.go

@@ -74,6 +74,9 @@ type RKE2ControlPlaneReconciler struct {
 
 	SecretCachingClient client.Client
 
+	// WatchFilterValue is the label value used to filter events prior to reconciliation.
+	WatchFilterValue string
+
 	managementClusterUncached rke2.ManagementCluster
 	managementCluster         rke2.ManagementCluster
 	recorder                  record.EventRecorder
@@ -220,7 +223,7 @@ func patchRKE2ControlPlane(ctx context.Context, patchHelper *patch.Helper, rcp *
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *RKE2ControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
+func (r *RKE2ControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, clientQPS float32, clientBurst int) error {
 	c, err := ctrl.NewControllerManagedBy(mgr).
 		For(&controlplanev1.RKE2ControlPlane{}).
 		Owns(&clusterv1.Machine{}).
@@ -230,8 +233,9 @@ func (r *RKE2ControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr c
 	}
 
 	err = c.Watch(
-		source.Kind(mgr.GetCache(), &clusterv1.Cluster{}),
-		handler.EnqueueRequestsFromMapFunc(r.ClusterToRKE2ControlPlane(ctx)),
+		source.Kind[client.Object](mgr.GetCache(), &clusterv1.Cluster{},
+			handler.EnqueueRequestsFromMapFunc((r.ClusterToRKE2ControlPlane(ctx))),
+		),
 	)
 	if err != nil {
 		return errors.Wrap(err, "failed adding Watch for Clusters to controller manager")
@@ -253,6 +257,8 @@ func (r *RKE2ControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr c
 				&corev1.ConfigMap{},
 				&corev1.Secret{},
 			},
+			ClientQPS:   clientQPS,
+			ClientBurst: clientBurst,
 		},
 	)
 	if err != nil {