Re sync fix (rancher#46312) (rancher#46922)

Use Mutex & Lease to stop concurrent secret creation This reduces the number of concurrent secrets created for service account access in clusters by mutexing the creation by cluster/namespace/name. This should reduce the created secrets. Backport of rancher#46312
HarrisonWAffel · Sep 4, 2024 · 11ae141 · 11ae141
1 parent 0d97d45
commit 11ae141
Show file tree

Hide file tree

Showing 20 changed files with 490 additions and 80 deletions.
diff --git a/pkg/agent/cluster/cluster.go b/pkg/agent/cluster/cluster.go
@@ -78,7 +78,7 @@ func getTokenFromAPI() ([]byte, []byte, error) {
 		}
 		return secret.Data[coreV1.ServiceAccountRootCAKey], secret.Data[coreV1.ServiceAccountTokenKey], nil
 	}
-	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, k8s, sa)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, k8s, sa, "")
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to ensure secret for service account %s/%s: %w", namespace.System, "cattle", err)
 	}

diff --git a/pkg/capr/common.go b/pkg/capr/common.go
@@ -25,6 +25,7 @@ import (
 	capicontrollers "github.com/rancher/rancher/pkg/generated/controllers/cluster.x-k8s.io/v1beta1"
 	rkecontroller "github.com/rancher/rancher/pkg/generated/controllers/rke.cattle.io/v1"
 	"github.com/rancher/rancher/pkg/serviceaccounttoken"
+	"github.com/rancher/rancher/pkg/utils"
 	"github.com/rancher/wrangler/v3/pkg/condition"
 	"github.com/rancher/wrangler/v3/pkg/data"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
@@ -322,7 +323,7 @@ func GetPlanServiceAccountTokenSecret(secretClient corecontrollers.SecretControl
 	if planSA == nil {
 		return nil, false, fmt.Errorf("planSA was nil")
 	}
-	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), secretClient.Cache(), k8s, planSA)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), secretClient.Cache(), k8s, planSA, utils.FormatPrefix("local"))
 	if err != nil {
 		return nil, false, fmt.Errorf("error ensuring secret for service account [%s:%s]: %w", planSA.Namespace, planSA.Name, err)
 	}

diff --git a/pkg/controllers/capr/bootstrap/controller.go b/pkg/controllers/capr/bootstrap/controller.go
@@ -18,6 +18,7 @@ import (
 	"github.com/rancher/rancher/pkg/namespace"
 	"github.com/rancher/rancher/pkg/serviceaccounttoken"
 	"github.com/rancher/rancher/pkg/tls"
+	"github.com/rancher/rancher/pkg/utils"
 	"github.com/rancher/rancher/pkg/wrangler"
 	appcontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
@@ -120,7 +121,7 @@ func (h *handler) getBootstrapSecret(namespace, name string, envVars []corev1.En
 	if err != nil {
 		return nil, err
 	}
-	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), h.secretCache, h.k8s, sa)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), h.secretCache, h.k8s, sa, utils.FormatPrefix("local"))
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/controllers/dashboard/apiservice/apiservice.go b/pkg/controllers/dashboard/apiservice/apiservice.go
@@ -11,6 +11,7 @@ import (
 	"github.com/rancher/rancher/pkg/namespace"
 	"github.com/rancher/rancher/pkg/serviceaccounttoken"
 	"github.com/rancher/rancher/pkg/settings"
+	"github.com/rancher/rancher/pkg/utils"
 	"github.com/rancher/rancher/pkg/wrangler"
 	appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
 	corev1controllers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
@@ -148,7 +149,7 @@ func (h *handler) getToken(sa *corev1.ServiceAccount) (string, error) {
 	}
 
 	// create a secret-based token for the service account if one does not exist
-	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(h.ctx, h.secretsCache, h.k8s, sa)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(h.ctx, h.secretsCache, h.k8s, sa, utils.FormatPrefix("local"))
 	if err != nil {
 		return "", fmt.Errorf("error ensuring secret for service account [%s:%s]: %w", sa.Namespace, sa.Name, err)
 	}

diff --git a/pkg/controllers/management/aks/aks_cluster_handler.go b/pkg/controllers/management/aks/aks_cluster_handler.go
@@ -341,7 +341,7 @@ func (e *aksOperatorController) generateAndSetServiceAccount(cluster *apimgmtv3.
 	}
 
 	restConfig.Dial = clusterDialer
-	saToken, err := clusteroperator.GenerateSAToken(restConfig)
+	saToken, err := clusteroperator.GenerateSAToken(restConfig, cluster.Name)
 	if err != nil {
 		return cluster, fmt.Errorf("error generating service account token: %v", err)
 	}
@@ -422,7 +422,7 @@ func (e *aksOperatorController) generateSATokenWithPublicAPI(cluster *apimgmtv3.
 		Timeout:   30 * time.Second,
 		KeepAlive: 30 * time.Second,
 	}).DialContext
-	serviceToken, err := clusteroperator.GenerateSAToken(restConfig)
+	serviceToken, err := clusteroperator.GenerateSAToken(restConfig, cluster.Name)
 	if err != nil {
 		*requiresTunnel = true
 		var dnsError *net.DNSError

diff --git a/pkg/controllers/management/clusteroperator/utils.go b/pkg/controllers/management/clusteroperator/utils.go
@@ -166,13 +166,13 @@ func (e *OperatorController) CheckCrdReady(cluster *mgmtv3.Cluster, clusterType
 	return cluster, nil
 }
 
-func GenerateSAToken(restConfig *rest.Config) (string, error) {
+func GenerateSAToken(restConfig *rest.Config, clusterName string) (string, error) {
 	clientSet, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
-		return "", fmt.Errorf("error creating clientset: %v", err)
+		return "", fmt.Errorf("error creating clientset for cluster %s: %v", clusterName, err)
 	}
 
-	return util.GenerateServiceAccountToken(clientSet)
+	return util.GenerateServiceAccountToken(clientSet, clusterName)
 }
 
 func addAdditionalCA(secretsCache wranglerv1.SecretCache, caCert string) (string, error) {

diff --git a/pkg/controllers/management/eks/eks_cluster_handler.go b/pkg/controllers/management/eks/eks_cluster_handler.go
@@ -449,7 +449,7 @@ func (e *eksOperatorController) generateAndSetServiceAccount(cluster *mgmtv3.Clu
 		return cluster, err
 	}
 
-	saToken, err := clusteroperator.GenerateSAToken(restConfig)
+	saToken, err := clusteroperator.GenerateSAToken(restConfig, cluster.Name)
 	if err != nil {
 		return cluster, err
 	}
@@ -529,7 +529,7 @@ func (e *eksOperatorController) generateSATokenWithPublicAPI(cluster *mgmtv3.Clu
 	}
 
 	requiresTunnel := new(bool)
-	serviceToken, err := clusteroperator.GenerateSAToken(restConfig)
+	serviceToken, err := clusteroperator.GenerateSAToken(restConfig, cluster.Name)
 	if err != nil {
 		*requiresTunnel = true
 		var dnsError *net.DNSError

diff --git a/pkg/controllers/management/gke/gke_cluster_handler.go b/pkg/controllers/management/gke/gke_cluster_handler.go
@@ -365,7 +365,7 @@ func (e *gkeOperatorController) generateAndSetServiceAccount(cluster *mgmtv3.Clu
 		return cluster, err
 	}
 
-	saToken, err := clusteroperator.GenerateSAToken(restConfig)
+	saToken, err := clusteroperator.GenerateSAToken(restConfig, cluster.Name)
 	if err != nil {
 		return cluster, fmt.Errorf("error generating service account token: %w", err)
 	}
@@ -444,7 +444,7 @@ func (e *gkeOperatorController) generateSATokenWithPublicAPI(cluster *mgmtv3.Clu
 		return "", nil, err
 	}
 	requiresTunnel := new(bool)
-	serviceToken, err := clusteroperator.GenerateSAToken(restConfig)
+	serviceToken, err := clusteroperator.GenerateSAToken(restConfig, cluster.Name)
 	if err != nil {
 		*requiresTunnel = true
 		if strings.Contains(err.Error(), "dial tcp") {

diff --git a/pkg/impersonation/impersonation.go b/pkg/impersonation/impersonation.go
@@ -12,6 +12,7 @@ import (
 	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
 	"github.com/rancher/rancher/pkg/serviceaccounttoken"
 	"github.com/rancher/rancher/pkg/types/config"
+	"github.com/rancher/rancher/pkg/utils"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
@@ -104,7 +105,7 @@ func (i *Impersonator) SetUpImpersonation() (*corev1.ServiceAccount, error) {
 
 // GetToken accepts a service account and returns the service account's token.
 func (i *Impersonator) GetToken(sa *corev1.ServiceAccount) (string, error) {
-	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), i.secretsCache, i.clusterContext.K8sClient, sa)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), i.secretsCache, i.clusterContext.K8sClient, sa, utils.FormatPrefix(i.clusterContext.ClusterName))
 	if err != nil {
 		return "", fmt.Errorf("error getting secret: %w", err)
 	}
@@ -170,7 +171,7 @@ func (i *Impersonator) createServiceAccount(role *rbacv1.ClusterRole) (*corev1.S
 		}
 	}
 	// create secret for service account if it was not automatically generated
-	_, err = serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), i.secretsCache, i.clusterContext.K8sClient, sa)
+	_, err = serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), i.secretsCache, i.clusterContext.K8sClient, sa, utils.FormatPrefix(i.clusterContext.ClusterName))
 	if err != nil {
 		return nil, fmt.Errorf("impersonation: error ensuring secret for service account %s: %w", name, err)
 	}

diff --git a/pkg/kontainer-engine/drivers/aks/aks_driver.go b/pkg/kontainer-engine/drivers/aks/aks_driver.go
@@ -1304,7 +1304,7 @@ func (d *Driver) PostCheck(ctx context.Context, info *types.ClusterInfo) (*types
 	failureCount := 0
 
 	for {
-		info.ServiceAccountToken, err = util.GenerateServiceAccountToken(clientset)
+		info.ServiceAccountToken, err = util.GenerateServiceAccountToken(clientset, "")
 
 		if err == nil {
 			logrus.Info("[azurekubernetesservice] service account token generated successfully")

diff --git a/pkg/kontainer-engine/drivers/eks/eks_driver.go b/pkg/kontainer-engine/drivers/eks/eks_driver.go
@@ -1175,7 +1175,7 @@ func (d *Driver) PostCheck(ctx context.Context, info *types.ClusterInfo) (*types
 
 	logrus.Infof("[amazonelasticcontainerservice] Generating service account token")
 
-	info.ServiceAccountToken, err = util.GenerateServiceAccountToken(clientset)
+	info.ServiceAccountToken, err = util.GenerateServiceAccountToken(clientset, "")
 	if err != nil {
 		return nil, fmt.Errorf("error generating service account token: %v", err)
 	}

diff --git a/pkg/kontainer-engine/drivers/import/import_driver.go b/pkg/kontainer-engine/drivers/import/import_driver.go
@@ -163,7 +163,7 @@ func (d *Driver) PostCheck(ctx context.Context, info *types.ClusterInfo) (*types
 		return nil, fmt.Errorf("failed to get Kubernetes server version: %v", err)
 	}
 
-	info.ServiceAccountToken, err = util.GenerateServiceAccountToken(clientset)
+	info.ServiceAccountToken, err = util.GenerateServiceAccountToken(clientset, "")
 
 	if err != nil {
 		return nil, err

diff --git a/pkg/kontainer-engine/drivers/rke/rke_driver.go b/pkg/kontainer-engine/drivers/rke/rke_driver.go
@@ -300,7 +300,7 @@ func (d *Driver) PostCheck(ctx context.Context, info *types.ClusterInfo) (*types
 			continue
 		}
 
-		token, err := util.GenerateServiceAccountToken(clientset)
+		token, err := util.GenerateServiceAccountToken(clientset, "")
 		if err != nil {
 			lastErr = err
 			time.Sleep(2 * time.Second)

diff --git a/pkg/kontainer-engine/drivers/util/utils.go b/pkg/kontainer-engine/drivers/util/utils.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/rancher/rancher/pkg/serviceaccounttoken"
+	"github.com/rancher/rancher/pkg/utils"
 	rketypes "github.com/rancher/rke/types"
 	"gopkg.in/yaml.v2"
 	v1 "k8s.io/api/core/v1"
@@ -25,7 +26,7 @@ const (
 )
 
 // GenerateServiceAccountToken generate a serviceAccountToken for clusterAdmin given a rest clientset
-func GenerateServiceAccountToken(clientset kubernetes.Interface) (string, error) {
+func GenerateServiceAccountToken(clientset kubernetes.Interface, clusterName string) (string, error) {
 	_, err := clientset.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: cattleNamespace,
@@ -95,7 +96,7 @@ func GenerateServiceAccountToken(clientset kubernetes.Interface) (string, error)
 	if serviceAccount, err = clientset.CoreV1().ServiceAccounts(cattleNamespace).Get(context.Background(), serviceAccount.Name, metav1.GetOptions{}); err != nil {
 		return "", fmt.Errorf("error getting service account: %w", err)
 	}
-	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, clientset, serviceAccount)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, clientset, serviceAccount, utils.FormatPrefix(clusterName))
 	if err != nil {
 		return "", fmt.Errorf("error ensuring secret for service account: %w", err)
 	}