Increase visibility of gitjob controller logs and events (#2906)

* Add `fleet_` prefix to events This will help users find events produced by Fleet, whether through the Rancher UI or `kubectl` for instance. * Reduce verbosity of Gitjob controller logs This makes reconciliation, resource creation, deletion and status update errors for `GitRepo`s more visible, ensuring that they appear in logs even when debug logging is disabled. * Add gitrepo field to bundle reconciler logs This will help map logs on `GitRepo`s, produced by the `gitjob` reconciler, with those on corresponding bundles from their reconciler. * Add commit field to bundle logs This will help map logs referring to a bundle with those referring to its `GitRepo`, across the `fleet` and `gitjob` controllers. * Update gitops event recorder name This clarifies what events are about, as concisely as possible,.
rancher · Sep 27, 2024 · 6000349 · 6000349
1 parent 5d934fa
commit 6000349
Show file tree

Hide file tree

Showing 9 changed files with 19 additions and 11 deletions.
diff --git a/internal/cmd/agent/deployer/deployer.go b/internal/cmd/agent/deployer/deployer.go
@@ -151,7 +151,7 @@ func (d *Deployer) helmdeploy(ctx context.Context, bd *fleet.BundleDeployment) (
 		}
 	}
 
-	manifest.Commit = bd.Labels["fleet.cattle.io/commit"]
+	manifest.Commit = bd.Labels[fleet.CommitLabel]
 	release, err := d.helm.Deploy(ctx, bd.Name, manifest, bd.Spec.Options)
 	if err != nil {
 		return "", err

diff --git a/internal/cmd/cli/apply.go b/internal/cmd/cli/apply.go
@@ -14,6 +14,7 @@ import (
 	command "github.com/rancher/fleet/internal/cmd"
 	"github.com/rancher/fleet/internal/cmd/cli/apply"
 	"github.com/rancher/fleet/internal/cmd/cli/writer"
+	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
@@ -75,7 +76,7 @@ func (a *Apply) Run(cmd *cobra.Command, args []string) error {
 		if labels == nil {
 			labels = map[string]string{}
 		}
-		labels["fleet.cattle.io/commit"] = a.Commit
+		labels[fleet.CommitLabel] = a.Commit
 	}
 
 	name := ""

diff --git a/internal/cmd/controller/gitops/operator.go b/internal/cmd/controller/gitops/operator.go
@@ -91,16 +91,16 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	var leaderElectionSuffix string
+	var shardIDSuffix string
 	if g.ShardID != "" {
-		leaderElectionSuffix = fmt.Sprintf("-%s", g.ShardID)
+		shardIDSuffix = fmt.Sprintf("-%s", g.ShardID)
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                  scheme,
 		Metrics:                 g.setupMetrics(),
 		LeaderElection:          g.EnableLeaderElection,
-		LeaderElectionID:        fmt.Sprintf("fleet-gitops-leader-election-shard%s", leaderElectionSuffix),
+		LeaderElectionID:        fmt.Sprintf("fleet-gitops-leader-election-shard%s", shardIDSuffix),
 		LeaderElectionNamespace: namespace,
 		LeaseDuration:           leaderOpts.LeaseDuration,
 		RenewDeadline:           leaderOpts.RenewDeadline,
@@ -132,7 +132,7 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error {
 		JobNodeSelector: g.ShardNodeSelector,
 		GitFetcher:      &git.Fetch{},
 		Clock:           reconciler.RealClock{},
-		Recorder:        mgr.GetEventRecorderFor(fmt.Sprintf("gitjob-controller%s", g.ShardID)),
+		Recorder:        mgr.GetEventRecorderFor(fmt.Sprintf("fleet-gitops%s", shardIDSuffix)),
 	}
 
 	configReconciler := &fcreconciler.ConfigReconciler{

diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go
@@ -223,7 +223,7 @@ func (r *GitJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	err = grutil.UpdateStatus(ctx, r.Client, req.NamespacedName, gitrepo.Status)
 	if err != nil {
-		logger.V(1).Error(err, "Reconcile failed final update to git repo status", "status", gitrepo.Status)
+		logger.Error(err, "Reconcile failed final update to git repo status", "status", gitrepo.Status)
 
 		return result(repoPolled, gitrepo), err
 	}
@@ -297,7 +297,7 @@ func (r *GitJobReconciler) manageGitJob(ctx context.Context, logger logr.Logger,
 }
 
 func (r *GitJobReconciler) cleanupGitRepo(ctx context.Context, logger logr.Logger, gitrepo *v1alpha1.GitRepo) error {
-	logger.V(1).Info("Gitrepo deleted, deleting bundle, image scans")
+	logger.Info("Gitrepo deleted, deleting bundle, image scans")
 
 	metrics.GitRepoCollector.Delete(gitrepo.Name, gitrepo.Namespace)
 

diff --git a/internal/cmd/controller/reconciler/bundle_controller.go b/internal/cmd/controller/reconciler/bundle_controller.go
@@ -131,6 +131,13 @@ func (r *BundleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	if bundle.Labels[fleet.RepoLabel] != "" {
+		logger = logger.WithValues(
+			"gitrepo", bundle.Labels[fleet.RepoLabel],
+			"commit", bundle.Labels[fleet.CommitLabel],
+		)
+	}
+
 	if !bundle.DeletionTimestamp.IsZero() {
 		if controllerutil.ContainsFinalizer(bundle, bundleFinalizer) {
 			metrics.BundleCollector.Delete(req.Name, req.Namespace)

diff --git a/internal/metrics/bundle_metrics.go b/internal/metrics/bundle_metrics.go
@@ -112,7 +112,7 @@ func collectBundleMetrics(obj any, metrics map[string]prometheus.Collector) {
 	labels := prometheus.Labels{
 		"name":       bundle.Name,
 		"namespace":  bundle.Namespace,
-		"commit":     bundle.ObjectMeta.Labels[commitLabel],
+		"commit":     bundle.ObjectMeta.Labels[fleet.CommitLabel],
 		"repo":       bundle.ObjectMeta.Labels[repoNameLabel],
 		"generation": fmt.Sprintf("%d", bundle.ObjectMeta.Generation),
 		"state":      string(currentState),

diff --git a/internal/metrics/bundledeployment_metrics.go b/internal/metrics/bundledeployment_metrics.go
@@ -55,7 +55,7 @@ func collectBundleDeploymentMetrics(obj any, metrics map[string]prometheus.Colle
 		"cluster_name":      bundleDep.ObjectMeta.Labels["fleet.cattle.io/cluster"],
 		"cluster_namespace": bundleDep.ObjectMeta.Labels["fleet.cattle.io/cluster-namespace"],
 		"repo":              bundleDep.ObjectMeta.Labels[repoNameLabel],
-		"commit":            bundleDep.ObjectMeta.Labels[commitLabel],
+		"commit":            bundleDep.ObjectMeta.Labels[fleet.CommitLabel],
 		"bundle":            bundleDep.ObjectMeta.Labels["fleet.cattle.io/bundle-name"],
 		"bundle_namespace":  bundleDep.ObjectMeta.Labels["fleet.cattle.io/bundle-namespace"],
 		"generation":        fmt.Sprintf("%d", bundleDep.ObjectMeta.Generation),

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
@@ -13,7 +13,6 @@ import (
 
 const (
 	metricPrefix  = "fleet"
-	commitLabel   = "fleet.cattle.io/commit"
 	repoNameLabel = "fleet.cattle.io/repo-name"
 )
 

diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go b/pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go
@@ -10,6 +10,7 @@ func init() {
 }
 
 var (
+	CommitLabel          = "fleet.cattle.io/commit"
 	RepoLabel            = "fleet.cattle.io/repo-name"
 	BundleLabel          = "fleet.cattle.io/bundle-name"
 	BundleNamespaceLabel = "fleet.cattle.io/bundle-namespace"
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,7 +13,6 @@ import ( @@
     const (
     	metricPrefix  = "fleet"
-    	commitLabel   = "fleet.cattle.io/commit"
     	repoNameLabel = "fleet.cattle.io/repo-name"
     )
@@ Expand Down @@