From 60003498bcf90510dfbf3533d75ecc67ff80d267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Corentin=20N=C3=A9au?= Date: Fri, 27 Sep 2024 16:49:26 +0200 Subject: [PATCH] Increase visibility of gitjob controller logs and events (#2906) * Add `fleet_` prefix to events This will help users find events produced by Fleet, whether through the Rancher UI or `kubectl` for instance. * Reduce verbosity of Gitjob controller logs This makes reconciliation, resource creation, deletion and status update errors for `GitRepo`s more visible, ensuring that they appear in logs even when debug logging is disabled. * Add gitrepo field to bundle reconciler logs This will help map logs on `GitRepo`s, produced by the `gitjob` reconciler, with those on corresponding bundles from their reconciler. * Add commit field to bundle logs This will help map logs referring to a bundle with those referring to its `GitRepo`, across the `fleet` and `gitjob` controllers. * Update gitops event recorder name This clarifies what events are about, as concisely as possible,. --- internal/cmd/agent/deployer/deployer.go | 2 +- internal/cmd/cli/apply.go | 3 ++- internal/cmd/controller/gitops/operator.go | 8 ++++---- .../cmd/controller/gitops/reconciler/gitjob_controller.go | 4 ++-- internal/cmd/controller/reconciler/bundle_controller.go | 7 +++++++ internal/metrics/bundle_metrics.go | 2 +- internal/metrics/bundledeployment_metrics.go | 2 +- internal/metrics/metrics.go | 1 - pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go | 1 + 9 files changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/cmd/agent/deployer/deployer.go b/internal/cmd/agent/deployer/deployer.go index 9462b7cf24..79340f25e2 100644 --- a/internal/cmd/agent/deployer/deployer.go +++ b/internal/cmd/agent/deployer/deployer.go @@ -151,7 +151,7 @@ func (d *Deployer) helmdeploy(ctx context.Context, bd *fleet.BundleDeployment) ( } } - manifest.Commit = bd.Labels["fleet.cattle.io/commit"] + manifest.Commit = bd.Labels[fleet.CommitLabel] release, err := d.helm.Deploy(ctx, bd.Name, manifest, bd.Spec.Options) if err != nil { return "", err diff --git a/internal/cmd/cli/apply.go b/internal/cmd/cli/apply.go index b41f08806f..5916693263 100644 --- a/internal/cmd/cli/apply.go +++ b/internal/cmd/cli/apply.go @@ -14,6 +14,7 @@ import ( command "github.com/rancher/fleet/internal/cmd" "github.com/rancher/fleet/internal/cmd/cli/apply" "github.com/rancher/fleet/internal/cmd/cli/writer" + fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1" "k8s.io/apimachinery/pkg/util/yaml" ) @@ -75,7 +76,7 @@ func (a *Apply) Run(cmd *cobra.Command, args []string) error { if labels == nil { labels = map[string]string{} } - labels["fleet.cattle.io/commit"] = a.Commit + labels[fleet.CommitLabel] = a.Commit } name := "" diff --git a/internal/cmd/controller/gitops/operator.go b/internal/cmd/controller/gitops/operator.go index 6c9a3e3331..8808cfb16e 100644 --- a/internal/cmd/controller/gitops/operator.go +++ b/internal/cmd/controller/gitops/operator.go @@ -91,16 +91,16 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error { return err } - var leaderElectionSuffix string + var shardIDSuffix string if g.ShardID != "" { - leaderElectionSuffix = fmt.Sprintf("-%s", g.ShardID) + shardIDSuffix = fmt.Sprintf("-%s", g.ShardID) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: g.setupMetrics(), LeaderElection: g.EnableLeaderElection, - LeaderElectionID: fmt.Sprintf("fleet-gitops-leader-election-shard%s", leaderElectionSuffix), + LeaderElectionID: fmt.Sprintf("fleet-gitops-leader-election-shard%s", shardIDSuffix), LeaderElectionNamespace: namespace, LeaseDuration: leaderOpts.LeaseDuration, RenewDeadline: leaderOpts.RenewDeadline, @@ -132,7 +132,7 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error { JobNodeSelector: g.ShardNodeSelector, GitFetcher: &git.Fetch{}, Clock: reconciler.RealClock{}, - Recorder: mgr.GetEventRecorderFor(fmt.Sprintf("gitjob-controller%s", g.ShardID)), + Recorder: mgr.GetEventRecorderFor(fmt.Sprintf("fleet-gitops%s", shardIDSuffix)), } configReconciler := &fcreconciler.ConfigReconciler{ diff --git a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go index 54709c3f02..c64b275483 100644 --- a/internal/cmd/controller/gitops/reconciler/gitjob_controller.go +++ b/internal/cmd/controller/gitops/reconciler/gitjob_controller.go @@ -223,7 +223,7 @@ func (r *GitJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr err = grutil.UpdateStatus(ctx, r.Client, req.NamespacedName, gitrepo.Status) if err != nil { - logger.V(1).Error(err, "Reconcile failed final update to git repo status", "status", gitrepo.Status) + logger.Error(err, "Reconcile failed final update to git repo status", "status", gitrepo.Status) return result(repoPolled, gitrepo), err } @@ -297,7 +297,7 @@ func (r *GitJobReconciler) manageGitJob(ctx context.Context, logger logr.Logger, } func (r *GitJobReconciler) cleanupGitRepo(ctx context.Context, logger logr.Logger, gitrepo *v1alpha1.GitRepo) error { - logger.V(1).Info("Gitrepo deleted, deleting bundle, image scans") + logger.Info("Gitrepo deleted, deleting bundle, image scans") metrics.GitRepoCollector.Delete(gitrepo.Name, gitrepo.Namespace) diff --git a/internal/cmd/controller/reconciler/bundle_controller.go b/internal/cmd/controller/reconciler/bundle_controller.go index 5c2e2d6663..72ae6ce040 100644 --- a/internal/cmd/controller/reconciler/bundle_controller.go +++ b/internal/cmd/controller/reconciler/bundle_controller.go @@ -131,6 +131,13 @@ func (r *BundleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, client.IgnoreNotFound(err) } + if bundle.Labels[fleet.RepoLabel] != "" { + logger = logger.WithValues( + "gitrepo", bundle.Labels[fleet.RepoLabel], + "commit", bundle.Labels[fleet.CommitLabel], + ) + } + if !bundle.DeletionTimestamp.IsZero() { if controllerutil.ContainsFinalizer(bundle, bundleFinalizer) { metrics.BundleCollector.Delete(req.Name, req.Namespace) diff --git a/internal/metrics/bundle_metrics.go b/internal/metrics/bundle_metrics.go index 14703a1f7a..1f37dd590a 100644 --- a/internal/metrics/bundle_metrics.go +++ b/internal/metrics/bundle_metrics.go @@ -112,7 +112,7 @@ func collectBundleMetrics(obj any, metrics map[string]prometheus.Collector) { labels := prometheus.Labels{ "name": bundle.Name, "namespace": bundle.Namespace, - "commit": bundle.ObjectMeta.Labels[commitLabel], + "commit": bundle.ObjectMeta.Labels[fleet.CommitLabel], "repo": bundle.ObjectMeta.Labels[repoNameLabel], "generation": fmt.Sprintf("%d", bundle.ObjectMeta.Generation), "state": string(currentState), diff --git a/internal/metrics/bundledeployment_metrics.go b/internal/metrics/bundledeployment_metrics.go index 92a87a7458..097f908344 100644 --- a/internal/metrics/bundledeployment_metrics.go +++ b/internal/metrics/bundledeployment_metrics.go @@ -55,7 +55,7 @@ func collectBundleDeploymentMetrics(obj any, metrics map[string]prometheus.Colle "cluster_name": bundleDep.ObjectMeta.Labels["fleet.cattle.io/cluster"], "cluster_namespace": bundleDep.ObjectMeta.Labels["fleet.cattle.io/cluster-namespace"], "repo": bundleDep.ObjectMeta.Labels[repoNameLabel], - "commit": bundleDep.ObjectMeta.Labels[commitLabel], + "commit": bundleDep.ObjectMeta.Labels[fleet.CommitLabel], "bundle": bundleDep.ObjectMeta.Labels["fleet.cattle.io/bundle-name"], "bundle_namespace": bundleDep.ObjectMeta.Labels["fleet.cattle.io/bundle-namespace"], "generation": fmt.Sprintf("%d", bundleDep.ObjectMeta.Generation), diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 8424a11a6e..9e2bb6a9b5 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -13,7 +13,6 @@ import ( const ( metricPrefix = "fleet" - commitLabel = "fleet.cattle.io/commit" repoNameLabel = "fleet.cattle.io/repo-name" ) diff --git a/pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go b/pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go index bbd3142e32..74467ac5f1 100644 --- a/pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go +++ b/pkg/apis/fleet.cattle.io/v1alpha1/gitrepo_types.go @@ -10,6 +10,7 @@ func init() { } var ( + CommitLabel = "fleet.cattle.io/commit" RepoLabel = "fleet.cattle.io/repo-name" BundleLabel = "fleet.cattle.io/bundle-name" BundleNamespaceLabel = "fleet.cattle.io/bundle-namespace"