From d478c2e2532448da98cdfe491730b81214f38b3b Mon Sep 17 00:00:00 2001
From: Oliver Gould <ver@buoyant.io>
Date: Wed, 1 Nov 2023 16:18:48 -0700
Subject: [PATCH] Log a warning when the controller clients receive an error
 (#2499)

The controller client includes a recovery/backoff module that causes
resolutions to be retried when an unexpected error is encountered.
These events are only logged at debugging and trace log levels.

This change updates the destination and policy controller recovery
modules to log unexpected errors as warnings.
---
 linkerd/app/inbound/src/policy/api.rs  | 6 +++++-
 linkerd/app/outbound/src/policy/api.rs | 8 ++++++--
 linkerd/app/src/dst.rs                 | 6 +++++-
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/linkerd/app/inbound/src/policy/api.rs b/linkerd/app/inbound/src/policy/api.rs
index ab646fe5b3..87e4638997 100644
--- a/linkerd/app/inbound/src/policy/api.rs
+++ b/linkerd/app/inbound/src/policy/api.rs
@@ -109,7 +109,11 @@ impl Recover<tonic::Status> for GrpcRecover {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected policy controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }
diff --git a/linkerd/app/outbound/src/policy/api.rs b/linkerd/app/outbound/src/policy/api.rs
index 57c55eff34..d4020677c8 100644
--- a/linkerd/app/outbound/src/policy/api.rs
+++ b/linkerd/app/outbound/src/policy/api.rs
@@ -113,8 +113,12 @@ impl Recover<tonic::Status> for GrpcRecover {
             tonic::Code::InvalidArgument | tonic::Code::FailedPrecondition => Err(status),
             // Indicates no policy for this target
             tonic::Code::NotFound | tonic::Code::Unimplemented => Err(status),
-            _ => {
-                tracing::debug!(%status, "Recovering");
+            code => {
+                tracing::warn!(
+                    grpc.status = %code,
+                    grpc.message = status.message(),
+                    "Unexpected policy controller response; retrying with a backoff",
+                );
                 Ok(self.0.stream())
             }
         }
diff --git a/linkerd/app/src/dst.rs b/linkerd/app/src/dst.rs
index 3c6c66f0ac..f0ce8622c4 100644
--- a/linkerd/app/src/dst.rs
+++ b/linkerd/app/src/dst.rs
@@ -94,7 +94,11 @@ impl Recover<tonic::Status> for BackoffUnlessInvalidArgument {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected destination controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }