core: add -failing filter to health command (smartcontractkit#13989)

chaingence · Aug 1, 2024 · 1d12fbf · 1d12fbf
1 parent f974754
commit 1d12fbf
Show file tree

Hide file tree

Showing 9 changed files with 103 additions and 2 deletions.
diff --git a/core/cmd/app.go b/core/cmd/app.go
@@ -168,6 +168,10 @@ func NewApp(s *Shell) *cli.App {
 			Usage:  "Prints a health report",
 			Action: s.Health,
 			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "failing, f",
+					Usage: "filter for failing services",
+				},
 				cli.BoolFlag{
 					Name:  "json, j",
 					Usage: "json output",

diff --git a/core/cmd/shell_remote.go b/core/cmd/shell_remote.go
@@ -517,7 +517,11 @@ func (s *Shell) Health(c *cli.Context) error {
 	if c.Bool("json") {
 		mime = gin.MIMEJSON
 	}
-	resp, err := s.HTTP.Get(s.ctx(), "/health", map[string]string{"Accept": mime})
+	u := "/health"
+	if c.Bool("failing") {
+		u += "?failing"
+	}
+	resp, err := s.HTTP.Get(s.ctx(), u, map[string]string{"Accept": mime})
 	if err != nil {
 		return s.errorOut(err)
 	}

diff --git a/core/web/health_controller.go b/core/web/health_controller.go
@@ -69,6 +69,8 @@ func (hc *HealthController) Readyz(c *gin.Context) {
 }
 
 func (hc *HealthController) Health(c *gin.Context) {
+	_, failing := c.GetQuery("failing")
+
 	status := http.StatusOK
 
 	checker := hc.App.GetHealthChecker()
@@ -89,6 +91,8 @@ func (hc *HealthController) Health(c *gin.Context) {
 		if err != nil {
 			status = HealthStatusFailing
 			output = err.Error()
+		} else if failing {
+			continue // omit from returned data
 		}
 
 		checks = append(checks, presenters.Check{

diff --git a/core/web/health_controller_test.go b/core/web/health_controller_test.go
@@ -97,6 +97,12 @@ var (
 	bodyHTML string
 	//go:embed testdata/body/health.txt
 	bodyTXT string
+	//go:embed testdata/body/health-failing.json
+	bodyJSONFailing string
+	//go:embed testdata/body/health-failing.html
+	bodyHTMLFailing string
+	//go:embed testdata/body/health-failing.txt
+	bodyTXTFailing string
 )
 
 func TestHealthController_Health_body(t *testing.T) {
@@ -111,6 +117,12 @@ func TestHealthController_Health_body(t *testing.T) {
 		{"html", "/health", map[string]string{"Accept": gin.MIMEHTML}, bodyHTML},
 		{"text", "/health", map[string]string{"Accept": gin.MIMEPlain}, bodyTXT},
 		{".txt", "/health.txt", nil, bodyTXT},
+
+		{"default-failing", "/health?failing", nil, bodyJSONFailing},
+		{"json-failing", "/health?failing", map[string]string{"Accept": gin.MIMEJSON}, bodyJSONFailing},
+		{"html-failing", "/health?failing", map[string]string{"Accept": gin.MIMEHTML}, bodyHTMLFailing},
+		{"text-failing", "/health?failing", map[string]string{"Accept": gin.MIMEPlain}, bodyTXTFailing},
+		{".txt-failing", "/health.txt?failing", nil, bodyTXTFailing},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			app := cltest.NewApplicationWithKey(t)

diff --git a/core/web/testdata/body/health-failing.html b/core/web/testdata/body/health-failing.html
@@ -0,0 +1,47 @@
+<style>
+    details {
+        margin: 0.0em 0.0em 0.0em 0.4em;
+        padding: 0.3em 0.0em 0.0em 0.4em;
+    }
+    pre {
+        margin-left:1em;
+        margin-top: 0;
+    }
+    summary {
+        padding-bottom: 0.4em;
+    }
+    details {
+        border: thin solid black;
+        border-bottom-color: rgba(0,0,0,0);
+        border-right-color: rgba(0,0,0,0);
+    }
+    .passing:after {
+        color: blue;
+        content: " - (Passing)";
+        font-size:small;
+        text-transform: uppercase;
+    }
+    .failing:after {
+        color: red;
+        content: " - (Failing)";
+        font-weight: bold;
+        font-size:small;
+        text-transform: uppercase;
+    }
+    summary.noexpand::marker {
+        color: rgba(100,101,10,0);
+    }
+</style>
+<details open>
+    <summary title=""><span class="">EVM</span></summary>
+    <details open>
+        <summary title=""><span class="">0</span></summary>
+        <details open>
+            <summary title=""><span class="">HeadTracker</span></summary>
+            <details open>
+                <summary title="EVM.0.HeadTracker.HeadListener"><span class="failing">HeadListener</span></summary>
+                <pre>Listener is not connected</pre>
+            </details>
+        </details>
+    </details>
+</details>
diff --git a/core/web/testdata/body/health-failing.json b/core/web/testdata/body/health-failing.json
@@ -0,0 +1 @@
+{"data":[{"type":"checks","id":"EVM.0.HeadTracker.HeadListener","attributes":{"name":"EVM.0.HeadTracker.HeadListener","status":"failing","output":"Listener is not connected"}}]}
diff --git a/core/web/testdata/body/health-failing.txt b/core/web/testdata/body/health-failing.txt
@@ -0,0 +1,2 @@
+!  EVM.0.HeadTracker.HeadListener
+	Listener is not connected
diff --git a/testdata/scripts/health/help.txtar b/testdata/scripts/health/help.txtar
@@ -9,5 +9,6 @@ USAGE:
    chainlink health [command options] [arguments...]
 
 OPTIONS:
-   --json, -j  json output
+   --failing, -f  filter for failing services
+   --json, -j     json output
 
diff --git a/testdata/scripts/health/multi-chain.txtar b/testdata/scripts/health/multi-chain.txtar
@@ -15,6 +15,14 @@ cp stdout compact.json
 exec jq . compact.json
 cmp stdout out.json
 
+exec chainlink --remote-node-url $NODEURL health -failing
+cmp stdout out-unhealthy.txt
+
+exec chainlink --remote-node-url $NODEURL health -f -json
+cp stdout compact.json
+exec jq . compact.json
+cmp stdout out-unhealthy.json
+
 -- testdb.txt --
 CL_DATABASE_URL
 -- testport.txt --
@@ -87,6 +95,10 @@ ok Solana.Bar
 ok StarkNet.Baz
 ok TelemetryManager
 
+-- out-unhealthy.txt --
+!  EVM.1.HeadTracker.HeadListener
+	Listener is not connected
+
 -- out.json --
 {
   "data": [
@@ -317,3 +329,17 @@ ok TelemetryManager
     }
   ]
 }
+-- out-unhealthy.json --
+{
+  "data": [
+    {
+      "type": "checks",
+      "id": "EVM.1.HeadTracker.HeadListener",
+      "attributes": {
+        "name": "EVM.1.HeadTracker.HeadListener",
+        "status": "failing",
+        "output": "Listener is not connected"
+      }
+    }
+  ]
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"data":[{"type":"checks","id":"EVM.0.HeadTracker.HeadListener","attributes":{"name":"EVM.0.HeadTracker.HeadListener","status":"failing","output":"Listener is not connected"}}]}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		! EVM.0.HeadTracker.HeadListener
		Listener is not connected