From 4e9cbb4e514b702ae4f4d8bfc3f8f33c4ab3842d Mon Sep 17 00:00:00 2001
From: Christian Svensson <blue@cmd.nu>
Date: Fri, 5 Jan 2024 13:51:38 +0100
Subject: [PATCH] fix: retry system resources probe when malformed

Fixes #266

Signed-off-by: Christian Svensson <blue@cmd.nu>
---
 pkg/probe/system_resources_usage.go      | 20 ++++++++++++++++++--
 pkg/probe/system_resources_usage_test.go |  9 +++++++++
 pkg/probe/testdata/usage-empty.jsonnet   | 13 +++++++++++++
 3 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 pkg/probe/testdata/usage-empty.jsonnet

diff --git a/pkg/probe/system_resources_usage.go b/pkg/probe/system_resources_usage.go
index b7ed14a..ad367c1 100644
--- a/pkg/probe/system_resources_usage.go
+++ b/pkg/probe/system_resources_usage.go
@@ -3,6 +3,7 @@ package probe
 import (
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/bluecmd/fortigate_exporter/pkg/http"
 	"github.com/prometheus/client_golang/prometheus"
@@ -53,8 +54,23 @@ func probeSystemResourceUsage(c http.FortiHTTP, meta *TargetMetadata) ([]prometh
 	}
 	var sr systemResourceUsage
 
-	if err := c.Get("api/v2/monitor/system/resource/usage", "interval=1-min&scope=global", &sr); err != nil {
-		log.Printf("Error: %v", err)
+	attempts := 3
+	for ; attempts > 0; attempts-- {
+		if err := c.Get("api/v2/monitor/system/resource/usage", "interval=1-min&scope=global", &sr); err != nil {
+			log.Printf("Error: %v", err)
+			return nil, false
+		}
+		// See https://github.com/bluecmd/fortigate_exporter/issues/266, sometimes FortiOS
+		// returns an empty resource document. If this happens, we try to fetch it up to three times.
+		if len(sr.Results.CPU) == 0 {
+			log.Printf("Got system resources, will retry fetch..")
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		break
+	}
+	if attempts == 0 {
+		log.Printf("Error: Out of attempts to fetch system resources, giving up")
 		return nil, false
 	}
 
diff --git a/pkg/probe/system_resources_usage_test.go b/pkg/probe/system_resources_usage_test.go
index 040024c..6dfaffb 100644
--- a/pkg/probe/system_resources_usage_test.go
+++ b/pkg/probe/system_resources_usage_test.go
@@ -60,3 +60,12 @@ func TestSystemVDOMResources(t *testing.T) {
 		t.Fatalf("metric compare: err %v", err)
 	}
 }
+
+func TestBrokenSystemResourceUsage(t *testing.T) {
+	c := newFakeClient()
+	c.prepare("api/v2/monitor/system/resource/usage", "testdata/usage-empty.jsonnet")
+	r := prometheus.NewPedanticRegistry()
+	if testProbe(probeSystemResourceUsage, c, r) {
+		t.Errorf("probeSystemResourceUsage() returned success, should have failed")
+	}
+}
diff --git a/pkg/probe/testdata/usage-empty.jsonnet b/pkg/probe/testdata/usage-empty.jsonnet
new file mode 100644
index 0000000..fcd5ec0
--- /dev/null
+++ b/pkg/probe/testdata/usage-empty.jsonnet
@@ -0,0 +1,13 @@
+# api/v2/monitor/system/resource/usage?scope=global
+{
+  "http_method":"GET",
+  "results": {},
+  "vdom":"root",
+  "path":"system",
+  "name":"resource",
+  "action":"usage",
+  "status":"success",
+  "serial":"FGVMEVZFNTS3OAC8",
+  "version":"v6.2.4",
+  "build":1112
+}