fix: retry system resources probe when malformed

Fixes #266

Signed-off-by: Christian Svensson <[email protected]>

pkg/probe/system_resources_usage.go

@@ -3,6 +3,7 @@ package probe
 import (
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/bluecmd/fortigate_exporter/pkg/http"
 	"github.com/prometheus/client_golang/prometheus"
@@ -53,8 +54,23 @@ func probeSystemResourceUsage(c http.FortiHTTP, meta *TargetMetadata) ([]prometh
 	}
 	var sr systemResourceUsage
 
-	if err := c.Get("api/v2/monitor/system/resource/usage", "interval=1-min&scope=global", &sr); err != nil {
-		log.Printf("Error: %v", err)
+	attempts := 3
+	for ; attempts > 0; attempts-- {
+		if err := c.Get("api/v2/monitor/system/resource/usage", "interval=1-min&scope=global", &sr); err != nil {
+			log.Printf("Error: %v", err)
+			return nil, false
+		}
+		// See https://github.com/bluecmd/fortigate_exporter/issues/266, sometimes FortiOS
+		// returns an empty resource document. If this happens, we try to fetch it up to three times.
+		if len(sr.Results.CPU) == 0 {
+			log.Printf("Got system resources, will retry fetch..")
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		break
+	}
+	if attempts == 0 {
+		log.Printf("Error: Out of attempts to fetch system resources, giving up")
 		return nil, false
 	}
 

pkg/probe/system_resources_usage_test.go

@@ -60,3 +60,12 @@ func TestSystemVDOMResources(t *testing.T) {
 		t.Fatalf("metric compare: err %v", err)
 	}
 }
+
+func TestBrokenSystemResourceUsage(t *testing.T) {
+	c := newFakeClient()
+	c.prepare("api/v2/monitor/system/resource/usage", "testdata/usage-empty.jsonnet")
+	r := prometheus.NewPedanticRegistry()
+	if testProbe(probeSystemResourceUsage, c, r) {
+		t.Errorf("probeSystemResourceUsage() returned success, should have failed")
+	}
+}

pkg/probe/testdata/usage-empty.jsonnet

@@ -0,0 +1,13 @@
+# api/v2/monitor/system/resource/usage?scope=global
+{
+  "http_method":"GET",
+  "results": {},
+  "vdom":"root",
+  "path":"system",
+  "name":"resource",
+  "action":"usage",
+  "status":"success",
+  "serial":"FGVMEVZFNTS3OAC8",
+  "version":"v6.2.4",
+  "build":1112
+}