[HA Agent] Add datadog.agent.ha_agent.integration_runs

comp/core/gui/guiimpl/checks.go

@@ -134,7 +134,7 @@ func runCheckOnce(w http.ResponseWriter, r *http.Request, ac autodiscovery.Compo
 		err := ch.Run()
 		warnings := ch.GetWarnings()
 		sStats, _ := ch.GetSenderStats()
-		s.Add(time.Since(t0), err, warnings, sStats)
+		s.Add(time.Since(t0), err, warnings, sStats, nil)
 
 		// Without a small delay some of the metrics will not show up
 		time.Sleep(100 * time.Millisecond)

comp/haagent/def/component.go

@@ -25,4 +25,7 @@ type Component interface {
 
 	// ShouldRunIntegration returns true if the integration should be run
 	ShouldRunIntegration(integrationName string) bool
+
+	// IsHaIntegration return true if it's an HA integration.
+	IsHaIntegration(integrationName string) bool
 }

comp/haagent/impl/haagent.go

@@ -79,6 +79,11 @@ func (h *haAgentImpl) ShouldRunIntegration(integrationName string) bool {
 	return true
 }
 
+// IsHaIntegration return true if it's an HA integration.
+func (h *haAgentImpl) IsHaIntegration(integrationName string) bool {
+	return validHaIntegrations[integrationName]
+}
+
 func (h *haAgentImpl) onHaAgentUpdate(updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) {
 	h.log.Debugf("Updates received: count=%d", len(updates))
 

comp/haagent/impl/haagent_test.go

@@ -283,6 +283,43 @@ func Test_haAgentImpl_ShouldRunIntegration(t *testing.T) {
 	}
 }
 
+func Test_haAgentImpl_IsIntegration(t *testing.T) {
+	testAgentHostname := "my-agent-hostname"
+	tests := []struct {
+		name                  string
+		leader                string
+		agentConfigs          map[string]interface{}
+		expectIsHaIntegration map[string]bool
+	}{
+		{
+			name: "base case",
+			// should run HA-integrations
+			// should run "non HA integrations"
+			agentConfigs: map[string]interface{}{
+				"hostname": testAgentHostname,
+			},
+			leader: testAgentHostname,
+			expectIsHaIntegration: map[string]bool{
+				"snmp":                true,
+				"cisco_aci":           true,
+				"cisco_sdwan":         true,
+				"network_path":        true,
+				"unknown_integration": false,
+				"cpu":                 false,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			haAgent := newTestHaAgentComponent(t, tt.agentConfigs)
+
+			for integrationName, shouldRun := range tt.expectIsHaIntegration {
+				assert.Equalf(t, shouldRun, haAgent.Comp.IsHaIntegration(integrationName), "fail for integration: "+integrationName)
+			}
+		})
+	}
+}
+
 func Test_haAgentImpl_resetAgentState(t *testing.T) {
 	// GIVEN
 	haAgent := newTestHaAgentComponent(t, nil)

comp/haagent/mock/mock.go

@@ -52,6 +52,10 @@ func (m *mockHaAgent) ShouldRunIntegration(_ string) bool {
 	return true
 }
 
+func (m *mockHaAgent) IsHaIntegration(_ string) bool {
+	return true
+}
+
 // Component is the component type.
 type Component interface {
 	haagent.Component

pkg/cli/subcommands/check/command.go

@@ -648,7 +648,7 @@ func runCheck(cliParams *cliParams, c check.Check, _ aggregator.Demultiplexer) *
 		err := c.Run()
 		warnings := c.GetWarnings()
 		sStats, _ := c.GetSenderStats()
-		s.Add(time.Since(t0), err, warnings, sStats)
+		s.Add(time.Since(t0), err, warnings, sStats, nil)
 		if pause > 0 && i < times-1 {
 			time.Sleep(time.Duration(pause) * time.Millisecond)
 		}

pkg/collector/check/stats/stats.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/mitchellh/mapstructure"
 
+	haagent "github.com/DataDog/datadog-agent/comp/haagent/def"
 	checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id"
 	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
 	"github.com/DataDog/datadog-agent/pkg/config/utils"
@@ -55,6 +56,13 @@ var (
 		"delay",
 		[]string{"check_name"},
 		"Check start time delay relative to the previous check run")
+	tlmHaAgentIntegrationRuns = telemetry.NewCounterWithOpts(
+		"ha_agent",
+		"integration_runs",
+		[]string{"integration", "config_id"},
+		"Tracks number of HA integrations runs.",
+		telemetry.Options{DefaultMetric: true},
+	)
 )
 
 // SenderStats contains statistics showing the count of various types of telemetry sent by a check sender
@@ -161,7 +169,7 @@ func NewStats(c StatsCheck) *Stats {
 }
 
 // Add tracks a new execution time
-func (cs *Stats) Add(t time.Duration, err error, warnings []error, metricStats SenderStats) {
+func (cs *Stats) Add(t time.Duration, err error, warnings []error, metricStats SenderStats, haagent haagent.Component) {
 	cs.m.Lock()
 	defer cs.m.Unlock()
 
@@ -249,6 +257,9 @@ func (cs *Stats) Add(t time.Duration, err error, warnings []error, metricStats S
 		cs.TotalEventPlatformEvents[k] = cs.TotalEventPlatformEvents[k] + v
 		cs.EventPlatformEvents[k] = v
 	}
+	if haagent != nil && haagent.Enabled() && haagent.IsHaIntegration(cs.CheckName) {
+		tlmHaAgentIntegrationRuns.Inc(cs.CheckName, pkgconfigsetup.Datadog().GetString("config_id"))
+	}
 }
 
 // SetStateCancelling sets the check stats to be in a cancelling state

pkg/collector/runner/expvars/expvars.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/mohae/deepcopy"
 
+	haagent "github.com/DataDog/datadog-agent/comp/haagent/def"
 	"github.com/DataDog/datadog-agent/pkg/collector/check"
 	checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id"
 	checkstats "github.com/DataDog/datadog-agent/pkg/collector/check/stats"
@@ -105,12 +106,12 @@ func GetCheckStats() map[string]map[checkid.ID]*checkstats.Stats {
 }
 
 // AddCheckStats adds runtime stats to the check's expvars
-func AddCheckStats(
-	c check.Check,
+func AddCheckStats(c check.Check,
 	execTime time.Duration,
 	err error,
 	warnings []error,
 	mStats checkstats.SenderStats,
+	haagent haagent.Component,
 ) {
 
 	var s *checkstats.Stats
@@ -133,7 +134,7 @@ func AddCheckStats(
 		stats[c.ID()] = s
 	}
 
-	s.Add(execTime, err, warnings, mStats)
+	s.Add(execTime, err, warnings, mStats, haagent)
 }
 
 // RemoveCheckStats removes a check from the check stats map

pkg/collector/runner/expvars/expvars_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock"
 	checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id"
 	"github.com/DataDog/datadog-agent/pkg/collector/check/stats"
 	"github.com/DataDog/datadog-agent/pkg/collector/check/stub"
@@ -171,7 +172,7 @@ func TestExpvarsReset(t *testing.T) {
 			testCheck := newTestCheck(checkID)
 
 			for runIdx := 0; runIdx < numCheckRuns; runIdx++ {
-				AddCheckStats(testCheck, 12345, nil, []error{}, stats.SenderStats{})
+				AddCheckStats(testCheck, 12345, nil, []error{}, stats.SenderStats{}, haagentmock.NewMockHaAgent())
 			}
 		}
 	}
@@ -243,7 +244,7 @@ func TestExpvarsCheckStats(t *testing.T) {
 
 					<-start
 
-					AddCheckStats(testCheck, duration, err, warnings, expectedStats)
+					AddCheckStats(testCheck, duration, err, warnings, expectedStats, haagentmock.NewMockHaAgent())
 
 					actualStats, found := CheckStats(testCheck.ID())
 					require.True(t, found)
@@ -319,7 +320,7 @@ func TestExpvarsGetChecksStatsClone(t *testing.T) {
 			testCheck := newTestCheck(checkID)
 
 			for runIdx := 0; runIdx < numCheckRuns; runIdx++ {
-				AddCheckStats(testCheck, 12345, nil, []error{}, stats.SenderStats{})
+				AddCheckStats(testCheck, 12345, nil, []error{}, stats.SenderStats{}, haagentmock.NewMockHaAgent())
 			}
 		}
 	}
@@ -424,7 +425,7 @@ func TestGetCheckStatsRace(t *testing.T) {
 
 					warnings := []error{errors.New("error1"), errors.New("error2"), errors.New("error3")}
 					for runIdx := 0; runIdx < numCheckRuns; runIdx++ {
-						AddCheckStats(testCheck, 12345, nil, warnings, stats.SenderStats{})
+						AddCheckStats(testCheck, 12345, nil, warnings, stats.SenderStats{}, haagentmock.NewMockHaAgent())
 					}
 				}
 			}

pkg/collector/worker/check_logger_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	haagentmock "github.com/DataDog/datadog-agent/comp/haagent/mock"
 	"github.com/DataDog/datadog-agent/pkg/collector/check"
 	checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id"
 	"github.com/DataDog/datadog-agent/pkg/collector/check/stats"
@@ -32,7 +33,7 @@ func newTestCheck(id string) *stubCheck {
 }
 
 func addExpvarsCheckStats(c check.Check) {
-	expvars.AddCheckStats(c, 0, nil, nil, stats.SenderStats{})
+	expvars.AddCheckStats(c, 0, nil, nil, stats.SenderStats{}, haagentmock.NewMockHaAgent())
 }
 
 func setUp() {

pkg/collector/worker/worker.go

@@ -213,7 +213,7 @@ func (w *Worker) Run() {
 			// otherwise only do so if the check is in the scheduler
 			if w.shouldAddCheckStatsFunc(check.ID()) {
 				sStats, _ := check.GetSenderStats()
-				expvars.AddCheckStats(check, time.Since(checkStartTime), checkErr, checkWarnings, sStats)
+				expvars.AddCheckStats(check, time.Since(checkStartTime), checkErr, checkWarnings, sStats, w.haAgent)
 			}
 		}