Skip to content

Commit

Permalink
PMM-12848 Enable currentopmetrics collector in mongodb_exporter. (#3118)
Browse files Browse the repository at this point in the history 
* PMM-12848 Enable currentopmetrics collector in mongodb_exporter.

* PMM-7 fix golint.

* PMM-12848 Add currentopmetrics to scrape config.

* PMM-12848 collect currentopmetrics in low resolution.

* PMM-12848 Fix tests.
  • Loading branch information
@BupycHuk
BupycHuk authored Aug 19, 2024
1 parent 7184dcc commit 49a0e81
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 167 deletions.
216 changes: 51 additions & 165 deletions managed/services/agents/mongodb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,13 @@ import (
"github.com/percona/pmm/version"
)

type collectorArgs struct {
enabled bool
enableParam string
}

var (
// New MongoDB Exporter will be released with PMM agent v2.10.0.
newMongoExporterPMMVersion = version.MustParse("2.9.99")
v2_25_0 = version.MustParse("2.25.0-0")
v2_26_0 = version.MustParse("2.26.0-0")
v2_41_1 = version.MustParse("2.41.1-0")
v2_42_0 = version.MustParse("2.42.0-0")
)

// mongodbExporterConfig returns desired configuration of mongodb_exporter process.
Expand All @@ -50,23 +46,10 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter
listenAddress := getExporterListenAddress(node, exporter)
tdp := exporter.TemplateDelimiters(service)

var args []string
// Starting with PMM 2.10.0, we are shipping the new mongodb_exporter
// Starting with PMM 2.25.0, we change the discovering-mode making it to discover all databases.
// Until now, discovering mode was not working properly and was enabled only if mongodb.collstats-colls=
// was specified in the command line.
// Starting with PMM 2.26.0, we disabled all collectors by default and added flags to enable them.
// Starting with PMM 2.41.1 we added shards collector.
if pmmAgentVersion.Less(v2_26_0) {
args = oldPMMAgentArgs(exporter, tdp, listenAddress, pmmAgentVersion)
} else {
args = v226Args(exporter, tdp, listenAddress, pmmAgentVersion)
}

args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors)
args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion)

if pointer.GetString(exporter.MetricsPath) != "" {
args = append(args, "--web.telemetry-path="+*exporter.MetricsPath)
args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) //nolint:goconst
}

args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true)
Expand Down Expand Up @@ -101,171 +84,74 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter
return res, nil
}

func oldPMMAgentArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string {
// getArgs returns the appropriate arguments based on the PMM agent version.
func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string {
var args []string

switch {
case !pmmAgentVersion.Less(v2_25_0): // >= 2.25
args = v225Args(exporter, tdp, listenAddress)
case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10
args = []string{
"--mongodb.global-conn-pool",
"--compatible-mode",
"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst
case !pmmAgentVersion.Less(v2_25_0): // >= 2.26.0
args = buildBaseArgs(listenAddress, tdp)
args = append(args, "--discovering-mode")

defaultEnabledCollectors := []string{"diagnosticdata", "replicasetstatus"}
collectAll := exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors

if !pmmAgentVersion.Less(v2_26_0) {
defaultEnabledCollectors = []string{}
args = append(args, "--collector.diagnosticdata", "--collector.replicasetstatus")
if collectAll {
args = append(args, "--collector.collstats", "--collector.dbstats", "--collector.indexstats", "--collector.topmetrics")
}
}
default:
args = []string{
"--collect.collection",
"--collect.database",
"--collect.topmetrics",
"--no-collect.connpoolstats",
"--no-collect.indexusage",
"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right,
if !pmmAgentVersion.Less(v2_41_1) && collectAll { // >= 2.41.1
args = append(args, "--collector.shards")
}
if !pmmAgentVersion.Less(v2_42_0) && collectAll { // >= 2.42.0
args = append(args, "--collector.currentopmetrics")
}
}
return args
}

func v226Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string, pmmAgentVersion *version.Parsed) []string {
collectAll := false
if exporter.MongoDBOptions != nil {
collectAll = exporter.MongoDBOptions.EnableAllCollectors
}

collstatsLimit := int32(200)
if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.CollectionsLimit != -1 {
collstatsLimit = exporter.MongoDBOptions.CollectionsLimit
}

collectors := defaultCollectors(collectAll)
args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors)
args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.DisabledCollectors)...)

if !pmmAgentVersion.Less(v2_41_1) { // >= 2.41.1
collectors["shards"] = collectorArgs{
enabled: collectAll,
enableParam: "--collector.shards",
if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 {
args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
if !pmmAgentVersion.Less(v2_26_0) {
args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
}
}
}

for _, collector := range exporter.DisabledCollectors {
col, ok := collectors[strings.ToLower(collector)]
if !ok {
continue
if exporter.MongoDBOptions != nil {
collstatsLimit := int32(200)
if exporter.MongoDBOptions.CollectionsLimit != -1 {
collstatsLimit = exporter.MongoDBOptions.CollectionsLimit
}
args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit))
}
col.enabled = false
collectors[strings.ToLower(collector)] = col
}

args := []string{
"--mongodb.global-conn-pool",
"--compatible-mode",
"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right,
"--discovering-mode",
}

if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 {
args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
}
case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10.0
args = buildBaseArgs(listenAddress, tdp)

if exporter.MongoDBOptions != nil {
args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit))
}

for _, collector := range collectors {
if collector.enabled && collector.enableParam != "" {
args = append(args, collector.enableParam)
default: // < 2.10.0
args = []string{
"--collect.collection",
"--collect.database",
"--collect.topmetrics",
"--no-collect.connpoolstats",
"--no-collect.indexusage",
"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst
}

args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors)
}

return args
}

func v225Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress string) []string {
type collectorArgs struct {
enabled bool
enableParam string
disableParam string
}

collectors := map[string]collectorArgs{
"diagnosticdata": {
enabled: true,
disableParam: "--no-collector.diagnosticdata",
},
"replicasetstatus": {
enabled: true,
disableParam: "--no-collector.replicasetstatus",
},
// disabled until we have better information on the resources usage impact
"dbstats": {
enabled: false,
enableParam: "--collector.dbstats",
},
// disabled until we have better information on the resources usage impact
"topmetrics": {
enabled: false,
enableParam: "--collector.topmetrics",
},
}

for _, collector := range exporter.DisabledCollectors {
col := collectors[strings.ToLower(collector)]
col.enabled = false
collectors[strings.ToLower(collector)] = col
}

args := []string{
func buildBaseArgs(listenAddress string, tdp *models.DelimiterPair) []string {
return []string{
"--mongodb.global-conn-pool",
"--compatible-mode",
"--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right,
"--discovering-mode",
}

if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 {
args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ","))
}

if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.CollectionsLimit != 0 {
args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", exporter.MongoDBOptions.CollectionsLimit))
}

for _, collector := range collectors {
if collector.enabled && collector.enableParam != "" {
args = append(args, collector.enableParam)
}
if !collector.enabled && collector.disableParam != "" {
args = append(args, collector.disableParam)
}
}

return args
}

func defaultCollectors(collectAll bool) map[string]collectorArgs {
return map[string]collectorArgs{
"diagnosticdata": {
enabled: true,
enableParam: "--collector.diagnosticdata",
},
"replicasetstatus": {
enabled: true,
enableParam: "--collector.replicasetstatus",
},
"collstats": {
enabled: collectAll,
enableParam: "--collector.collstats",
},
"dbstats": {
enabled: collectAll,
enableParam: "--collector.dbstats",
},
"indexstats": {
enabled: collectAll,
enableParam: "--collector.indexstats",
},
"topmetrics": {
enabled: collectAll,
enableParam: "--collector.topmetrics",
},
}
}

Expand Down
29 changes: 29 additions & 0 deletions managed/services/agents/mongodb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,35 @@ func TestMongodbExporterConfig2411(t *testing.T) {
require.NoError(t, err)
require.Equal(t, expected.Args, actual.Args)
})

t.Run("Enable all collectors and disable some", func(t *testing.T) {
exporter.MongoDBOptions = &models.MongoDBOptions{
EnableAllCollectors: true,
StatsCollections: []string{"db1.col1.one", "db2.col2", "db3"},
}
exporter.DisabledCollectors = []string{"dbstats", "topmetrics"}

expected.Args = []string{
"--collector.collstats",
"--collector.collstats-limit=0",
"--collector.diagnosticdata",
"--collector.indexstats",
"--collector.replicasetstatus",
"--collector.shards",
"--compatible-mode",
"--discovering-mode",
// this should be here even if limit=0 because it could be used to filter dbstats
// since dbstats is not depending the number of collections present in the db.
"--mongodb.collstats-colls=db1.col1.one,db2.col2,db3",
"--mongodb.global-conn-pool",
"--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3",
"--web.listen-address=0.0.0.0:{{ .listen_port }}",
"--web.config={{ .TextFiles.webConfigPlaceholder }}",
}
actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion)
require.NoError(t, err)
require.Equal(t, expected.Args, actual.Args)
})
}

func TestMongodbExporterConfig(t *testing.T) {
Expand Down
3 changes: 3 additions & 0 deletions managed/services/victoriametrics/scrape_configs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,9 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra
if !params.pmmAgentVersion.Less(version.MustParse("2.41.1-0")) {
defaultCollectors = append(defaultCollectors, "shards")
}
if !params.pmmAgentVersion.Less(version.MustParse("2.42.0-0")) {
defaultCollectors = append(defaultCollectors, "currentopmetrics")
}

lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, defaultCollectors)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions managed/services/victoriametrics/scrape_configs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -758,7 +758,7 @@ func TestScrapeConfig(t *testing.T) {
ScrapeTimeout: scrapeTimeout(s.LR),
MetricsPath: "/metrics",
Params: map[string][]string{
"collect[]": {"collstats", "dbstats", "indexstats", "shards"},
"collect[]": {"collstats", "currentopmetrics", "dbstats", "indexstats", "shards"},
},
HTTPClientConfig: config.HTTPClientConfig{
BasicAuth: &config.BasicAuth{
Expand Down Expand Up @@ -790,7 +790,7 @@ func TestScrapeConfig(t *testing.T) {
node: node,
service: service,
agent: agent,
pmmAgentVersion: version.MustParse("2.41.1"),
pmmAgentVersion: version.MustParse("2.42.0"),
metricsResolution: s,
})
require.NoError(t, err)
Expand Down
17 changes: 17 additions & 0 deletions managed/utils/collectors/collectors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,20 @@ func FilterOutCollectors(prefix string, args, disabledCollectors []string) []str
}
return enabledArgs
}

// DisableDefaultEnabledCollectors returns CLI arguments to disable default enabled collectors based on input.
// DefaultCollectors and disabledCollectors should be collector names without prefix.
// Result will be returned with prefix.
func DisableDefaultEnabledCollectors(prefix string, defaultCollectors []string, disabledCollectors []string) []string {
defaultCollectorsMap := make(map[string]struct{})
for _, defaultCollector := range defaultCollectors {
defaultCollectorsMap[defaultCollector] = struct{}{}
}
args := []string{}
for _, collector := range disabledCollectors {
if _, ok := defaultCollectorsMap[collector]; ok {
args = append(args, fmt.Sprintf("%s%s", prefix, collector))
}
}
return args
}

0 comments on commit 49a0e81

Please sign in to comment.