From 8a5b5a5f627a44fca3edf6c8dd36b96d1943f8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 15 Dec 2023 08:11:59 +0100 Subject: [PATCH 01/15] PMM-12712 Add sharded collection into HR resolution. --- managed/services/victoriametrics/scrape_configs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index e47df63673..bc31502158 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -388,6 +388,7 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap "diagnosticdata", "replicasetstatus", "topmetrics", + "sharded", }) if err != nil { return nil, err From 0979dfe93b8a601194a57601174e05eaba983f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 18 Dec 2023 09:54:37 +0100 Subject: [PATCH 02/15] PMM-12712 Sharded collector. --- managed/services/agents/mongodb.go | 5 +++++ managed/services/victoriametrics/scrape_configs.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 971593fb5f..3ccd05101c 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -252,6 +252,11 @@ func defaultCollectors(collectAll bool) map[string]collectorArgs { enabled: collectAll, enableParam: "--collector.topmetrics", }, + // disabled until we have better information on the resources usage impact + "sharded": { + enabled: collectAll, + enableParam: "--collector.sharded", + }, } } diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index bc31502158..738eb37923 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -388,7 +388,6 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap "diagnosticdata", "replicasetstatus", "topmetrics", - "sharded", }) if err != nil { return nil, err @@ -403,6 +402,7 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap "dbstats", "indexstats", "collstats", + "sharded", }) if err != nil { return nil, err From 35ec24d45a52958f749cc5ebd2efa6b30f86cab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 18 Dec 2023 12:03:24 +0100 Subject: [PATCH 03/15] PMM-12712 Fix tests. --- managed/services/agents/mongodb.go | 5 +++++ managed/services/agents/mongodb_test.go | 3 +++ managed/services/victoriametrics/scrape_configs_test.go | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 3ccd05101c..bd4d86ba16 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -187,6 +187,11 @@ func v225Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress s enabled: false, enableParam: "--collector.topmetrics", }, + // disabled until we have better information on the resources usage impact + "sharded": { + enabled: false, + enableParam: "--collector.sharded", + }, } for _, collector := range exporter.DisabledCollectors { diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 2f11b0940f..1318b5cb88 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -162,6 +162,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--collector.diagnosticdata", "--collector.indexstats", "--collector.replicasetstatus", + "--collector.sharded", "--collector.topmetrics", "--compatible-mode", "--discovering-mode", @@ -188,6 +189,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--collector.diagnosticdata", "--collector.indexstats", "--collector.replicasetstatus", + "--collector.sharded", "--collector.topmetrics", "--compatible-mode", "--discovering-mode", @@ -217,6 +219,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--collector.diagnosticdata", "--collector.indexstats", "--collector.replicasetstatus", + "--collector.sharded", "--collector.topmetrics", "--compatible-mode", "--discovering-mode", diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 29d08272dc..22c5664b4c 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -752,7 +752,7 @@ func TestScrapeConfig(t *testing.T) { ScrapeTimeout: scrapeTimeout(s.LR), MetricsPath: "/metrics", Params: map[string][]string{ - "collect[]": {"collstats", "dbstats", "indexstats"}, + "collect[]": {"collstats", "dbstats", "indexstats", "sharded"}, }, HTTPClientConfig: config.HTTPClientConfig{ BasicAuth: &config.BasicAuth{ From 6c15dddd4d558dec907fe55f99f1dc4e2f64630b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 18 Dec 2023 12:32:09 +0100 Subject: [PATCH 04/15] PMM-12712 Fix VM test. --- managed/services/victoriametrics/victoriametrics_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 413afcf5f2..7012c5e8bc 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -385,6 +385,7 @@ scrape_configs: - collstats - dbstats - indexstats + - sharded scrape_interval: 1m scrape_timeout: 54s metrics_path: /metrics From 16c1f113e41272686083b5107bf7100e9eb5e270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 8 Jan 2024 12:28:56 +0100 Subject: [PATCH 05/15] PMM-12712 Renaming. --- managed/services/agents/mongodb.go | 8 ++++---- managed/services/agents/mongodb_test.go | 6 +++--- managed/services/victoriametrics/scrape_configs.go | 2 +- managed/services/victoriametrics/scrape_configs_test.go | 2 +- managed/services/victoriametrics/victoriametrics_test.go | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index bd4d86ba16..e731b4ccef 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -188,9 +188,9 @@ func v225Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress s enableParam: "--collector.topmetrics", }, // disabled until we have better information on the resources usage impact - "sharded": { + "shards": { enabled: false, - enableParam: "--collector.sharded", + enableParam: "--collector.shards", }, } @@ -258,9 +258,9 @@ func defaultCollectors(collectAll bool) map[string]collectorArgs { enableParam: "--collector.topmetrics", }, // disabled until we have better information on the resources usage impact - "sharded": { + "shards": { enabled: collectAll, - enableParam: "--collector.sharded", + enableParam: "--collector.shards", }, } } diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 1318b5cb88..8d2bd4d0a2 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -162,7 +162,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--collector.diagnosticdata", "--collector.indexstats", "--collector.replicasetstatus", - "--collector.sharded", + "--collector.shards", "--collector.topmetrics", "--compatible-mode", "--discovering-mode", @@ -189,7 +189,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--collector.diagnosticdata", "--collector.indexstats", "--collector.replicasetstatus", - "--collector.sharded", + "--collector.shards", "--collector.topmetrics", "--compatible-mode", "--discovering-mode", @@ -219,7 +219,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--collector.diagnosticdata", "--collector.indexstats", "--collector.replicasetstatus", - "--collector.sharded", + "--collector.shards", "--collector.topmetrics", "--compatible-mode", "--discovering-mode", diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 738eb37923..604bfdd232 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -402,7 +402,7 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap "dbstats", "indexstats", "collstats", - "sharded", + "shards", }) if err != nil { return nil, err diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 22c5664b4c..cd5727f36f 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -752,7 +752,7 @@ func TestScrapeConfig(t *testing.T) { ScrapeTimeout: scrapeTimeout(s.LR), MetricsPath: "/metrics", Params: map[string][]string{ - "collect[]": {"collstats", "dbstats", "indexstats", "sharded"}, + "collect[]": {"collstats", "dbstats", "indexstats", "shards"}, }, HTTPClientConfig: config.HTTPClientConfig{ BasicAuth: &config.BasicAuth{ diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 7012c5e8bc..3103cd807f 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -385,7 +385,7 @@ scrape_configs: - collstats - dbstats - indexstats - - sharded + - shards scrape_interval: 1m scrape_timeout: 54s metrics_path: /metrics From 513da6bbbc6bcbba914049687ff752a69ced18bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 9 Jan 2024 10:20:12 +0100 Subject: [PATCH 06/15] PMM-12712 Remove unclear comments. --- managed/services/agents/mongodb.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index e731b4ccef..7afca7b8dd 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -237,27 +237,22 @@ func defaultCollectors(collectAll bool) map[string]collectorArgs { enabled: true, enableParam: "--collector.replicasetstatus", }, - // disabled until we have better information on the resources usage impact "collstats": { enabled: collectAll, enableParam: "--collector.collstats", }, - // disabled until we have better information on the resources usage impact "dbstats": { enabled: collectAll, enableParam: "--collector.dbstats", }, - // disabled until we have better information on the resources usage impact "indexstats": { enabled: collectAll, enableParam: "--collector.indexstats", }, - // disabled until we have better information on the resources usage impact "topmetrics": { enabled: collectAll, enableParam: "--collector.topmetrics", }, - // disabled until we have better information on the resources usage impact "shards": { enabled: collectAll, enableParam: "--collector.shards", From 28bb907aa73f6d1ac4cae6df4acdc358ec2ef3b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 10:09:03 +0100 Subject: [PATCH 07/15] PMM-12712 Check for 2.41.1 version. --- managed/services/agents/mongodb.go | 17 ++- managed/services/agents/mongodb_test.go | 152 ++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 9 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 7afca7b8dd..786c652229 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -40,6 +40,7 @@ var ( newMongoExporterPMMVersion = version.MustParse("2.9.99") v2_24_99 = version.MustParse("2.24.99") v2_25_99 = version.MustParse("2.25.99") + v2_41_01 = version.MustParse("2.41.01") ) // mongodbExporterConfig returns desired configuration of mongodb_exporter process. @@ -52,9 +53,16 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter var args []string // Starting with PMM 2.10.0, we are shipping the new mongodb_exporter // Starting with PMM 2.25.0, we change the discovering-mode making it to discover all databases. + // Starting with PMM 2.41.1 we added shards collector. // Until now, discovering mode was not working properly and was enabled only if mongodb.collstats-colls= // was specified in the command line. switch { + case !pmmAgentVersion.Less(v2_41_01): // >= 2.41.1 + args = v226Args(exporter, tdp, listenAddress) + + if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors { + args = append(args, "--collector.shards") + } case !pmmAgentVersion.Less(v2_25_99): // >= 2.26 args = v226Args(exporter, tdp, listenAddress) case !pmmAgentVersion.Less(v2_24_99): // >= 2.25 @@ -187,11 +195,6 @@ func v225Args(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress s enabled: false, enableParam: "--collector.topmetrics", }, - // disabled until we have better information on the resources usage impact - "shards": { - enabled: false, - enableParam: "--collector.shards", - }, } for _, collector := range exporter.DisabledCollectors { @@ -253,10 +256,6 @@ func defaultCollectors(collectAll bool) map[string]collectorArgs { enabled: collectAll, enableParam: "--collector.topmetrics", }, - "shards": { - enabled: collectAll, - enableParam: "--collector.shards", - }, } } diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 8d2bd4d0a2..8accc04e9e 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -148,6 +148,155 @@ func TestMongodbExporterConfig226(t *testing.T) { require.Equal(t, expected.Args, actual.Args) }) + t.Run("Enabling all collectors with non zero limit", func(t *testing.T) { + exporter.MongoDBOptions = &models.MongoDBOptions{ + StatsCollections: []string{"col1", "col2", "col3"}, + CollectionsLimit: 79014, + EnableAllCollectors: true, + } + + expected.Args = []string{ + "--collector.collstats", + "--collector.collstats-limit=79014", + "--collector.dbstats", + "--collector.diagnosticdata", + "--collector.indexstats", + "--collector.replicasetstatus", + "--collector.topmetrics", + "--compatible-mode", + "--discovering-mode", + "--mongodb.collstats-colls=col1,col2,col3", + "--mongodb.global-conn-pool", + "--mongodb.indexstats-colls=col1,col2,col3", + "--web.listen-address=0.0.0.0:{{ .listen_port }}", + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) + require.NoError(t, err) + require.Equal(t, expected.Args, actual.Args) + }) + + t.Run("Enabling all collectors", func(t *testing.T) { + exporter.MongoDBOptions = &models.MongoDBOptions{ + EnableAllCollectors: true, + StatsCollections: []string{"db1.col1.one", "db2.col2", "db3"}, + } + + expected.Args = []string{ + "--collector.collstats", + "--collector.collstats-limit=0", + "--collector.dbstats", + "--collector.diagnosticdata", + "--collector.indexstats", + "--collector.replicasetstatus", + "--collector.topmetrics", + "--compatible-mode", + "--discovering-mode", + // this should be here even if limit=0 because it could be used to filter dbstats + // since dbstats is not depending the number of collections present in the db. + "--mongodb.collstats-colls=db1.col1.one,db2.col2,db3", + "--mongodb.global-conn-pool", + "--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3", + "--web.listen-address=0.0.0.0:{{ .listen_port }}", + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) + require.NoError(t, err) + require.Equal(t, expected.Args, actual.Args) + }) + + t.Run("collstats-limit=-1 -> automatically set the limit", func(t *testing.T) { + exporter.MongoDBOptions = &models.MongoDBOptions{ + EnableAllCollectors: true, + StatsCollections: []string{"db1.col1.one", "db2.col2", "db3"}, + CollectionsLimit: -1, + } + + expected.Args = []string{ + "--collector.collstats", + "--collector.collstats-limit=200", // 200 is the default for auto-set + "--collector.dbstats", + "--collector.diagnosticdata", + "--collector.indexstats", + "--collector.replicasetstatus", + "--collector.topmetrics", + "--compatible-mode", + "--discovering-mode", + "--mongodb.collstats-colls=db1.col1.one,db2.col2,db3", + "--mongodb.global-conn-pool", + "--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3", + "--web.listen-address=0.0.0.0:{{ .listen_port }}", + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) + require.NoError(t, err) + require.Equal(t, expected.Args, actual.Args) + }) +} + +func TestMongodbExporterConfig2411(t *testing.T) { + pmmAgentVersion := version.MustParse("2.41.1") + node := &models.Node{ + Address: "1.2.3.4", + } + mongodb := &models.Service{ + Address: pointer.ToString("1.2.3.4"), + Port: pointer.ToUint16(27017), + } + exporter := &models.Agent{ + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) + expected := &agentpb.SetStateRequest_AgentProcess{ + Type: inventorypb.AgentType_MONGODB_EXPORTER, + TemplateLeftDelim: "{{", + TemplateRightDelim: "}}", + Args: []string{ + "--collector.diagnosticdata", + "--collector.replicasetstatus", + "--compatible-mode", + "--discovering-mode", + "--mongodb.global-conn-pool", + "--web.listen-address=0.0.0.0:{{ .listen_port }}", + "--web.config={{ .TextFiles.webConfigPlaceholder }}", + }, + Env: []string{ + "MONGODB_URI=mongodb://username:s3cur3%20p%40$$w0r4.@1.2.3.4:27017/?connectTimeoutMS=1000&directConnection=true&serverSelectionTimeoutMS=1000", + }, + RedactWords: []string{"s3cur3 p@$$w0r4.", "agent-password"}, + TextFiles: map[string]string{ + "webConfigPlaceholder": "basic_auth_users:\n pmm: agent-password\n", + }, + } + require.NoError(t, err) + requireNoDuplicateFlags(t, actual.Args) + require.Equal(t, expected.Args, actual.Args) + require.Equal(t, expected.Env, actual.Env) + require.Equal(t, expected, actual) + + t.Run("Having collstats limit", func(t *testing.T) { + exporter.MongoDBOptions = &models.MongoDBOptions{ + StatsCollections: []string{"col1", "col2", "col3"}, + CollectionsLimit: 79014, + } + expected.Args = []string{ + "--collector.collstats-limit=79014", + "--collector.diagnosticdata", + "--collector.replicasetstatus", + "--compatible-mode", + "--discovering-mode", + "--mongodb.collstats-colls=col1,col2,col3", + "--mongodb.global-conn-pool", + "--mongodb.indexstats-colls=col1,col2,col3", + "--web.listen-address=0.0.0.0:{{ .listen_port }}", + "--web.config={{ .TextFiles.webConfigPlaceholder }}", + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) + require.NoError(t, err) + require.Equal(t, expected.Args, actual.Args) + }) + t.Run("Enabling all collectors with non zero limit", func(t *testing.T) { exporter.MongoDBOptions = &models.MongoDBOptions{ StatsCollections: []string{"col1", "col2", "col3"}, @@ -170,6 +319,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--mongodb.global-conn-pool", "--mongodb.indexstats-colls=col1,col2,col3", "--web.listen-address=0.0.0.0:{{ .listen_port }}", + "--web.config={{ .TextFiles.webConfigPlaceholder }}", } actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) require.NoError(t, err) @@ -199,6 +349,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--mongodb.global-conn-pool", "--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3", "--web.listen-address=0.0.0.0:{{ .listen_port }}", + "--web.config={{ .TextFiles.webConfigPlaceholder }}", } actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) require.NoError(t, err) @@ -227,6 +378,7 @@ func TestMongodbExporterConfig226(t *testing.T) { "--mongodb.global-conn-pool", "--mongodb.indexstats-colls=db1.col1.one,db2.col2,db3", "--web.listen-address=0.0.0.0:{{ .listen_port }}", + "--web.config={{ .TextFiles.webConfigPlaceholder }}", } actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) require.NoError(t, err) From a09134a4bdf2d7c3f35a10abfeaa36dc52fc46c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 10:14:23 +0100 Subject: [PATCH 08/15] PMM-12712 Small change. --- managed/services/agents/mongodb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 786c652229..a75ae50a1e 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -40,7 +40,7 @@ var ( newMongoExporterPMMVersion = version.MustParse("2.9.99") v2_24_99 = version.MustParse("2.24.99") v2_25_99 = version.MustParse("2.25.99") - v2_41_01 = version.MustParse("2.41.01") + v2_41_1 = version.MustParse("2.41.1") ) // mongodbExporterConfig returns desired configuration of mongodb_exporter process. @@ -57,7 +57,7 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter // Until now, discovering mode was not working properly and was enabled only if mongodb.collstats-colls= // was specified in the command line. switch { - case !pmmAgentVersion.Less(v2_41_01): // >= 2.41.1 + case !pmmAgentVersion.Less(v2_41_1): // >= 2.41.1 args = v226Args(exporter, tdp, listenAddress) if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors { From 9769e951f54d62ca988860615403bb2ce7d3ab93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 10:28:28 +0100 Subject: [PATCH 09/15] PMM-12712 Version check for scrape config for VM. --- managed/services/victoriametrics/scrape_configs.go | 10 +++++++--- .../services/victoriametrics/scrape_configs_test.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 604bfdd232..02a6d05bb5 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -398,12 +398,16 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap r = append(r, hr) } if params.agent.MongoDBOptions != nil && params.agent.MongoDBOptions.EnableAllCollectors { - lr, err := scrapeConfigForStandardExporter("lr", s.LR, params, []string{ + defaultCollectors := []string{ "dbstats", "indexstats", "collstats", - "shards", - }) + } + if params.pmmAgentVersion != nil && !params.pmmAgentVersion.Less(version.MustParse("2.41.1")) { + defaultCollectors = append(defaultCollectors, "shards") + } + + lr, err := scrapeConfigForStandardExporter("lr", s.LR, params, defaultCollectors) if err != nil { return nil, err } diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index cd5727f36f..c8ac503da9 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -784,7 +784,7 @@ func TestScrapeConfig(t *testing.T) { node: node, service: service, agent: agent, - pmmAgentVersion: version.MustParse("2.26.0"), + pmmAgentVersion: version.MustParse("2.41.1"), }) require.NoError(t, err) require.Len(t, actual, len(expected)) From a28a6cb510ecb3f1741203f2aedbdf7928201785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 10:40:02 +0100 Subject: [PATCH 10/15] PMM-12712 VM test. --- managed/services/victoriametrics/victoriametrics_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 3103cd807f..413afcf5f2 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -385,7 +385,6 @@ scrape_configs: - collstats - dbstats - indexstats - - shards scrape_interval: 1m scrape_timeout: 54s metrics_path: /metrics From 5885134ec4f1ba0931d034af1174bf2eae271c41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 12:36:40 +0100 Subject: [PATCH 11/15] PMM-12712 Fix version check for QA versions. --- managed/services/agents/mongodb.go | 6 +-- managed/services/agents/mongodb_test.go | 38 +++++++++++++++++++ .../victoriametrics/scrape_configs.go | 2 +- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index a75ae50a1e..1a0e193a46 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -40,7 +40,7 @@ var ( newMongoExporterPMMVersion = version.MustParse("2.9.99") v2_24_99 = version.MustParse("2.24.99") v2_25_99 = version.MustParse("2.25.99") - v2_41_1 = version.MustParse("2.41.1") + v2_41_2 = version.MustParse("2.41.2") ) // mongodbExporterConfig returns desired configuration of mongodb_exporter process. @@ -53,11 +53,11 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter var args []string // Starting with PMM 2.10.0, we are shipping the new mongodb_exporter // Starting with PMM 2.25.0, we change the discovering-mode making it to discover all databases. - // Starting with PMM 2.41.1 we added shards collector. // Until now, discovering mode was not working properly and was enabled only if mongodb.collstats-colls= // was specified in the command line. + // Starting with PMM 2.41.1 we added shards collector. switch { - case !pmmAgentVersion.Less(v2_41_1): // >= 2.41.1 + case strings.Contains(pmmAgentVersion.String(), "2.41.1") || !pmmAgentVersion.Less(v2_41_2): // >= 2.41.1 args = v226Args(exporter, tdp, listenAddress) if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors { diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 8accc04e9e..89dddbff25 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -231,6 +231,44 @@ func TestMongodbExporterConfig226(t *testing.T) { }) } +func Test241PatchVersion(t *testing.T) { + versions := map[string]bool{ + "2.43.0": true, + "2.42.2": true, + "2.41.1": true, + "2.41.1-HEAD-xyz": true, + "2.41.0": false, + } + for pmmVersion, shardsCompatibility := range versions { + pmmAgentVersion := version.MustParse(pmmVersion) + node := &models.Node{ + Address: "1.2.3.4", + } + mongodb := &models.Service{ + Address: pointer.ToString("1.2.3.4"), + Port: pointer.ToUint16(27017), + } + exporter := &models.Agent{ + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + } + exporter.MongoDBOptions = &models.MongoDBOptions{ + EnableAllCollectors: true, + } + actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) + require.NoError(t, err) + if shardsCompatibility { + require.Contains(t, actual.Args, "--collector.shards") + } else { + require.NotContains(t, actual.Args, "--collector.shards") + } + + } +} + func TestMongodbExporterConfig2411(t *testing.T) { pmmAgentVersion := version.MustParse("2.41.1") node := &models.Node{ diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 02a6d05bb5..3da84f7f00 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -403,7 +403,7 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap "indexstats", "collstats", } - if params.pmmAgentVersion != nil && !params.pmmAgentVersion.Less(version.MustParse("2.41.1")) { + if params.pmmAgentVersion != nil && (strings.Contains(params.pmmAgentVersion.String(), "2.41.1") || !params.pmmAgentVersion.Less(version.MustParse("2.41.2"))) { defaultCollectors = append(defaultCollectors, "shards") } From 8e0532467af923c48b5cb05bc723d7556c84641d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 12:39:14 +0100 Subject: [PATCH 12/15] PMM-12712 Small renaming. --- managed/services/agents/mongodb_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 89dddbff25..388818b37d 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -232,14 +232,14 @@ func TestMongodbExporterConfig226(t *testing.T) { } func Test241PatchVersion(t *testing.T) { - versions := map[string]bool{ + shardsCompatibility := map[string]bool{ "2.43.0": true, "2.42.2": true, "2.41.1": true, "2.41.1-HEAD-xyz": true, "2.41.0": false, } - for pmmVersion, shardsCompatibility := range versions { + for pmmVersion, shardsSupported := range shardsCompatibility { pmmAgentVersion := version.MustParse(pmmVersion) node := &models.Node{ Address: "1.2.3.4", @@ -260,7 +260,7 @@ func Test241PatchVersion(t *testing.T) { } actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) require.NoError(t, err) - if shardsCompatibility { + if shardsSupported { require.Contains(t, actual.Args, "--collector.shards") } else { require.NotContains(t, actual.Args, "--collector.shards") From 19ea041c52eac8c80fcaefbc3d61bcef76cf3d4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 12:40:41 +0100 Subject: [PATCH 13/15] PMM-12712 Few more versions in test. --- managed/services/agents/mongodb_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 388818b37d..5375053a24 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -237,7 +237,9 @@ func Test241PatchVersion(t *testing.T) { "2.42.2": true, "2.41.1": true, "2.41.1-HEAD-xyz": true, + "2.41.0-HEAD-abc": false, "2.41.0": false, + "2.26.1": false, } for pmmVersion, shardsSupported := range shardsCompatibility { pmmAgentVersion := version.MustParse(pmmVersion) From 3b6463234467d8321965d72b7c8290cabb1ad88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 12:46:49 +0100 Subject: [PATCH 14/15] PMM-12712 Lint. --- managed/services/agents/mongodb_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 5375053a24..0d9c0dd534 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -267,7 +267,6 @@ func Test241PatchVersion(t *testing.T) { } else { require.NotContains(t, actual.Args, "--collector.shards") } - } } From 6121ff273ca6f2f6a0b5d949ce237edc5af518a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Wed, 10 Jan 2024 13:48:01 +0100 Subject: [PATCH 15/15] PMM-12712 Simplify check. --- managed/services/agents/mongodb.go | 4 +- managed/services/agents/mongodb_test.go | 39 ------------------- .../victoriametrics/scrape_configs.go | 2 +- 3 files changed, 3 insertions(+), 42 deletions(-) diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index 1a0e193a46..37cac807af 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -40,7 +40,7 @@ var ( newMongoExporterPMMVersion = version.MustParse("2.9.99") v2_24_99 = version.MustParse("2.24.99") v2_25_99 = version.MustParse("2.25.99") - v2_41_2 = version.MustParse("2.41.2") + v2_41_1 = version.MustParse("2.41.1-0") ) // mongodbExporterConfig returns desired configuration of mongodb_exporter process. @@ -57,7 +57,7 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter // was specified in the command line. // Starting with PMM 2.41.1 we added shards collector. switch { - case strings.Contains(pmmAgentVersion.String(), "2.41.1") || !pmmAgentVersion.Less(v2_41_2): // >= 2.41.1 + case !pmmAgentVersion.Less(v2_41_1): // >= 2.41.1 args = v226Args(exporter, tdp, listenAddress) if exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors { diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index 0d9c0dd534..8accc04e9e 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -231,45 +231,6 @@ func TestMongodbExporterConfig226(t *testing.T) { }) } -func Test241PatchVersion(t *testing.T) { - shardsCompatibility := map[string]bool{ - "2.43.0": true, - "2.42.2": true, - "2.41.1": true, - "2.41.1-HEAD-xyz": true, - "2.41.0-HEAD-abc": false, - "2.41.0": false, - "2.26.1": false, - } - for pmmVersion, shardsSupported := range shardsCompatibility { - pmmAgentVersion := version.MustParse(pmmVersion) - node := &models.Node{ - Address: "1.2.3.4", - } - mongodb := &models.Service{ - Address: pointer.ToString("1.2.3.4"), - Port: pointer.ToUint16(27017), - } - exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), - } - exporter.MongoDBOptions = &models.MongoDBOptions{ - EnableAllCollectors: true, - } - actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) - require.NoError(t, err) - if shardsSupported { - require.Contains(t, actual.Args, "--collector.shards") - } else { - require.NotContains(t, actual.Args, "--collector.shards") - } - } -} - func TestMongodbExporterConfig2411(t *testing.T) { pmmAgentVersion := version.MustParse("2.41.1") node := &models.Node{ diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index 3da84f7f00..6c05e6c716 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -403,7 +403,7 @@ func scrapeConfigsForMongoDBExporter(s *models.MetricsResolutions, params *scrap "indexstats", "collstats", } - if params.pmmAgentVersion != nil && (strings.Contains(params.pmmAgentVersion.String(), "2.41.1") || !params.pmmAgentVersion.Less(version.MustParse("2.41.2"))) { + if params.pmmAgentVersion != nil && !params.pmmAgentVersion.Less(version.MustParse("2.41.1-0")) { defaultCollectors = append(defaultCollectors, "shards") }