From fba392897b8cb67a240ae394b8057cbd1c39dd29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= <62988319+JiriCtvrtka@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:41:15 +0100 Subject: [PATCH] PMM-5086-12634 Agent model refactor. (#3338) * PMM-5086-12634 Structure draft. * PMM-5086-12634 Some changes. * PMM-5086-12634 Another changes. * PMM-5086-12634 Another changes. * PMM-5086-12634 Another changes. * PMM-5086-12634 Revert changed makefile. * PMM-5086-12634 Changes. * PMM-5086-12634 Fix for encryption. * PMM-5086-12634 Database changes. * PMM-5086-12634 Fix DB JSONB conversion. * PMM-5086-12634 Missed type. * PMM-5086-12634 Typo in mongo options renaming. * PMM-5086-12634 Changes. * PMM-5086-12634 Nil check. * PMM-5086-12634 Migrations, nil checks. * PMM-5086-12634 Two models of encryption, db changes. * PMM-5086-12634 Format. * PMM-5086-12634 Fix double decrypt bug. * PMM-508612634 Helpers push metrics fix. * PMM-12634 Expect false in exporter options. * Revert "PMM-5086-12634 Fix double decrypt bug." This reverts commit e72961b7f5e42c21d4888e0e5aafa1a7aa85e136. * PMM-5086-12634 Handle nil in scrape configs. * PMM-5086-12634 Fix encryption rotation test. * PMM-5086-12634 Mongo options nil. * PMM-5086-12634 Converters. * PMM-5086-12634 MySQL fix. * PMM-5086-12634 Fix proxysql nil. * PMM-5086-12634 Fix roster test. * PMM-5086-12634 Fix for TextFiles and templateParams. * PMM-5086-12634 Fix PG test. * PMM-5086-12634 Fix wrong annotation. * PMM-5086-12634 Fix lint about receivers. * PMM-5086-12634 Migration changes. * PMM-5086-12634 Correct push_metrics fields naming. * PMM-5086-12634 Lint. * PMM-5086-12634 Fix migrations, helpers test. * PMM-5086-12634 Fix listen address test. * PMM-5086-12634 Fix agent management tests. * PMM-5086-12634 Mongo nil handling. * PMM-5086-12634 MySQL nil handling. * PMM-5086 Fix Node nil handling. * PMM-5086-12634 Fix Postgres nil handling. * PMM-5086-12634 Fix RDS nil handling. * PMM-5086-12634 Fix scraped configs. * PMM-5086-12634 Lint. * PMM-5086-12634 Fix another tests. * PMM-5086-12634 Format. * PMM-5086-12634 Fix for another tests. * PMM-5086-12634 CreateAgent options preparation. * PMM-5086-12634 Fix another tests. * PMM-5086-12634 PG test. * PMM-5086-12634 Fix PG test suite. * PMM-5086-12634 Fix proxy exporter. * PMM-5086-12634 Another MySQL fix. * PMM-5086-12634 Cleanup. * PMM-5086-12634 API tests/logic fix. * PMM-5086-12634 Another cleanup. * PMM-5086-12634 Remove migration test to avoid two encryption models. * PMM-5086-12634 Remove options assignment in case of nil option. * PMM-5086-12634 Remove the forgotten comments. --- managed/models/agent_helpers.go | 208 ++++++------ managed/models/agent_helpers_test.go | 298 +++++++++++------- managed/models/agent_model.go | 244 ++++++++------ managed/models/agent_model_reform.go | 123 ++------ managed/models/agent_model_test.go | 197 +++++++----- managed/models/database.go | 101 +++++- managed/models/database_test.go | 119 ++----- managed/models/encryption_helpers.go | 141 +++++---- managed/services/agents/agents.go | 31 +- managed/services/agents/agents_test.go | 20 +- managed/services/agents/connection_checker.go | 2 +- managed/services/agents/mongodb.go | 36 +-- managed/services/agents/mongodb_test.go | 115 ++++--- managed/services/agents/mysql.go | 24 +- managed/services/agents/mysql_test.go | 44 +-- managed/services/agents/node.go | 14 +- managed/services/agents/node_test.go | 33 +- managed/services/agents/postgresql.go | 31 +- managed/services/agents/postgresql_test.go | 97 ++++-- managed/services/agents/proxysql.go | 8 +- managed/services/agents/proxysql_test.go | 31 +- managed/services/agents/rds.go | 8 +- managed/services/agents/rds_test.go | 26 +- managed/services/agents/registry.go | 12 +- managed/services/agents/roster_test.go | 17 +- .../services/agents/service_info_broker.go | 6 +- managed/services/agents/state.go | 2 +- managed/services/converters.go | 107 +++---- .../encryption/encryption_rotation.go | 4 +- .../encryption/encryption_rotation_test.go | 9 +- managed/services/inventory/agents.go | 254 ++++++++------- managed/services/management/agent.go | 109 ++++--- managed/services/management/agent_test.go | 154 +++++++-- managed/services/management/azure_database.go | 32 +- managed/services/management/mongodb.go | 44 +-- managed/services/management/mysql.go | 82 ++--- managed/services/management/postgresql.go | 70 ++-- managed/services/management/proxysql.go | 24 +- managed/services/management/rds.go | 97 +++--- .../services/victoriametrics/prometheus.go | 14 +- .../victoriametrics/scrape_configs.go | 38 +-- .../victoriametrics/scrape_configs_test.go | 130 ++++---- .../victoriametrics/victoriametrics_test.go | 8 +- 43 files changed, 1799 insertions(+), 1365 deletions(-) diff --git a/managed/models/agent_helpers.go b/managed/models/agent_helpers.go index 4be799dee7..f588fd4f17 100644 --- a/managed/models/agent_helpers.go +++ b/managed/models/agent_helpers.go @@ -31,6 +31,11 @@ import ( "github.com/percona/pmm/version" ) +const ( + pushMetricsTrue = "((exporter_options ? 'push_metrics') AND (exporter_options->>'push_metrics')::boolean = true)" + pushMetricsFalse = "(NOT (exporter_options ? 'push_metrics') OR (exporter_options->>'push_metrics')::boolean = false)" +) + // MySQLOptionsParams contains methods to create MySQLOptions object. type MySQLOptionsParams interface { GetTlsCa() string @@ -47,7 +52,7 @@ func MySQLOptionsFromRequest(params MySQLOptionsParams) *MySQLOptions { TLSKey: params.GetTlsKey(), } } - return nil + return &MySQLOptions{} } // PostgreSQLOptionsParams contains methods to create PostgreSQLOptions object. @@ -74,7 +79,7 @@ func PostgreSQLOptionsFromRequest(params PostgreSQLOptionsParams) *PostgreSQLOpt // PostgreSQL exporter has these parameters but they are not needed for QAN agent. if extendedOptions, ok := params.(PostgreSQLExtendedOptionsParams); ok && extendedOptions != nil { - res.AutoDiscoveryLimit = extendedOptions.GetAutoDiscoveryLimit() + res.AutoDiscoveryLimit = pointer.ToInt32(extendedOptions.GetAutoDiscoveryLimit()) res.MaxExporterConnections = extendedOptions.GetMaxExporterConnections() } @@ -99,7 +104,7 @@ type MongoDBExtendedOptionsParams interface { // MongoDBOptionsFromRequest creates MongoDBOptionsParams object from request. func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions { - var mdbOptions *MongoDBOptions + mdbOptions := &MongoDBOptions{} if params.GetTlsCertificateKey() != "" || params.GetTlsCertificateKeyFilePassword() != "" || params.GetTlsCa() != "" { mdbOptions = &MongoDBOptions{} @@ -109,9 +114,6 @@ func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions { } if params.GetAuthenticationMechanism() != "" || params.GetAuthenticationDatabase() != "" { - if mdbOptions == nil { - mdbOptions = &MongoDBOptions{} - } mdbOptions.AuthenticationMechanism = params.GetAuthenticationMechanism() mdbOptions.AuthenticationDatabase = params.GetAuthenticationDatabase() } @@ -119,9 +121,6 @@ func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions { // MongoDB exporter has these parameters but they are not needed for QAN agent. if extendedOptions, ok := params.(MongoDBExtendedOptionsParams); ok { if extendedOptions != nil { - if mdbOptions == nil { - mdbOptions = &MongoDBOptions{} - } mdbOptions.StatsCollections = extendedOptions.GetStatsCollections() mdbOptions.CollectionsLimit = extendedOptions.GetCollectionsLimit() mdbOptions.EnableAllCollectors = extendedOptions.GetEnableAllCollectors() @@ -221,7 +220,7 @@ func FindAgents(q *reform.Querier, filters AgentFilters) ([]*Agent, error) { idx++ } if filters.AWSAccessKey != "" { - conditions = append(conditions, fmt.Sprintf("aws_access_key = %s", q.Placeholder(idx))) + conditions = append(conditions, fmt.Sprintf("(aws_options ? 'aws_access_key' AND aws_options->>'aws_access_key' = %s)", q.Placeholder(idx))) args = append(args, filters.AWSAccessKey) } @@ -474,9 +473,9 @@ func FindAgentsForScrapeConfig(q *reform.Querier, pmmAgentID *string, pushMetric } if pushMetrics { - conditions = append(conditions, "push_metrics") + conditions = append(conditions, pushMetricsTrue) } else { - conditions = append(conditions, "NOT push_metrics") + conditions = append(conditions, pushMetricsFalse) } conditions = append(conditions, "NOT disabled", "listen_port IS NOT NULL") @@ -496,7 +495,7 @@ func FindAgentsForScrapeConfig(q *reform.Querier, pmmAgentID *string, pushMetric // FindPMMAgentsIDsWithPushMetrics returns pmm-agents-ids with agent, that use push_metrics mode. func FindPMMAgentsIDsWithPushMetrics(q *reform.Querier) ([]string, error) { - structs, err := q.SelectAllFrom(AgentTable, "WHERE NOT disabled AND pmm_agent_id IS NOT NULL AND push_metrics ORDER BY agent_id") + structs, err := q.SelectAllFrom(AgentTable, fmt.Sprintf("WHERE NOT disabled AND pmm_agent_id IS NOT NULL AND %s ORDER BY agent_id", pushMetricsTrue)) if err != nil { return nil, status.Error(codes.FailedPrecondition, "Couldn't get agents") } @@ -640,15 +639,17 @@ func CreateNodeExporter(q *reform.Querier, " it doesn't support it, minimum supported version=%q", pointer.GetString(pmmAgent.Version), PMMAgentWithPushMetricsSupport.String()) } row := &Agent{ - AgentID: id, - AgentType: NodeExporterType, - PMMAgentID: &pmmAgentID, - NodeID: pmmAgent.RunsOnNodeID, - PushMetrics: pushMetrics, - DisabledCollectors: disableCollectors, - AgentPassword: agentPassword, - LogLevel: pointer.ToStringOrNil(logLevel), - ExposeExporter: exposeExporter, + AgentID: id, + AgentType: NodeExporterType, + PMMAgentID: &pmmAgentID, + NodeID: pmmAgent.RunsOnNodeID, + AgentPassword: agentPassword, + ExporterOptions: &ExporterOptions{ + ExposeExporter: exposeExporter, + PushMetrics: pushMetrics, + DisabledCollectors: disableCollectors, + }, + LogLevel: pointer.ToStringOrNil(logLevel), } if err := row.SetCustomLabels(customLabels); err != nil { return nil, err @@ -725,17 +726,19 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar metricsPath = "/metrics" } row := &Agent{ - PMMAgentID: pmmAgentID, - AgentID: id, - AgentType: ExternalExporterType, - RunsOnNodeID: runsOnNodeID, - ServiceID: pointer.ToStringOrNil(params.ServiceID), - Username: pointer.ToStringOrNil(params.Username), - Password: pointer.ToStringOrNil(params.Password), - MetricsScheme: &scheme, - MetricsPath: &metricsPath, - ListenPort: pointer.ToUint16(uint16(params.ListenPort)), - PushMetrics: params.PushMetrics, + PMMAgentID: pmmAgentID, + AgentID: id, + AgentType: ExternalExporterType, + RunsOnNodeID: runsOnNodeID, + ServiceID: pointer.ToStringOrNil(params.ServiceID), + Username: pointer.ToStringOrNil(params.Username), + Password: pointer.ToStringOrNil(params.Password), + ListenPort: pointer.ToUint16(uint16(params.ListenPort)), + ExporterOptions: &ExporterOptions{ + PushMetrics: params.PushMetrics, + MetricsPath: metricsPath, + MetricsScheme: scheme, + }, } if err := row.SetCustomLabels(params.CustomLabels); err != nil { return nil, err @@ -752,33 +755,23 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar // CreateAgentParams params for add common exporter. type CreateAgentParams struct { - PMMAgentID string - NodeID string - ServiceID string - Username string - Password string - AgentPassword string - CustomLabels map[string]string - TLS bool - TLSSkipVerify bool - MySQLOptions *MySQLOptions - MongoDBOptions *MongoDBOptions - PostgreSQLOptions *PostgreSQLOptions - TableCountTablestatsGroupLimit int32 - MaxQueryLength int32 - QueryExamplesDisabled bool - CommentsParsingDisabled bool - MaxQueryLogSize int64 - AWSAccessKey string - AWSSecretKey string - RDSBasicMetricsDisabled bool - RDSEnhancedMetricsDisabled bool - AzureOptions *AzureOptions - PushMetrics bool - ExposeExporter bool - DisableCollectors []string - LogLevel string - MetricsResolutions *MetricsResolutions + PMMAgentID string + NodeID string + ServiceID string + Username string + Password string + AgentPassword string + CustomLabels map[string]string + TLS bool + TLSSkipVerify bool + LogLevel string + ExporterOptions *ExporterOptions + QANOptions *QANOptions + AWSOptions *AWSOptions + AzureOptions *AzureOptions + MongoDBOptions *MongoDBOptions + MySQLOptions *MySQLOptions + PostgreSQLOptions *PostgreSQLOptions } func compatibleNodeAndAgent(nodeType NodeType, agentType AgentType) bool { @@ -859,6 +852,32 @@ func compatibleServiceAndAgent(serviceType ServiceType, agentType AgentType) boo return false } +func prepareOptionsParams(params *CreateAgentParams) *CreateAgentParams { + if params.ExporterOptions == nil { + params.ExporterOptions = &ExporterOptions{} + } + if params.QANOptions == nil { + params.QANOptions = &QANOptions{} + } + if params.AWSOptions == nil { + params.AWSOptions = &AWSOptions{} + } + if params.AzureOptions == nil { + params.AzureOptions = &AzureOptions{} + } + if params.MongoDBOptions == nil { + params.MongoDBOptions = &MongoDBOptions{} + } + if params.MySQLOptions == nil { + params.MySQLOptions = &MySQLOptions{} + } + if params.PostgreSQLOptions == nil { + params.PostgreSQLOptions = &PostgreSQLOptions{} + } + + return params +} + // CreateAgent creates Agent with given type. func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentParams) (*Agent, error) { //nolint:unparam id := uuid.New().String() @@ -866,12 +885,14 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara return nil, err } + params = prepareOptionsParams(params) + pmmAgent, err := FindAgentByID(q, params.PMMAgentID) if err != nil { return nil, err } // check version for agent, if it exists. - if params.PushMetrics { + if params.ExporterOptions.PushMetrics { // special case for vmAgent, it always supports push metrics. if agentType != VMAgentType && !IsPushMetricsSupported(pmmAgent.Version) { return nil, status.Errorf(codes.FailedPrecondition, "cannot use push_metrics_enabled with pmm_agent version=%q,"+ @@ -902,33 +923,24 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara } row := &Agent{ - AgentID: id, - AgentType: agentType, - PMMAgentID: ¶ms.PMMAgentID, - ServiceID: pointer.ToStringOrNil(params.ServiceID), - NodeID: pointer.ToStringOrNil(params.NodeID), - Username: pointer.ToStringOrNil(params.Username), - Password: pointer.ToStringOrNil(params.Password), - AgentPassword: pointer.ToStringOrNil(params.AgentPassword), - TLS: params.TLS, - TLSSkipVerify: params.TLSSkipVerify, - MySQLOptions: params.MySQLOptions, - MongoDBOptions: params.MongoDBOptions, - PostgreSQLOptions: params.PostgreSQLOptions, - TableCountTablestatsGroupLimit: params.TableCountTablestatsGroupLimit, - MaxQueryLength: params.MaxQueryLength, - QueryExamplesDisabled: params.QueryExamplesDisabled, - CommentsParsingDisabled: params.CommentsParsingDisabled, - MaxQueryLogSize: params.MaxQueryLogSize, - AWSAccessKey: pointer.ToStringOrNil(params.AWSAccessKey), - AWSSecretKey: pointer.ToStringOrNil(params.AWSSecretKey), - RDSBasicMetricsDisabled: params.RDSBasicMetricsDisabled, - RDSEnhancedMetricsDisabled: params.RDSEnhancedMetricsDisabled, - AzureOptions: params.AzureOptions, - PushMetrics: params.PushMetrics, - ExposeExporter: params.ExposeExporter, - DisabledCollectors: params.DisableCollectors, - LogLevel: pointer.ToStringOrNil(params.LogLevel), + AgentID: id, + AgentType: agentType, + PMMAgentID: ¶ms.PMMAgentID, + ServiceID: pointer.ToStringOrNil(params.ServiceID), + NodeID: pointer.ToStringOrNil(params.NodeID), + Username: pointer.ToStringOrNil(params.Username), + Password: pointer.ToStringOrNil(params.Password), + AgentPassword: pointer.ToStringOrNil(params.AgentPassword), + TLS: params.TLS, + TLSSkipVerify: params.TLSSkipVerify, + ExporterOptions: params.ExporterOptions, + QANOptions: params.QANOptions, + AWSOptions: params.AWSOptions, + AzureOptions: params.AzureOptions, + MongoDBOptions: params.MongoDBOptions, + MySQLOptions: params.MySQLOptions, + PostgreSQLOptions: params.PostgreSQLOptions, + LogLevel: pointer.ToStringOrNil(params.LogLevel), } if err := row.SetCustomLabels(params.CustomLabels); err != nil { return nil, err @@ -970,7 +982,7 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } if params.EnablePushMetrics != nil { - row.PushMetrics = *params.EnablePushMetrics + row.ExporterOptions.PushMetrics = *params.EnablePushMetrics if row.AgentType == ExternalExporterType { if err := updateExternalExporterParams(q, row); err != nil { return nil, errors.Wrap(err, "failed to update External exporterParams for PushMetrics") @@ -990,22 +1002,22 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar } } - if row.MetricsResolutions == nil { - row.MetricsResolutions = &MetricsResolutions{} + if row.ExporterOptions.MetricsResolutions == nil { + row.ExporterOptions.MetricsResolutions = &MetricsResolutions{} } if params.MetricsResolutions.LR != nil { - row.MetricsResolutions.LR = *params.MetricsResolutions.LR + row.ExporterOptions.MetricsResolutions.LR = *params.MetricsResolutions.LR } if params.MetricsResolutions.MR != nil { - row.MetricsResolutions.MR = *params.MetricsResolutions.MR + row.ExporterOptions.MetricsResolutions.MR = *params.MetricsResolutions.MR } if params.MetricsResolutions.HR != nil { - row.MetricsResolutions.HR = *params.MetricsResolutions.HR + row.ExporterOptions.MetricsResolutions.HR = *params.MetricsResolutions.HR } // If all resolutions are empty, then drop whole MetricsResolution field. - if row.MetricsResolutions.HR == 0 && row.MetricsResolutions.MR == 0 && row.MetricsResolutions.LR == 0 { - row.MetricsResolutions = nil + if row.ExporterOptions.MetricsResolutions.HR == 0 && row.ExporterOptions.MetricsResolutions.MR == 0 && row.ExporterOptions.MetricsResolutions.LR == 0 { + row.ExporterOptions.MetricsResolutions = nil } if err = q.Update(row); err != nil { @@ -1057,7 +1069,7 @@ func RemoveAgent(q *reform.Querier, id string, mode RemoveMode) (*Agent, error) // for external exporter, is needed for push_metrics mode. func updateExternalExporterParams(q *reform.Querier, row *Agent) error { // with push metrics, external exporter must have PMMAgent id without RunsOnNodeID - if row.PushMetrics && row.PMMAgentID == nil { + if row.ExporterOptions.PushMetrics && row.PMMAgentID == nil { pmmAgent, err := FindPMMAgentsRunningOnNode(q, pointer.GetString(row.RunsOnNodeID)) if err != nil { return err @@ -1074,7 +1086,7 @@ func updateExternalExporterParams(q *reform.Querier, row *Agent) error { row.PMMAgentID = pointer.ToString(pmmAgent[0].AgentID) } // without push metrics, external exporter must have RunsOnNodeID without PMMAgentID - if !row.PushMetrics && row.RunsOnNodeID == nil { + if !row.ExporterOptions.PushMetrics && row.RunsOnNodeID == nil { pmmAgent, err := FindAgentByID(q, pointer.GetString(row.PMMAgentID)) if err != nil { return err diff --git a/managed/models/agent_helpers_test.go b/managed/models/agent_helpers_test.go index fb9f0e6645..b4706fee0e 100644 --- a/managed/models/agent_helpers_test.go +++ b/managed/models/agent_helpers_test.go @@ -103,8 +103,10 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N2"), - PushMetrics: true, ListenPort: pointer.ToUint16(8200), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, }, &models.Agent{ AgentID: "A6", @@ -112,7 +114,6 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N2"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), }, &models.Agent{ @@ -121,20 +122,21 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A4"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + MetricsResolutions: &models.MetricsResolutions{ + HR: 1 * time.Minute, + MR: 5 * time.Minute, + LR: 15 * time.Minute, + }, + }, PostgreSQLOptions: &models.PostgreSQLOptions{ SSLCa: "ssl_ca", SSLCert: "ssl_cert", SSLKey: "ssl_key", }, - MetricsResolutions: &models.MetricsResolutions{ - HR: 1 * time.Minute, - MR: 5 * time.Minute, - LR: 15 * time.Minute, - }, }, &models.Agent{ AgentID: "A8", @@ -142,7 +144,6 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A8"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, @@ -162,7 +163,6 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A9"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, @@ -183,11 +183,10 @@ func TestAgentHelpers(t *testing.T) { PMMAgentID: pointer.ToString("A10"), RunsOnNodeID: nil, NodeID: pointer.ToString("N1"), - PushMetrics: false, ListenPort: pointer.ToUint16(8200), TLS: true, TLSSkipVerify: true, - MongoDBOptions: nil, // this test is specific for nil MongoDBOptions + MongoDBOptions: nil, }, } { require.NoError(t, q.Insert(str)) @@ -208,29 +207,41 @@ func TestAgentHelpers(t *testing.T) { require.NoError(t, err) expected := []*models.Agent{ { - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - AgentID: "A10", - AgentType: models.MongoDBExporterType, - PMMAgentID: pointer.ToString("A10"), - RunsOnNodeID: nil, - NodeID: pointer.ToString("N1"), - PushMetrics: false, - ListenPort: pointer.ToUint16(8200), - TLS: true, - TLSSkipVerify: true, - MongoDBOptions: nil, // this test is specific for nil MongoDBOptions + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + AgentID: "A10", + AgentType: models.MongoDBExporterType, + PMMAgentID: pointer.ToString("A10"), + RunsOnNodeID: nil, + NodeID: pointer.ToString("N1"), + ListenPort: pointer.ToUint16(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { - AgentID: "A3", - AgentType: models.NodeExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - NodeID: pointer.ToString("N1"), - Status: models.AgentStatusUnknown, + AgentID: "A3", + AgentType: models.NodeExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + NodeID: pointer.ToString("N1"), + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { AgentID: "A7", @@ -243,28 +254,39 @@ func TestAgentHelpers(t *testing.T) { ListenPort: pointer.ToUint16OrNil(8200), TLS: true, TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{ + MetricsResolutions: &models.MetricsResolutions{ + HR: 1 * time.Minute, + MR: 5 * time.Minute, + LR: 15 * time.Minute, + }, + }, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, PostgreSQLOptions: &models.PostgreSQLOptions{ SSLCa: "ssl_ca", SSLCert: "ssl_cert", SSLKey: "ssl_key", }, - MetricsResolutions: &models.MetricsResolutions{ - HR: 1 * time.Minute, - MR: 5 * time.Minute, - LR: 15 * time.Minute, - }, }, { - AgentID: "A8", - AgentType: "mongodb_exporter", - NodeID: pointer.ToStringOrNil("N1"), - PMMAgentID: pointer.ToStringOrNil("A8"), - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - ListenPort: pointer.ToUint16OrNil(8200), - TLS: true, - TLSSkipVerify: true, + AgentID: "A8", + AgentType: "mongodb_exporter", + NodeID: pointer.ToStringOrNil("N1"), + PMMAgentID: pointer.ToStringOrNil("A8"), + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ListenPort: pointer.ToUint16OrNil(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -274,18 +296,24 @@ func TestAgentHelpers(t *testing.T) { StatsCollections: nil, CollectionsLimit: 0, // no limit }, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { - AgentID: "A9", - AgentType: "mongodb_exporter", - NodeID: pointer.ToStringOrNil("N1"), - PMMAgentID: pointer.ToStringOrNil("A9"), - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, - ListenPort: pointer.ToUint16OrNil(8200), - TLS: true, - TLSSkipVerify: true, + AgentID: "A9", + AgentType: "mongodb_exporter", + NodeID: pointer.ToStringOrNil("N1"), + PMMAgentID: pointer.ToStringOrNil("A9"), + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ListenPort: pointer.ToUint16OrNil(8200), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, MongoDBOptions: &models.MongoDBOptions{ TLSCertificateKey: "tls_certificate_key", TLSCertificateKeyFilePassword: "tls_certificate_key_file_password", @@ -296,6 +324,8 @@ func TestAgentHelpers(t *testing.T) { CollectionsLimit: 79014, EnableAllCollectors: true, }, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, } assert.Equal(t, expected, agents) @@ -308,23 +338,37 @@ func TestAgentHelpers(t *testing.T) { agents, err := models.FindAgents(q, models.AgentFilters{PMMAgentID: "A1"}) require.NoError(t, err) expected := []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }, { - AgentID: "A3", - AgentType: models.NodeExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - NodeID: pointer.ToString("N1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A3", + AgentType: models.NodeExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + NodeID: pointer.ToString("N1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) }) @@ -336,14 +380,21 @@ func TestAgentHelpers(t *testing.T) { agents, err := models.FindAgents(q, models.AgentFilters{PMMAgentID: "A1", AgentType: pointerToAgentType(models.MySQLdExporterType)}) require.NoError(t, err) expected := []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) }) @@ -355,28 +406,42 @@ func TestAgentHelpers(t *testing.T) { agents, err := models.FindAgents(q, models.AgentFilters{ServiceID: "S1"}) require.NoError(t, err) expected := []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) agents, err = models.FindAgents(q, models.AgentFilters{ServiceID: "S1", AgentType: pointerToAgentType(models.MySQLdExporterType)}) require.NoError(t, err) expected = []*models.Agent{{ - AgentID: "A2", - AgentType: models.MySQLdExporterType, - PMMAgentID: pointer.ToStringOrNil("A1"), - ServiceID: pointer.ToString("S1"), - RunsOnNodeID: nil, - CreatedAt: now, - UpdatedAt: now, - Status: models.AgentStatusUnknown, + AgentID: "A2", + AgentType: models.MySQLdExporterType, + PMMAgentID: pointer.ToStringOrNil("A1"), + ServiceID: pointer.ToString("S1"), + RunsOnNodeID: nil, + CreatedAt: now, + UpdatedAt: now, + Status: models.AgentStatusUnknown, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, }} assert.Equal(t, expected, agents) @@ -406,11 +471,18 @@ func TestAgentHelpers(t *testing.T) { tests.AssertGRPCError(t, status.New(codes.FailedPrecondition, `pmm-agent with ID A1 has agents.`), err) expected := &models.Agent{ - AgentID: "A1", - AgentType: models.PMMAgentType, - RunsOnNodeID: pointer.ToString("N1"), - CreatedAt: now, - UpdatedAt: now, + AgentID: "A1", + AgentType: models.PMMAgentType, + RunsOnNodeID: pointer.ToString("N1"), + CreatedAt: now, + UpdatedAt: now, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + AWSOptions: &models.AWSOptions{}, + AzureOptions: &models.AzureOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } agent, err = models.RemoveAgent(q, "A1", models.RemoveCascade) assert.Equal(t, expected, agent) @@ -468,15 +540,17 @@ func TestAgentHelpers(t *testing.T) { }) require.NoError(t, err) assert.Equal(t, &models.Agent{ - AgentID: agent.AgentID, - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("N1"), - ServiceID: pointer.ToString("S1"), - ListenPort: pointer.ToUint16(9104), - MetricsPath: pointer.ToString("/metrics"), - MetricsScheme: pointer.ToString("http"), - CreatedAt: now, - UpdatedAt: now, + AgentID: agent.AgentID, + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("N1"), + ServiceID: pointer.ToString("S1"), + ListenPort: pointer.ToUint16(9104), + ExporterOptions: &models.ExporterOptions{ + MetricsPath: "/metrics", + MetricsScheme: "http", + }, + CreatedAt: now, + UpdatedAt: now, }, agent) }) t.Run("Invalid listen port", func(t *testing.T) { diff --git a/managed/models/agent_model.go b/managed/models/agent_model.go index a7f1822804..5cc5b70ca8 100644 --- a/managed/models/agent_model.go +++ b/managed/models/agent_model.go @@ -78,18 +78,64 @@ var v2_42 = version.MustParse("2.42.0-0") // PMMServerAgentID is a special Agent ID representing pmm-agent on PMM Server. const PMMServerAgentID = string("pmm-server") // a special ID, reserved for PMM Server -// MySQLOptions represents structure for special MySQL options. -type MySQLOptions struct { - TLSCa string `json:"tls_ca"` - TLSCert string `json:"tls_cert"` - TLSKey string `json:"tls_key"` +// ExporterOptions represents structure for special Exporter options. +type ExporterOptions struct { + ExposeExporter bool `json:"expose_exporter"` + PushMetrics bool `json:"push_metrics"` + DisabledCollectors pq.StringArray `json:"disabled_collectors"` + MetricsResolutions *MetricsResolutions `json:"metrics_resolutions"` + MetricsPath string `json:"metrics_path"` + MetricsScheme string `json:"metrics_scheme"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c MySQLOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *ExporterOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. -func (c *MySQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } +func (c *ExporterOptions) Scan(src interface{}) error { return jsonScan(c, src) } + +// QANOptions represents structure for special QAN options. +type QANOptions struct { + MaxQueryLength int32 `json:"max_query_length"` + MaxQueryLogSize int64 `json:"max_query_log_size"` + QueryExamplesDisabled bool `json:"query_examples_disabled"` + CommentsParsingDisabled bool `json:"comments_parsing_disabled"` +} + +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c *QANOptions) Value() (driver.Value, error) { return jsonValue(c) } + +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *QANOptions) Scan(src interface{}) error { return jsonScan(c, src) } + +// AWSOptions represents structure for special AWS options. +type AWSOptions struct { + AWSAccessKey string `json:"aws_access_key"` + AWSSecretKey string `json:"aws_secret_key"` + RDSBasicMetricsDisabled bool `json:"rds_basic_metrics_disabled"` + RDSEnhancedMetricsDisabled bool `json:"rds_enhanced_metrics_disabled"` +} + +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c *AWSOptions) Value() (driver.Value, error) { return jsonValue(c) } + +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *AWSOptions) Scan(src interface{}) error { return jsonScan(c, src) } + +// AzureOptions represents structure for special Azure options. +type AzureOptions struct { + SubscriptionID string `json:"subscription_id"` + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + TenantID string `json:"tenant_id"` + ResourceGroup string `json:"resource_group"` +} + +// Value implements database/sql/driver.Valuer interface. Should be defined on the value. +func (c *AzureOptions) Value() (driver.Value, error) { return jsonValue(c) } + +// Scan implements database/sql.Scanner interface. Should be defined on the pointer. +func (c *AzureOptions) Scan(src interface{}) error { return jsonScan(c, src) } // MongoDBOptions represents structure for special MongoDB options. type MongoDBOptions struct { @@ -104,39 +150,46 @@ type MongoDBOptions struct { } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c MongoDBOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *MongoDBOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *MongoDBOptions) Scan(src interface{}) error { return jsonScan(c, src) } -// AzureOptions represents structure for special Azure options. -type AzureOptions struct { - SubscriptionID string `json:"subscription_id"` - ClientID string `json:"client_id"` - ClientSecret string `json:"client_secret"` - TenantID string `json:"tenant_id"` - ResourceGroup string `json:"resource_group"` +// MySQLOptions represents structure for special MySQL options. +type MySQLOptions struct { + TLSCa string `json:"tls_ca"` + TLSCert string `json:"tls_cert"` + TLSKey string `json:"tls_key"` + + // TableCount stores last known table count. NULL if unknown. + TableCount *int32 `json:"table_count"` + + // Tablestats group collectors are disabled if there are more than that number of tables. + // 0 means tablestats group collectors are always enabled (no limit). + // Negative value means tablestats group collectors are always disabled. + // See IsMySQLTablestatsGroupEnabled method. + TableCountTablestatsGroupLimit int32 `json:"table_count_tablestats_group_limit"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c AzureOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *MySQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. -func (c *AzureOptions) Scan(src interface{}) error { return jsonScan(c, src) } +func (c *MySQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } // PostgreSQLOptions represents structure for special PostgreSQL options. type PostgreSQLOptions struct { SSLCa string `json:"ssl_ca"` SSLCert string `json:"ssl_cert"` SSLKey string `json:"ssl_key"` - AutoDiscoveryLimit int32 `json:"auto_discovery_limit"` + AutoDiscoveryLimit *int32 `json:"auto_discovery_limit"` DatabaseCount int32 `json:"database_count"` PGSMVersion *string `json:"pgsm_version"` MaxExporterConnections int32 `json:"max_exporter_connections"` } // Value implements database/sql/driver.Valuer interface. Should be defined on the value. -func (c PostgreSQLOptions) Value() (driver.Value, error) { return jsonValue(c) } +func (c *PostgreSQLOptions) Value() (driver.Value, error) { return jsonValue(c) } // Scan implements database/sql.Scanner interface. Should be defined on the pointer. func (c *PostgreSQLOptions) Scan(src interface{}) error { return jsonScan(c, src) } @@ -172,39 +225,16 @@ type Agent struct { TLS bool `reform:"tls"` TLSSkipVerify bool `reform:"tls_skip_verify"` - AWSAccessKey *string `reform:"aws_access_key"` - AWSSecretKey *string `reform:"aws_secret_key"` - - AzureOptions *AzureOptions `reform:"azure_options"` + LogLevel *string `reform:"log_level"` - // TableCount stores last known table count. NULL if unknown. - TableCount *int32 `reform:"table_count"` - - // Tablestats group collectors are disabled if there are more than that number of tables. - // 0 means tablestats group collectors are always enabled (no limit). - // Negative value means tablestats group collectors are always disabled. - // See IsMySQLTablestatsGroupEnabled method. - TableCountTablestatsGroupLimit int32 `reform:"table_count_tablestats_group_limit"` - - MaxQueryLength int32 `reform:"max_query_length"` - QueryExamplesDisabled bool `reform:"query_examples_disabled"` - CommentsParsingDisabled bool `reform:"comments_parsing_disabled"` - MaxQueryLogSize int64 `reform:"max_query_log_size"` - MetricsPath *string `reform:"metrics_path"` - MetricsScheme *string `reform:"metrics_scheme"` - - RDSBasicMetricsDisabled bool `reform:"rds_basic_metrics_disabled"` - RDSEnhancedMetricsDisabled bool `reform:"rds_enhanced_metrics_disabled"` - PushMetrics bool `reform:"push_metrics"` - DisabledCollectors pq.StringArray `reform:"disabled_collectors"` - MetricsResolutions *MetricsResolutions `reform:"metrics_resolutions"` + ExporterOptions *ExporterOptions `reform:"exporter_options"` + QANOptions *QANOptions `reform:"qan_options"` + AWSOptions *AWSOptions `reform:"aws_options"` + AzureOptions *AzureOptions `reform:"azure_options"` + MongoDBOptions *MongoDBOptions `reform:"mongo_options"` MySQLOptions *MySQLOptions `reform:"mysql_options"` - MongoDBOptions *MongoDBOptions `reform:"mongo_db_tls_options"` PostgreSQLOptions *PostgreSQLOptions `reform:"postgresql_options"` - LogLevel *string `reform:"log_level"` - - ExposeExporter bool `reform:"expose_exporter"` } // BeforeInsert implements reform.BeforeInserter interface. @@ -454,22 +484,20 @@ func (s *Agent) DSN(service *Service, dsnParams DSNParams, tdp *DelimiterPair, p } } - if s.MongoDBOptions != nil { - if s.MongoDBOptions.TLSCertificateKey != "" { - q.Add("tlsCertificateKeyFile", tdp.Left+".TextFiles."+certificateKeyFilePlaceholder+tdp.Right) - } - if s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { - q.Add("tlsCertificateKeyFilePassword", s.MongoDBOptions.TLSCertificateKeyFilePassword) - } - if s.MongoDBOptions.TLSCa != "" { - q.Add("tlsCaFile", tdp.Left+".TextFiles."+caFilePlaceholder+tdp.Right) - } - if s.MongoDBOptions.AuthenticationMechanism != "" { - q.Add("authMechanism", s.MongoDBOptions.AuthenticationMechanism) - } - if s.MongoDBOptions.AuthenticationDatabase != "" { - q.Add("authSource", s.MongoDBOptions.AuthenticationDatabase) - } + if s.MongoDBOptions.TLSCertificateKey != "" { + q.Add("tlsCertificateKeyFile", tdp.Left+".TextFiles."+certificateKeyFilePlaceholder+tdp.Right) + } + if s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { + q.Add("tlsCertificateKeyFilePassword", s.MongoDBOptions.TLSCertificateKeyFilePassword) + } + if s.MongoDBOptions.TLSCa != "" { + q.Add("tlsCaFile", tdp.Left+".TextFiles."+caFilePlaceholder+tdp.Right) + } + if s.MongoDBOptions.AuthenticationMechanism != "" { + q.Add("authMechanism", s.MongoDBOptions.AuthenticationMechanism) + } + if s.MongoDBOptions.AuthenticationDatabase != "" { + q.Add("authSource", s.MongoDBOptions.AuthenticationDatabase) } address := socket @@ -563,14 +591,14 @@ func (s *Agent) DSN(service *Service, dsnParams DSNParams, tdp *DelimiterPair, p // ExporterURL composes URL to an external exporter. func (s *Agent) ExporterURL(q *reform.Querier) (string, error) { - scheme := pointer.GetString(s.MetricsScheme) - path := pointer.GetString(s.MetricsPath) + scheme := s.ExporterOptions.MetricsScheme + path := s.ExporterOptions.MetricsPath listenPort := int(pointer.GetUint16(s.ListenPort)) username := pointer.GetString(s.Username) password := pointer.GetString(s.Password) host := "127.0.0.1" - if !s.PushMetrics { + if !s.ExporterOptions.PushMetrics { node, err := FindNodeByID(q, *s.RunsOnNodeID) if err != nil { return "", err @@ -609,14 +637,14 @@ func (s *Agent) IsMySQLTablestatsGroupEnabled() bool { } switch { - case s.TableCountTablestatsGroupLimit == 0: // server defined + case s.MySQLOptions.TableCountTablestatsGroupLimit == 0: // server defined return true - case s.TableCountTablestatsGroupLimit < 0: // always disabled + case s.MySQLOptions.TableCountTablestatsGroupLimit < 0: // always disabled return false - case s.TableCount == nil: // for compatibility with 2.0 + case s.MySQLOptions.TableCount == nil: // for compatibility with 2.0 return true default: - return *s.TableCount <= s.TableCountTablestatsGroupLimit + return *s.MySQLOptions.TableCount <= s.MySQLOptions.TableCountTablestatsGroupLimit } } @@ -624,49 +652,55 @@ func (s *Agent) IsMySQLTablestatsGroupEnabled() bool { func (s Agent) Files() map[string]string { switch s.AgentType { case MySQLdExporterType, QANMySQLPerfSchemaAgentType, QANMySQLSlowlogAgentType: - if s.MySQLOptions != nil { - files := make(map[string]string) - if s.MySQLOptions.TLSCa != "" { - files["tlsCa"] = s.MySQLOptions.TLSCa - } - if s.MySQLOptions.TLSCert != "" { - files["tlsCert"] = s.MySQLOptions.TLSCert - } - if s.MySQLOptions.TLSKey != "" { - files["tlsKey"] = s.MySQLOptions.TLSKey - } + files := make(map[string]string) + if s.MySQLOptions.TLSCa != "" { + files["tlsCa"] = s.MySQLOptions.TLSCa + } + if s.MySQLOptions.TLSCert != "" { + files["tlsCert"] = s.MySQLOptions.TLSCert + } + if s.MySQLOptions.TLSKey != "" { + files["tlsKey"] = s.MySQLOptions.TLSKey + } + + if len(files) != 0 { return files } + return nil case ProxySQLExporterType: return nil case QANMongoDBProfilerAgentType, MongoDBExporterType: - if s.MongoDBOptions != nil { - files := make(map[string]string) - if s.MongoDBOptions.TLSCa != "" { - files[caFilePlaceholder] = s.MongoDBOptions.TLSCa - } - if s.MongoDBOptions.TLSCertificateKey != "" { - files[certificateKeyFilePlaceholder] = s.MongoDBOptions.TLSCertificateKey - } + files := make(map[string]string) + if s.MongoDBOptions.TLSCa != "" { + files[caFilePlaceholder] = s.MongoDBOptions.TLSCa + } + if s.MongoDBOptions.TLSCertificateKey != "" { + files[certificateKeyFilePlaceholder] = s.MongoDBOptions.TLSCertificateKey + } + + if len(files) != 0 { return files } + return nil case PostgresExporterType, QANPostgreSQLPgStatementsAgentType, QANPostgreSQLPgStatMonitorAgentType: - if s.PostgreSQLOptions != nil { - files := make(map[string]string) + files := make(map[string]string) - if s.PostgreSQLOptions.SSLCa != "" { - files[caFilePlaceholder] = s.PostgreSQLOptions.SSLCa - } - if s.PostgreSQLOptions.SSLCert != "" { - files[certificateFilePlaceholder] = s.PostgreSQLOptions.SSLCert - } - if s.PostgreSQLOptions.SSLKey != "" { - files[certificateKeyFilePlaceholder] = s.PostgreSQLOptions.SSLKey - } + if s.PostgreSQLOptions.SSLCa != "" { + files[caFilePlaceholder] = s.PostgreSQLOptions.SSLCa + } + if s.PostgreSQLOptions.SSLCert != "" { + files[certificateFilePlaceholder] = s.PostgreSQLOptions.SSLCert + } + if s.PostgreSQLOptions.SSLKey != "" { + files[certificateKeyFilePlaceholder] = s.PostgreSQLOptions.SSLKey + } + + if len(files) != 0 { return files } + return nil default: panic(fmt.Errorf("unhandled AgentType %q", s.AgentType)) @@ -679,20 +713,20 @@ func (s Agent) TemplateDelimiters(svc *Service) *DelimiterPair { pointer.GetString(svc.Address), pointer.GetString(s.Username), pointer.GetString(s.Password), - pointer.GetString(s.MetricsPath), + s.ExporterOptions.MetricsPath, } switch svc.ServiceType { case MySQLServiceType: - if s.MySQLOptions != nil { + if s.MySQLOptions.TLSKey != "" { templateParams = append(templateParams, s.MySQLOptions.TLSKey) } case MongoDBServiceType: - if s.MongoDBOptions != nil { + if s.MongoDBOptions.TLSCertificateKeyFilePassword != "" { templateParams = append(templateParams, s.MongoDBOptions.TLSCertificateKeyFilePassword) } case PostgreSQLServiceType: - if s.PostgreSQLOptions != nil { + if s.PostgreSQLOptions.SSLKey != "" { templateParams = append(templateParams, s.PostgreSQLOptions.SSLKey) } case ProxySQLServiceType: diff --git a/managed/models/agent_model_reform.go b/managed/models/agent_model_reform.go index 0f6c88c440..2bce8110e1 100644 --- a/managed/models/agent_model_reform.go +++ b/managed/models/agent_model_reform.go @@ -47,27 +47,14 @@ func (v *agentTableType) Columns() []string { "agent_password", "tls", "tls_skip_verify", - "aws_access_key", - "aws_secret_key", + "log_level", + "exporter_options", + "qan_options", + "aws_options", "azure_options", - "table_count", - "table_count_tablestats_group_limit", - "max_query_length", - "query_examples_disabled", - "comments_parsing_disabled", - "max_query_log_size", - "metrics_path", - "metrics_scheme", - "rds_basic_metrics_disabled", - "rds_enhanced_metrics_disabled", - "push_metrics", - "disabled_collectors", - "metrics_resolutions", + "mongo_options", "mysql_options", - "mongo_db_tls_options", "postgresql_options", - "log_level", - "expose_exporter", } } @@ -111,27 +98,14 @@ var AgentTable = &agentTableType{ {Name: "AgentPassword", Type: "*string", Column: "agent_password"}, {Name: "TLS", Type: "bool", Column: "tls"}, {Name: "TLSSkipVerify", Type: "bool", Column: "tls_skip_verify"}, - {Name: "AWSAccessKey", Type: "*string", Column: "aws_access_key"}, - {Name: "AWSSecretKey", Type: "*string", Column: "aws_secret_key"}, + {Name: "LogLevel", Type: "*string", Column: "log_level"}, + {Name: "ExporterOptions", Type: "*ExporterOptions", Column: "exporter_options"}, + {Name: "QANOptions", Type: "*QANOptions", Column: "qan_options"}, + {Name: "AWSOptions", Type: "*AWSOptions", Column: "aws_options"}, {Name: "AzureOptions", Type: "*AzureOptions", Column: "azure_options"}, - {Name: "TableCount", Type: "*int32", Column: "table_count"}, - {Name: "TableCountTablestatsGroupLimit", Type: "int32", Column: "table_count_tablestats_group_limit"}, - {Name: "MaxQueryLength", Type: "int32", Column: "max_query_length"}, - {Name: "QueryExamplesDisabled", Type: "bool", Column: "query_examples_disabled"}, - {Name: "CommentsParsingDisabled", Type: "bool", Column: "comments_parsing_disabled"}, - {Name: "MaxQueryLogSize", Type: "int64", Column: "max_query_log_size"}, - {Name: "MetricsPath", Type: "*string", Column: "metrics_path"}, - {Name: "MetricsScheme", Type: "*string", Column: "metrics_scheme"}, - {Name: "RDSBasicMetricsDisabled", Type: "bool", Column: "rds_basic_metrics_disabled"}, - {Name: "RDSEnhancedMetricsDisabled", Type: "bool", Column: "rds_enhanced_metrics_disabled"}, - {Name: "PushMetrics", Type: "bool", Column: "push_metrics"}, - {Name: "DisabledCollectors", Type: "pq.StringArray", Column: "disabled_collectors"}, - {Name: "MetricsResolutions", Type: "*MetricsResolutions", Column: "metrics_resolutions"}, + {Name: "MongoDBOptions", Type: "*MongoDBOptions", Column: "mongo_options"}, {Name: "MySQLOptions", Type: "*MySQLOptions", Column: "mysql_options"}, - {Name: "MongoDBOptions", Type: "*MongoDBOptions", Column: "mongo_db_tls_options"}, {Name: "PostgreSQLOptions", Type: "*PostgreSQLOptions", Column: "postgresql_options"}, - {Name: "LogLevel", Type: "*string", Column: "log_level"}, - {Name: "ExposeExporter", Type: "bool", Column: "expose_exporter"}, }, PKFieldIndex: 0, }, @@ -140,7 +114,7 @@ var AgentTable = &agentTableType{ // String returns a string representation of this struct or record. func (s Agent) String() string { - res := make([]string, 40) + res := make([]string, 27) res[0] = "AgentID: " + reform.Inspect(s.AgentID, true) res[1] = "AgentType: " + reform.Inspect(s.AgentType, true) res[2] = "RunsOnNodeID: " + reform.Inspect(s.RunsOnNodeID, true) @@ -160,27 +134,14 @@ func (s Agent) String() string { res[16] = "AgentPassword: " + reform.Inspect(s.AgentPassword, true) res[17] = "TLS: " + reform.Inspect(s.TLS, true) res[18] = "TLSSkipVerify: " + reform.Inspect(s.TLSSkipVerify, true) - res[19] = "AWSAccessKey: " + reform.Inspect(s.AWSAccessKey, true) - res[20] = "AWSSecretKey: " + reform.Inspect(s.AWSSecretKey, true) - res[21] = "AzureOptions: " + reform.Inspect(s.AzureOptions, true) - res[22] = "TableCount: " + reform.Inspect(s.TableCount, true) - res[23] = "TableCountTablestatsGroupLimit: " + reform.Inspect(s.TableCountTablestatsGroupLimit, true) - res[24] = "MaxQueryLength: " + reform.Inspect(s.MaxQueryLength, true) - res[25] = "QueryExamplesDisabled: " + reform.Inspect(s.QueryExamplesDisabled, true) - res[26] = "CommentsParsingDisabled: " + reform.Inspect(s.CommentsParsingDisabled, true) - res[27] = "MaxQueryLogSize: " + reform.Inspect(s.MaxQueryLogSize, true) - res[28] = "MetricsPath: " + reform.Inspect(s.MetricsPath, true) - res[29] = "MetricsScheme: " + reform.Inspect(s.MetricsScheme, true) - res[30] = "RDSBasicMetricsDisabled: " + reform.Inspect(s.RDSBasicMetricsDisabled, true) - res[31] = "RDSEnhancedMetricsDisabled: " + reform.Inspect(s.RDSEnhancedMetricsDisabled, true) - res[32] = "PushMetrics: " + reform.Inspect(s.PushMetrics, true) - res[33] = "DisabledCollectors: " + reform.Inspect(s.DisabledCollectors, true) - res[34] = "MetricsResolutions: " + reform.Inspect(s.MetricsResolutions, true) - res[35] = "MySQLOptions: " + reform.Inspect(s.MySQLOptions, true) - res[36] = "MongoDBOptions: " + reform.Inspect(s.MongoDBOptions, true) - res[37] = "PostgreSQLOptions: " + reform.Inspect(s.PostgreSQLOptions, true) - res[38] = "LogLevel: " + reform.Inspect(s.LogLevel, true) - res[39] = "ExposeExporter: " + reform.Inspect(s.ExposeExporter, true) + res[19] = "LogLevel: " + reform.Inspect(s.LogLevel, true) + res[20] = "ExporterOptions: " + reform.Inspect(s.ExporterOptions, true) + res[21] = "QANOptions: " + reform.Inspect(s.QANOptions, true) + res[22] = "AWSOptions: " + reform.Inspect(s.AWSOptions, true) + res[23] = "AzureOptions: " + reform.Inspect(s.AzureOptions, true) + res[24] = "MongoDBOptions: " + reform.Inspect(s.MongoDBOptions, true) + res[25] = "MySQLOptions: " + reform.Inspect(s.MySQLOptions, true) + res[26] = "PostgreSQLOptions: " + reform.Inspect(s.PostgreSQLOptions, true) return strings.Join(res, ", ") } @@ -207,27 +168,14 @@ func (s *Agent) Values() []interface{} { s.AgentPassword, s.TLS, s.TLSSkipVerify, - s.AWSAccessKey, - s.AWSSecretKey, + s.LogLevel, + s.ExporterOptions, + s.QANOptions, + s.AWSOptions, s.AzureOptions, - s.TableCount, - s.TableCountTablestatsGroupLimit, - s.MaxQueryLength, - s.QueryExamplesDisabled, - s.CommentsParsingDisabled, - s.MaxQueryLogSize, - s.MetricsPath, - s.MetricsScheme, - s.RDSBasicMetricsDisabled, - s.RDSEnhancedMetricsDisabled, - s.PushMetrics, - s.DisabledCollectors, - s.MetricsResolutions, - s.MySQLOptions, s.MongoDBOptions, + s.MySQLOptions, s.PostgreSQLOptions, - s.LogLevel, - s.ExposeExporter, } } @@ -254,27 +202,14 @@ func (s *Agent) Pointers() []interface{} { &s.AgentPassword, &s.TLS, &s.TLSSkipVerify, - &s.AWSAccessKey, - &s.AWSSecretKey, + &s.LogLevel, + &s.ExporterOptions, + &s.QANOptions, + &s.AWSOptions, &s.AzureOptions, - &s.TableCount, - &s.TableCountTablestatsGroupLimit, - &s.MaxQueryLength, - &s.QueryExamplesDisabled, - &s.CommentsParsingDisabled, - &s.MaxQueryLogSize, - &s.MetricsPath, - &s.MetricsScheme, - &s.RDSBasicMetricsDisabled, - &s.RDSEnhancedMetricsDisabled, - &s.PushMetrics, - &s.DisabledCollectors, - &s.MetricsResolutions, - &s.MySQLOptions, &s.MongoDBOptions, + &s.MySQLOptions, &s.PostgreSQLOptions, - &s.LogLevel, - &s.ExposeExporter, } } diff --git a/managed/models/agent_model_test.go b/managed/models/agent_model_test.go index 4fb68a0a9f..1b7ab39dbc 100644 --- a/managed/models/agent_model_test.go +++ b/managed/models/agent_model_test.go @@ -48,8 +48,13 @@ func TestAgent(t *testing.T) { t.Run("DSN", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Address: pointer.ToString("1.2.3.4"), @@ -80,8 +85,11 @@ func TestAgent(t *testing.T) { t.Run("DSN socket", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MySQLOptions: &models.MySQLOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/mysqld/mysqld.sock"), @@ -101,8 +109,11 @@ func TestAgent(t *testing.T) { t.Run("DSN timeout", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/mysqld/mysqld.sock"), @@ -139,6 +150,7 @@ func TestAgent(t *testing.T) { Username: pointer.ToString("username"), Password: pointer.ToString("s3cur3 p@$$w0r4."), TLS: true, + ExporterOptions: &models.ExporterOptions{}, MongoDBOptions: &mongoDBOptions, MySQLOptions: &mysqlOptions, PostgreSQLOptions: &postgresqlOptions, @@ -197,10 +209,15 @@ func TestAgent(t *testing.T) { t.Run("DSN ssl-skip-verify", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, - TLSSkipVerify: true, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + TLSSkipVerify: true, + ExporterOptions: &models.ExporterOptions{}, + QANOptions: &models.QANOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, + MySQLOptions: &models.MySQLOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Address: pointer.ToString("1.2.3.4"), @@ -232,9 +249,11 @@ func TestAgent(t *testing.T) { func TestPostgresAgentTLS(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentType: models.PostgresExporterType, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Address: pointer.ToString("1.2.3.4"), @@ -260,7 +279,7 @@ func TestPostgresAgentTLS(t *testing.T) { t.Run(fmt.Sprintf("AutodiscoveryLimit set TLS:%v/TLSSkipVerify:%v", testCase.tls, testCase.tlsSkipVerify), func(t *testing.T) { agent.TLS = testCase.tls agent.TLSSkipVerify = testCase.tlsSkipVerify - agent.PostgreSQLOptions = &models.PostgreSQLOptions{AutoDiscoveryLimit: 10} + agent.PostgreSQLOptions = &models.PostgreSQLOptions{AutoDiscoveryLimit: pointer.ToInt32(10)} assert.Equal(t, testCase.expected, agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: "database"}, nil, nil)) }) } @@ -269,10 +288,12 @@ func TestPostgresAgentTLS(t *testing.T) { func TestPostgresWithSocket(t *testing.T) { t.Run("empty-password", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - AgentType: models.PostgresExporterType, - TLS: true, - TLSSkipVerify: false, + Username: pointer.ToString("username"), + AgentType: models.PostgresExporterType, + TLS: true, + TLSSkipVerify: false, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/postgres"), @@ -283,7 +304,9 @@ func TestPostgresWithSocket(t *testing.T) { t.Run("empty-user-password", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.PostgresExporterType, + AgentType: models.PostgresExporterType, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Socket: pointer.ToString("/var/run/postgres"), @@ -294,7 +317,9 @@ func TestPostgresWithSocket(t *testing.T) { t.Run("dir-with-symbols", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.PostgresExporterType, + AgentType: models.PostgresExporterType, + ExporterOptions: &models.ExporterOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } service := &models.Service{ Socket: pointer.ToString(`/tmp/123\ A0m\%\$\@\8\,\+\-`), @@ -307,10 +332,12 @@ func TestPostgresWithSocket(t *testing.T) { func TestMongoWithSocket(t *testing.T) { t.Run("empty-password", func(t *testing.T) { agent := &models.Agent{ - Username: pointer.ToString("username"), - AgentType: models.MongoDBExporterType, - TLS: true, - TLSSkipVerify: false, + Username: pointer.ToString("username"), + AgentType: models.MongoDBExporterType, + TLS: true, + TLSSkipVerify: false, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString("/tmp/mongodb-27017.sock"), @@ -321,7 +348,9 @@ func TestMongoWithSocket(t *testing.T) { t.Run("empty-user-password", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.MongoDBExporterType, + AgentType: models.MongoDBExporterType, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString("/tmp/mongodb-27017.sock"), @@ -332,7 +361,9 @@ func TestMongoWithSocket(t *testing.T) { t.Run("dir-with-symbols", func(t *testing.T) { agent := &models.Agent{ - AgentType: models.MongoDBExporterType, + AgentType: models.MongoDBExporterType, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } service := &models.Service{ Socket: pointer.ToString(`/tmp/123\ A0m\%\$\@\8\,\+\-/mongodb-27017.sock`), @@ -364,9 +395,11 @@ func TestIsMySQLTablestatsGroupEnabled(t *testing.T) { } t.Run(fmt.Sprintf("Count:%s/Limit:%d", c, testCase.limit), func(t *testing.T) { agent := &models.Agent{ - AgentType: models.MySQLdExporterType, - TableCount: testCase.count, - TableCountTablestatsGroupLimit: testCase.limit, + AgentType: models.MySQLdExporterType, + MySQLOptions: &models.MySQLOptions{ + TableCount: testCase.count, + TableCountTablestatsGroupLimit: testCase.limit, + }, } assert.Equal(t, testCase.expected, agent.IsMySQLTablestatsGroupEnabled()) }) @@ -439,66 +472,76 @@ func TestExporterURL(t *testing.T) { }, &models.Agent{ - AgentID: "ExporterAgentPush", - AgentType: models.ExternalExporterType, - ServiceID: pointer.ToString("external"), - RunsOnNodeID: pointer.ToString("ExporterNodeID"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: true, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics"), + AgentID: "ExporterAgentPush", + AgentType: models.ExternalExporterType, + ServiceID: pointer.ToString("external"), + RunsOnNodeID: pointer.ToString("ExporterNodeID"), + ListenPort: pointer.ToUint16(9121), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + MetricsPath: "/metrics", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterAgentPull", - AgentType: models.ExternalExporterType, - ServiceID: pointer.ToString("external"), - RunsOnNodeID: pointer.ToString("ExporterNodeID"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterAgentPull", + AgentType: models.ExternalExporterType, + ServiceID: pointer.ToString("external"), + RunsOnNodeID: pointer.ToString("ExporterNodeID"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/metrics", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterServerless", - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID"), - ServiceID: pointer.ToString("redis_exporter-external"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterServerless", + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID"), + ServiceID: pointer.ToString("redis_exporter-external"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/metrics", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterServerlessWithQueryParams", - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), - ServiceID: pointer.ToString("nomad_exporter-external"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/metrics?format=prometheus&output=json"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterServerlessWithQueryParams", + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), + ServiceID: pointer.ToString("nomad_exporter-external"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/metrics?format=prometheus&output=json", + MetricsScheme: "http", + }, }, &models.Agent{ - AgentID: "ExporterServerlessWithEmptyMetricsPath", - AgentType: models.ExternalExporterType, - RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), - ServiceID: pointer.ToString("nomad_exporter-external"), - MetricsScheme: pointer.ToString("http"), - PushMetrics: false, - ListenPort: pointer.ToUint16(9121), - MetricsPath: pointer.ToString("/"), - Username: pointer.ToString("user"), - Password: pointer.ToString("secret"), + AgentID: "ExporterServerlessWithEmptyMetricsPath", + AgentType: models.ExternalExporterType, + RunsOnNodeID: pointer.ToString("ExporterServerlessNodeID2"), + ServiceID: pointer.ToString("nomad_exporter-external"), + ListenPort: pointer.ToUint16(9121), + Username: pointer.ToString("user"), + Password: pointer.ToString("secret"), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + MetricsPath: "/", + MetricsScheme: "http", + }, }, } { require.NoError(t, q.Insert(str), "failed to INSERT %+v", str) diff --git a/managed/models/database.go b/managed/models/database.go index d7a9778721..7126979431 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -60,21 +60,20 @@ const ( VerifyFullSSLMode string = "verify-full" ) -// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. -var DefaultAgentEncryptionColumns = []encryption.Table{ +// DefaultAgentEncryptionColumnsV3 since 3.0.0 contains all tables and it's columns to be encrypted in PMM Server DB. +var DefaultAgentEncryptionColumnsV3 = []encryption.Table{ { Name: "agents", Identifiers: []string{"agent_id"}, Columns: []encryption.Column{ {Name: "username"}, {Name: "password"}, - {Name: "aws_access_key"}, - {Name: "aws_secret_key"}, - {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, + {Name: "agent_password"}, + {Name: "aws_options", CustomHandler: EncryptAWSOptionsHandler}, {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, + {Name: "mongo_options", CustomHandler: EncryptMongoDBOptionsHandler}, {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, - {Name: "agent_password"}, }, }, } @@ -1077,6 +1076,74 @@ var databaseSchema = [][]string{ `ALTER TABLE user_flags ADD COLUMN snoozed_pmm_version VARCHAR NOT NULL DEFAULT ''`, }, + 107: { + `ALTER TABLE agents ADD COLUMN exporter_options JSONB`, + `UPDATE agents SET exporter_options = '{}'::jsonb`, + `ALTER TABLE agents ADD COLUMN qan_options JSONB`, + `UPDATE agents SET qan_options = '{}'::jsonb`, + `ALTER TABLE agents ADD COLUMN aws_options JSONB`, + `UPDATE agents SET aws_options = '{}'::jsonb`, + + `ALTER TABLE agents ALTER COLUMN azure_options TYPE JSONB USING to_jsonb(azure_options)`, + `UPDATE agents SET azure_options = '{}'::jsonb WHERE azure_options IS NULL`, + `ALTER TABLE agents ALTER COLUMN mysql_options TYPE JSONB USING to_jsonb(mysql_options)`, + `UPDATE agents SET mysql_options = '{}'::jsonb WHERE mysql_options IS NULL`, + + `ALTER TABLE agents RENAME COLUMN mongo_db_tls_options TO mongo_options`, + `UPDATE agents SET mongo_options = '{}'::jsonb WHERE mongo_options IS NULL`, + + `UPDATE agents SET postgresql_options = '{}'::jsonb WHERE postgresql_options IS NULL`, + + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{expose_exporter}', to_jsonb(expose_exporter));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{push_metrics}', to_jsonb(push_metrics));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{disabled_collectors}', to_jsonb(disabled_collectors));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{metrics_resolutions}', to_jsonb(metrics_resolutions));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{metrics_path}', to_jsonb(metrics_path));`, + `UPDATE agents SET exporter_options = jsonb_set(exporter_options, '{metrics_scheme}', to_jsonb(metrics_scheme));`, + `ALTER TABLE agents DROP COLUMN expose_exporter`, + `ALTER TABLE agents DROP COLUMN push_metrics`, + `ALTER TABLE agents DROP COLUMN disabled_collectors`, + `ALTER TABLE agents DROP COLUMN metrics_resolutions`, + `ALTER TABLE agents DROP COLUMN metrics_path`, + `ALTER TABLE agents DROP COLUMN metrics_scheme`, + + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{max_query_length}', to_jsonb(max_query_length));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{max_query_log_size}', to_jsonb(max_query_log_size));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{query_examples_disabled}', to_jsonb(query_examples_disabled));`, + `UPDATE agents SET qan_options = jsonb_set(qan_options, '{comments_parsing_disabled}', to_jsonb(comments_parsing_disabled));`, + `ALTER TABLE agents DROP COLUMN max_query_length`, + `ALTER TABLE agents DROP COLUMN max_query_log_size`, + `ALTER TABLE agents DROP COLUMN query_examples_disabled`, + `ALTER TABLE agents DROP COLUMN comments_parsing_disabled`, + + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{aws_access_key}', to_jsonb(aws_access_key));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{aws_secret_key}', to_jsonb(aws_secret_key));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{rds_basic_metrics_disabled}', to_jsonb(rds_basic_metrics_disabled));`, + `UPDATE agents SET aws_options = jsonb_set(aws_options, '{rds_enhanced_metrics_disabled}', to_jsonb(rds_enhanced_metrics_disabled));`, + `ALTER TABLE agents DROP COLUMN aws_access_key`, + `ALTER TABLE agents DROP COLUMN aws_secret_key`, + `ALTER TABLE agents DROP COLUMN rds_basic_metrics_disabled`, + `ALTER TABLE agents DROP COLUMN rds_enhanced_metrics_disabled`, + + `UPDATE agents SET mysql_options = jsonb_set(mysql_options, '{table_count}', to_jsonb(table_count));`, + `UPDATE agents SET mysql_options = jsonb_set(mysql_options, '{table_count_tablestats_group_limit}', to_jsonb(table_count_tablestats_group_limit));`, + `ALTER TABLE agents DROP COLUMN table_count`, + `ALTER TABLE agents DROP COLUMN table_count_tablestats_group_limit`, + + `UPDATE settings SET settings = jsonb_set(settings, '{encrypted_items}', + '[ + "pmm-managed.agents.username", + "pmm-managed.agents.password", + "pmm-managed.agents.agent_password", + "pmm-managed.agents.aws_options", + "pmm-managed.agents.azure_options", + "pmm-managed.agents.mongo_options", + "pmm-managed.agents.mysql_options", + "pmm-managed.agents.postgresql_options" + ]'::jsonb + ) + WHERE settings ? 'encrypted_items';`, + }, } // ^^^ Avoid default values in schema definition. ^^^ @@ -1167,7 +1234,7 @@ func SetupDB(ctx context.Context, sqlDB *sql.DB, params SetupDBParams) (*reform. return nil, errCV } - if err := migrateDB(db, params, DefaultAgentEncryptionColumns); err != nil { + if err := migrateDB(db, params); err != nil { return nil, err } @@ -1304,7 +1371,7 @@ func initWithRoot(params SetupDBParams) error { } // migrateDB runs PostgreSQL database migrations. -func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption.Table) error { +func migrateDB(db *reform.DB, params SetupDBParams) error { var currentVersion int errDB := db.QueryRow("SELECT id FROM schema_migrations ORDER BY id DESC LIMIT 1").Scan(¤tVersion) // undefined_table (see https://www.postgresql.org/docs/current/errcodes-appendix.html) @@ -1340,7 +1407,7 @@ func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption. } } - err := EncryptDB(tx, params.Name, itemsToEncrypt) + err := EncryptDB(tx, params.Name, DefaultAgentEncryptionColumnsV3) if err != nil { return err } @@ -1414,13 +1481,15 @@ func setupPMMServerAgents(q *reform.Querier, params SetupDBParams) error { } ap := &CreateAgentParams{ - PMMAgentID: PMMServerAgentID, - ServiceID: service.ServiceID, - TLS: params.SSLMode != DisableSSLMode, - TLSSkipVerify: params.SSLMode == DisableSSLMode || params.SSLMode == VerifyCaSSLMode, - CommentsParsingDisabled: true, - Username: params.Username, - Password: params.Password, + PMMAgentID: PMMServerAgentID, + ServiceID: service.ServiceID, + TLS: params.SSLMode != DisableSSLMode, + TLSSkipVerify: params.SSLMode == DisableSSLMode || params.SSLMode == VerifyCaSSLMode, + Username: params.Username, + Password: params.Password, + QANOptions: &QANOptions{ + CommentsParsingDisabled: true, + }, } if ap.TLS { ap.PostgreSQLOptions = &PostgreSQLOptions{} diff --git a/managed/models/database_test.go b/managed/models/database_test.go index efafd5e2f7..42175ac305 100644 --- a/managed/models/database_test.go +++ b/managed/models/database_test.go @@ -17,12 +17,10 @@ package models_test import ( - "context" "database/sql" "fmt" "testing" - "github.com/AlekSi/pointer" "github.com/lib/pq" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -205,8 +203,8 @@ func TestDatabaseChecks(t *testing.T) { now, now) require.NoError(t, err) _, err = db.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('1', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, true, true, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('1', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": true, "rds_enhanced_metrics_disabled": true}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) @@ -216,13 +214,13 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('2', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('2', 'pmm-agent', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('3', 'mysqld_exporter', NULL, '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('3', 'mysqld_exporter', NULL, '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) require.NoError(t, err) }) @@ -232,8 +230,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('4', 'mysqld_exporter', NULL, NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('4', 'mysqld_exporter', NULL, NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_xor_pmm_agent_id") }) @@ -243,8 +241,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('5', 'pmm-agent', '1', '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('5', 'pmm-agent', '1', '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_xor_pmm_agent_id") }) @@ -255,8 +253,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('6', 'mysqld_exporter', '1', NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('6', 'mysqld_exporter', '1', NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_only_for_pmm_agent") }) @@ -266,8 +264,8 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('7', 'pmm-agent', NULL, '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('7', 'pmm-agent', NULL, '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "runs_on_node_id_only_for_pmm_agent") }) @@ -280,10 +278,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'node_exporter', NULL, '1', '1', NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'node_exporter', NULL, '1', '1', NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assert.NoError(t, err) }) @@ -292,10 +289,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assert.NoError(t, err) }) @@ -304,10 +300,9 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, NULL, false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', NULL, NULL, false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) - assertCheckViolation(t, err, "agents", "node_id_or_service_id_for_non_pmm_agent") }) @@ -316,81 +311,11 @@ func TestDatabaseChecks(t *testing.T) { defer rollback() _, err = tx.Exec( - "INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('8', 'mysqld_exporter', NULL, '1', '1', '1', false, '', $1, $2, false, false, 0, false, true, 0, 0, false, false, false, false)", + `INSERT INTO agents (agent_id, agent_type, runs_on_node_id, pmm_agent_id, node_id, service_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('8', 'mysqld_exporter', NULL, '1', '1', '1', false, '', $1, $2, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": false, "rds_enhanced_metrics_disabled": false}', '{"push_metrics": false, "expose_exporter": false}')`, now, now) assertCheckViolation(t, err, "agents", "node_id_or_service_id_for_non_pmm_agent") }) }) }) } - -func TestDatabaseMigrations(t *testing.T) { - t.Run("stats_collections field migration: string to string array", func(t *testing.T) { - sqlDB := testdb.Open(t, models.SkipFixtures, pointer.ToInt(57)) - defer sqlDB.Close() //nolint:errcheck - - // Insert dummy node in DB - _, err := sqlDB.ExecContext(context.Background(), - `INSERT INTO - nodes(node_id, node_type, node_name, distro, node_model, az, address, created_at, updated_at) - VALUES - ('node_id', 'node_type', 'node_name', 'distro', 'node_model', 'az', 'address', '03/03/2014 02:03:04', '03/03/2014 02:03:04')`, - ) - require.NoError(t, err) - - // Insert dummy agent in DB - _, err = sqlDB.ExecContext(context.Background(), - `INSERT INTO - agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) - VALUES - ('{"stats_collections": "db.col1,db.col2,db.col3"}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, - ) - require.NoError(t, err) - - // Apply migration - testdb.SetupDB(t, sqlDB, models.SkipFixtures, pointer.ToInt(68)) - - var agentID string - var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) - - require.NoError(t, err) - require.Equal(t, "id", agentID) - require.Equal(t, []string{"db.col1", "db.col2", "db.col3"}, mongoDBOptions.StatsCollections) - }) - - t.Run("stats_collections field migration: string array to string array", func(t *testing.T) { - sqlDB := testdb.Open(t, models.SkipFixtures, pointer.ToInt(58)) - defer sqlDB.Close() //nolint:errcheck - - // Insert dummy node in DB - _, err := sqlDB.ExecContext(context.Background(), - `INSERT INTO - nodes(node_id, node_type, node_name, distro, node_model, az, address, created_at, updated_at) - VALUES - ('node_id', 'generic', 'node_name', 'distro', 'node_model', 'az', 'address', '03/03/2014 02:03:04', '03/03/2014 02:03:04')`, - ) - require.NoError(t, err) - - // Insert dummy agent in DB - _, err = sqlDB.ExecContext(context.Background(), - `INSERT INTO - agents(mongo_db_tls_options, agent_id, agent_type, runs_on_node_id, created_at, updated_at, disabled, status, tls, tls_skip_verify, query_examples_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics) - VALUES - ('{"stats_collections": ["db.col1", "db.col2", "db.col3"]}', 'id', 'pmm-agent', 'node_id' , '03/03/2014 02:03:04', '03/03/2014 02:03:04', false, 'alive', false, false, false, 0, 0, false, false, false)`, - ) - require.NoError(t, err) - - // Apply migration - testdb.SetupDB(t, sqlDB, models.SkipFixtures, pointer.ToInt(68)) - - var agentID string - var mongoDBOptions *models.MongoDBOptions - err = sqlDB.QueryRow(`SELECT agent_id, mongo_db_tls_options FROM agents WHERE agent_id = $1`, "id").Scan(&agentID, &mongoDBOptions) - - require.NoError(t, err) - require.Equal(t, "id", agentID) - require.Equal(t, []string{"db.col1", "db.col2", "db.col3"}, mongoDBOptions.StatsCollections) - }) -} diff --git a/managed/models/encryption_helpers.go b/managed/models/encryption_helpers.go index f3ae2e01a8..e7929dc13d 100644 --- a/managed/models/encryption_helpers.go +++ b/managed/models/encryption_helpers.go @@ -59,40 +59,33 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { agent.AgentPassword = &agentPassword } - if agent.AWSAccessKey != nil { - awsAccessKey, err := handler(*agent.AWSAccessKey) + var err error + if agent.AWSOptions != nil { + agent.AWSOptions.AWSAccessKey, err = handler(agent.AWSOptions.AWSAccessKey) if err != nil { logrus.Warning(err) } - agent.AWSAccessKey = &awsAccessKey - } - if agent.AWSSecretKey != nil { - awsSecretKey, err := handler(*agent.AWSSecretKey) + agent.AWSOptions.AWSSecretKey, err = handler(agent.AWSOptions.AWSSecretKey) if err != nil { logrus.Warning(err) } - agent.AWSSecretKey = &awsSecretKey } - var err error - if agent.MySQLOptions != nil { - agent.MySQLOptions.TLSCert, err = handler(agent.MySQLOptions.TLSCert) + if agent.AzureOptions != nil { + agent.AzureOptions.ClientID, err = handler(agent.AzureOptions.ClientID) if err != nil { logrus.Warning(err) } - agent.MySQLOptions.TLSKey, err = handler(agent.MySQLOptions.TLSKey) + agent.AzureOptions.ClientSecret, err = handler(agent.AzureOptions.ClientSecret) if err != nil { logrus.Warning(err) } - } - - if agent.PostgreSQLOptions != nil { - agent.PostgreSQLOptions.SSLCert, err = handler(agent.PostgreSQLOptions.SSLCert) + agent.AzureOptions.SubscriptionID, err = handler(agent.AzureOptions.SubscriptionID) if err != nil { logrus.Warning(err) } - agent.PostgreSQLOptions.SSLKey, err = handler(agent.PostgreSQLOptions.SSLKey) + agent.AzureOptions.TenantID, err = handler(agent.AzureOptions.TenantID) if err != nil { logrus.Warning(err) } @@ -109,20 +102,23 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { } } - if agent.AzureOptions != nil { - agent.AzureOptions.ClientID, err = handler(agent.AzureOptions.ClientID) + if agent.MySQLOptions != nil { + agent.MySQLOptions.TLSCert, err = handler(agent.MySQLOptions.TLSCert) if err != nil { logrus.Warning(err) } - agent.AzureOptions.ClientSecret, err = handler(agent.AzureOptions.ClientSecret) + agent.MySQLOptions.TLSKey, err = handler(agent.MySQLOptions.TLSKey) if err != nil { logrus.Warning(err) } - agent.AzureOptions.SubscriptionID, err = handler(agent.AzureOptions.SubscriptionID) + } + + if agent.PostgreSQLOptions != nil { + agent.PostgreSQLOptions.SSLCert, err = handler(agent.PostgreSQLOptions.SSLCert) if err != nil { logrus.Warning(err) } - agent.AzureOptions.TenantID, err = handler(agent.AzureOptions.TenantID) + agent.PostgreSQLOptions.SSLKey, err = handler(agent.PostgreSQLOptions.SSLKey) if err != nil { logrus.Warning(err) } @@ -131,18 +127,18 @@ func agentEncryption(agent Agent, handler func(string) (string, error)) Agent { return agent } -// EncryptMySQLOptionsHandler returns encrypted MySQL Options. -func EncryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return mySQLOptionsHandler(val, e.Encrypt) +// EncryptAWSOptionsHandler returns encrypted AWS Options. +func EncryptAWSOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return awsOptionsHandler(val, e.Encrypt) } -// DecryptMySQLOptionsHandler returns decrypted MySQL Options. -func DecryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return mySQLOptionsHandler(val, e.Decrypt) +// DecryptAWSOptionsHandler returns decrypted AWS Options. +func DecryptAWSOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return awsOptionsHandler(val, e.Decrypt) } -func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { - o := MySQLOptions{} +func awsOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := AWSOptions{} value := val.(*sql.NullString) //nolint:forcetypeassert if !value.Valid { return sql.NullString{}, nil @@ -153,11 +149,11 @@ func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, er return nil, err } - o.TLSCert, err = handler(o.TLSCert) + o.AWSAccessKey, err = handler(o.AWSAccessKey) if err != nil { return nil, err } - o.TLSKey, err = handler(o.TLSKey) + o.AWSSecretKey, err = handler(o.AWSSecretKey) if err != nil { return nil, err } @@ -170,18 +166,18 @@ func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, er return res, nil } -// EncryptPostgreSQLOptionsHandler returns encrypted PostgreSQL Options. -func EncryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return postgreSQLOptionsHandler(val, e.Encrypt) +// EncryptAzureOptionsHandler returns encrypted Azure Options. +func EncryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return azureOptionsHandler(val, e.Encrypt) } -// DecryptPostgreSQLOptionsHandler returns decrypted PostgreSQL Options. -func DecryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return postgreSQLOptionsHandler(val, e.Decrypt) +// DecryptAzureOptionsHandler returns decrypted Azure Options. +func DecryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return azureOptionsHandler(val, e.Decrypt) } -func postgreSQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { - o := PostgreSQLOptions{} +func azureOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := AzureOptions{} value := val.(*sql.NullString) //nolint:forcetypeassert if !value.Valid { return sql.NullString{}, nil @@ -192,11 +188,19 @@ func postgreSQLOptionsHandler(val any, handler func(string) (string, error)) (an return nil, err } - o.SSLCert, err = handler(o.SSLCert) + o.ClientID, err = handler(o.ClientID) if err != nil { return nil, err } - o.SSLKey, err = handler(o.SSLKey) + o.ClientSecret, err = handler(o.ClientSecret) + if err != nil { + return nil, err + } + o.SubscriptionID, err = handler(o.SubscriptionID) + if err != nil { + return nil, err + } + o.TenantID, err = handler(o.TenantID) if err != nil { return nil, err } @@ -248,18 +252,18 @@ func mongoDBOptionsHandler(val any, handler func(string) (string, error)) (any, return res, nil } -// EncryptAzureOptionsHandler returns encrypted Azure Options. -func EncryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return azureOptionsHandler(val, e.Encrypt) +// EncryptMySQLOptionsHandler returns encrypted MySQL Options. +func EncryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return mySQLOptionsHandler(val, e.Encrypt) } -// DecryptAzureOptionsHandler returns decrypted Azure Options. -func DecryptAzureOptionsHandler(e *encryption.Encryption, val any) (any, error) { - return azureOptionsHandler(val, e.Decrypt) +// DecryptMySQLOptionsHandler returns decrypted MySQL Options. +func DecryptMySQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return mySQLOptionsHandler(val, e.Decrypt) } -func azureOptionsHandler(val any, handler func(string) (string, error)) (any, error) { - o := AzureOptions{} +func mySQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := MySQLOptions{} value := val.(*sql.NullString) //nolint:forcetypeassert if !value.Valid { return sql.NullString{}, nil @@ -270,19 +274,50 @@ func azureOptionsHandler(val any, handler func(string) (string, error)) (any, er return nil, err } - o.ClientID, err = handler(o.ClientID) + o.TLSCert, err = handler(o.TLSCert) if err != nil { return nil, err } - o.ClientSecret, err = handler(o.ClientSecret) + o.TLSKey, err = handler(o.TLSKey) if err != nil { return nil, err } - o.SubscriptionID, err = handler(o.SubscriptionID) + + res, err := json.Marshal(o) if err != nil { return nil, err } - o.TenantID, err = handler(o.TenantID) + + return res, nil +} + +// EncryptPostgreSQLOptionsHandler returns encrypted PostgreSQL Options. +func EncryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return postgreSQLOptionsHandler(val, e.Encrypt) +} + +// DecryptPostgreSQLOptionsHandler returns decrypted PostgreSQL Options. +func DecryptPostgreSQLOptionsHandler(e *encryption.Encryption, val any) (any, error) { + return postgreSQLOptionsHandler(val, e.Decrypt) +} + +func postgreSQLOptionsHandler(val any, handler func(string) (string, error)) (any, error) { + o := PostgreSQLOptions{} + value := val.(*sql.NullString) //nolint:forcetypeassert + if !value.Valid { + return sql.NullString{}, nil + } + + err := json.Unmarshal([]byte(value.String), &o) + if err != nil { + return nil, err + } + + o.SSLCert, err = handler(o.SSLCert) + if err != nil { + return nil, err + } + o.SSLKey, err = handler(o.SSLKey) if err != nil { return nil, err } diff --git a/managed/services/agents/agents.go b/managed/services/agents/agents.go index ea5ea813ad..90c898be57 100644 --- a/managed/services/agents/agents.go +++ b/managed/services/agents/agents.go @@ -101,29 +101,31 @@ func redactWords(agent *models.Agent) []string { if s := pointer.GetString(agent.AgentPassword); s != "" { words = append(words, s) } - if s := pointer.GetString(agent.AWSSecretKey); s != "" { - words = append(words, s) + if agent.AWSOptions != nil { + if s := agent.AWSOptions.AWSSecretKey; s != "" { + words = append(words, s) + } } if agent.AzureOptions != nil { if s := agent.AzureOptions.ClientSecret; s != "" { words = append(words, s) } } - if agent.MySQLOptions != nil { - if s := agent.MySQLOptions.TLSKey; s != "" { + if agent.MongoDBOptions != nil { + if s := agent.MongoDBOptions.TLSCertificateKey; s != "" { words = append(words, s) } - } - if agent.PostgreSQLOptions != nil { - if s := agent.PostgreSQLOptions.SSLKey; s != "" { + if s := agent.MongoDBOptions.TLSCertificateKeyFilePassword; s != "" { words = append(words, s) } } - if agent.MongoDBOptions != nil { - if s := agent.MongoDBOptions.TLSCertificateKey; s != "" { + if agent.MySQLOptions != nil { + if s := agent.MySQLOptions.TLSKey; s != "" { words = append(words, s) } - if s := agent.MongoDBOptions.TLSCertificateKeyFilePassword; s != "" { + } + if agent.PostgreSQLOptions != nil { + if s := agent.PostgreSQLOptions.SSLKey; s != "" { words = append(words, s) } } @@ -170,11 +172,12 @@ func ensureAuthParams(exporter *models.Agent, params *agentv1.SetStateRequest_Ag // getExporterListenAddress returns the appropriate listen address to use for a given exporter. func getExporterListenAddress(_ *models.Node, exporter *models.Agent) string { switch { - case exporter.ExposeExporter: + case exporter.ExporterOptions.ExposeExporter: return "0.0.0.0" - case exporter.PushMetrics: + case exporter.ExporterOptions.PushMetrics: return "127.0.0.1" - default: - return "0.0.0.0" + } + + return "0.0.0.0" } diff --git a/managed/services/agents/agents_test.go b/managed/services/agents/agents_test.go index 4c3115a551..846700ac39 100644 --- a/managed/services/agents/agents_test.go +++ b/managed/services/agents/agents_test.go @@ -56,7 +56,9 @@ func TestGetExporterListenAddress(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - PushMetrics: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, } assert.Equal(t, "127.0.0.1", getExporterListenAddress(node, exporter)) @@ -66,8 +68,10 @@ func TestGetExporterListenAddress(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - PushMetrics: true, - ExposeExporter: true, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: true, + PushMetrics: true, + }, } assert.Equal(t, "0.0.0.0", getExporterListenAddress(node, exporter)) @@ -77,15 +81,19 @@ func TestGetExporterListenAddress(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - PushMetrics: false, - ExposeExporter: true, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: true, + PushMetrics: false, + }, } assert.Equal(t, "0.0.0.0", getExporterListenAddress(node, exporter)) }) t.Run("exposes exporter address if node IP is unavailable in pull mode", func(t *testing.T) { exporter := &models.Agent{ - PushMetrics: false, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, } assert.Equal(t, "0.0.0.0", getExporterListenAddress(nil, exporter)) diff --git a/managed/services/agents/connection_checker.go b/managed/services/agents/connection_checker.go index 93e5c92a73..388af0e8f5 100644 --- a/managed/services/agents/connection_checker.go +++ b/managed/services/agents/connection_checker.go @@ -60,7 +60,7 @@ func (c *ConnectionChecker) CheckConnectionToService(ctx context.Context, q *ref }() pmmAgentID := pointer.GetString(agent.PMMAgentID) - if !agent.PushMetrics && (service.ServiceType == models.ExternalServiceType || service.ServiceType == models.HAProxyServiceType) { + if !agent.ExporterOptions.PushMetrics && (service.ServiceType == models.ExternalServiceType || service.ServiceType == models.HAProxyServiceType) { pmmAgentID = models.PMMServerAgentID } diff --git a/managed/services/agents/mongodb.go b/managed/services/agents/mongodb.go index a61d6af06d..1fd19b8d60 100644 --- a/managed/services/agents/mongodb.go +++ b/managed/services/agents/mongodb.go @@ -21,8 +21,6 @@ import ( "strings" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -49,18 +47,15 @@ func mongodbExporterConfig(node *models.Node, service *models.Service, exporter args := getArgs(exporter, tdp, listenAddress, pmmAgentVersion) - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) //nolint:goconst + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) sort.Strings(args) - database := "" - if exporter.MongoDBOptions != nil { - database = exporter.MongoDBOptions.AuthenticationDatabase - } + database := exporter.MongoDBOptions.AuthenticationDatabase env := []string{ fmt.Sprintf("MONGODB_URI=%s", exporter.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: database}, tdp, pmmAgentVersion)), } @@ -95,7 +90,7 @@ func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress st args = append(args, "--discovering-mode") defaultEnabledCollectors := []string{"diagnosticdata", "replicasetstatus"} - collectAll := exporter.MongoDBOptions != nil && exporter.MongoDBOptions.EnableAllCollectors + collectAll := exporter.MongoDBOptions.EnableAllCollectors if !pmmAgentVersion.Less(v2_26_0) { defaultEnabledCollectors = []string{} @@ -117,23 +112,21 @@ func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress st args = append(args, "--collector.pbm") } - args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors) - args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.DisabledCollectors)...) + args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) + args = append(args, collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultEnabledCollectors, exporter.ExporterOptions.DisabledCollectors)...) - if exporter.MongoDBOptions != nil && len(exporter.MongoDBOptions.StatsCollections) != 0 { + if len(exporter.MongoDBOptions.StatsCollections) != 0 { args = append(args, "--mongodb.collstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) if !pmmAgentVersion.Less(v2_26_0) { args = append(args, "--mongodb.indexstats-colls="+strings.Join(exporter.MongoDBOptions.StatsCollections, ",")) } } - if exporter.MongoDBOptions != nil { - collstatsLimit := int32(200) - if exporter.MongoDBOptions.CollectionsLimit != -1 { - collstatsLimit = exporter.MongoDBOptions.CollectionsLimit - } - args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit)) + collstatsLimit := int32(200) + if exporter.MongoDBOptions.CollectionsLimit != -1 { + collstatsLimit = exporter.MongoDBOptions.CollectionsLimit } + args = append(args, fmt.Sprintf("--collector.collstats-limit=%d", collstatsLimit)) case !pmmAgentVersion.Less(newMongoExporterPMMVersion): // >= 2.10.0 args = buildBaseArgs(listenAddress, tdp) @@ -148,7 +141,7 @@ func getArgs(exporter *models.Agent, tdp *models.DelimiterPair, listenAddress st "--web.listen-address=" + listenAddress + ":" + tdp.Left + " .listen_port " + tdp.Right, //nolint:goconst } - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) } return args @@ -165,11 +158,12 @@ func buildBaseArgs(listenAddress string, tdp *models.DelimiterPair) []string { // qanMongoDBProfilerAgentConfig returns desired configuration of qan-mongodb-profiler-agent built-in agent. func qanMongoDBProfilerAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MONGODB_PROFILER_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), - DisableQueryExamples: agent.QueryExamplesDisabled, - MaxQueryLength: agent.MaxQueryLength, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, diff --git a/managed/services/agents/mongodb_test.go b/managed/services/agents/mongodb_test.go index c0357e7464..6cb5f285a2 100644 --- a/managed/services/agents/mongodb_test.go +++ b/managed/services/agents/mongodb_test.go @@ -38,11 +38,13 @@ func TestMongodbExporterConfig225(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) @@ -51,6 +53,7 @@ func TestMongodbExporterConfig225(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--compatible-mode", "--discovering-mode", "--mongodb.global-conn-pool", @@ -97,11 +100,13 @@ func TestMongodbExporterConfig226(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -109,6 +114,7 @@ func TestMongodbExporterConfig226(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.replicasetstatus", "--compatible-mode", @@ -242,11 +248,13 @@ func TestMongodbExporterConfig2411(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -254,6 +262,7 @@ func TestMongodbExporterConfig2411(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.replicasetstatus", "--compatible-mode", @@ -387,11 +396,13 @@ func TestMongodbExporterConfig2411(t *testing.T) { }) t.Run("Enable all collectors and disable some", func(t *testing.T) { + exporter.ExporterOptions = &models.ExporterOptions{ + DisabledCollectors: []string{"dbstats", "topmetrics"}, + } exporter.MongoDBOptions = &models.MongoDBOptions{ EnableAllCollectors: true, StatsCollections: []string{"db1.col1.one", "db2.col2", "db3"}, } - exporter.DisabledCollectors = []string{"dbstats", "topmetrics"} expected.Args = []string{ "--collector.collstats", @@ -426,11 +437,13 @@ func TestMongodbExporterConfig2430(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -438,6 +451,7 @@ func TestMongodbExporterConfig2430(t *testing.T) { TemplateLeftDelim: "{{", TemplateRightDelim: "}}", Args: []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.fcv", "--collector.pbm", @@ -506,11 +520,13 @@ func TestMongodbExporterConfig(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -597,7 +613,9 @@ func TestMongodbExporterConfig(t *testing.T) { }) t.Run("DisabledCollectors", func(t *testing.T) { - exporter.DisabledCollectors = []string{"topmetrics"} + exporter.ExporterOptions = &models.ExporterOptions{ + DisabledCollectors: []string{"topmetrics"}, + } actual, err := mongodbExporterConfig(node, mongodb, exporter, exposeSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ Type: inventoryv1.AgentType_AGENT_TYPE_MONGODB_EXPORTER, @@ -627,10 +645,12 @@ func TestNewMongodbExporterConfig(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, exporter, redactSecrets, pmmAgentVersion) require.NoError(t, err) @@ -683,13 +703,16 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { Port: pointer.ToUint16(27017), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MongoDBExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.MongoDBExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } expectedArgs := []string{ + "--collector.collstats-limit=0", "--collector.diagnosticdata", "--collector.replicasetstatus", "--compatible-mode", @@ -707,11 +730,13 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { t.Parallel() localExporter := &models.Agent{ - AgentID: exporter.AgentID, - AgentType: exporter.AgentType, - Username: exporter.Username, - Password: exporter.Password, - AgentPassword: pointer.ToString("agent-custom-password"), + AgentID: exporter.AgentID, + AgentType: exporter.AgentType, + Username: exporter.Username, + Password: exporter.Password, + AgentPassword: pointer.ToString("agent-custom-password"), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, localExporter, redactSecrets, pmmAgentVersion) @@ -736,10 +761,12 @@ func TestMongodbExporterConfig228_WebConfigAuth(t *testing.T) { t.Parallel() localExporter := &models.Agent{ - AgentID: exporter.AgentID, - AgentType: exporter.AgentType, - Username: exporter.Username, - Password: exporter.Password, + AgentID: exporter.AgentID, + AgentType: exporter.AgentType, + Username: exporter.Username, + Password: exporter.Password, + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{}, } actual, err := mongodbExporterConfig(node, mongodb, localExporter, redactSecrets, pmmAgentVersion) diff --git a/managed/services/agents/mysql.go b/managed/services/agents/mysql.go index 54f13f9dfc..349cfde831 100644 --- a/managed/services/agents/mysql.go +++ b/managed/services/agents/mysql.go @@ -20,8 +20,6 @@ import ( "sort" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -106,10 +104,10 @@ func mysqldExporterConfig( args = append(args, tablestatsGroup...) } - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } files := exporter.Files() @@ -156,12 +154,13 @@ func mysqldExporterConfig( // qanMySQLPerfSchemaAgentConfig returns desired configuration of qan-mysql-perfschema built-in agent. func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_PERFSCHEMA_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), - MaxQueryLength: agent.MaxQueryLength, - DisableQueryExamples: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, @@ -174,13 +173,14 @@ func qanMySQLPerfSchemaAgentConfig(service *models.Service, agent *models.Agent, // qanMySQLSlowlogAgentConfig returns desired configuration of qan-mysql-slowlog built-in agent. func qanMySQLSlowlogAgentConfig(service *models.Service, agent *models.Agent, pmmAgentVersion *version.Parsed) *agentv1.SetStateRequest_BuiltinAgent { tdp := agent.TemplateDelimiters(service) + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_MYSQL_SLOWLOG_AGENT, Dsn: agent.DSN(service, models.DSNParams{DialTimeout: time.Second, Database: ""}, nil, pmmAgentVersion), - MaxQueryLength: agent.MaxQueryLength, - DisableQueryExamples: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, - MaxQueryLogSize: agent.MaxQueryLogSize, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, + MaxQueryLogSize: agent.QANOptions.MaxQueryLogSize, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, diff --git a/managed/services/agents/mysql_test.go b/managed/services/agents/mysql_test.go index 6cee95abeb..8bd6ca1f4e 100644 --- a/managed/services/agents/mysql_test.go +++ b/managed/services/agents/mysql_test.go @@ -37,11 +37,13 @@ func TestMySQLdExporterConfig(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, + MySQLOptions: &models.MySQLOptions{}, } pmmAgentVersion := version.MustParse("2.21.0") @@ -141,16 +143,17 @@ func TestMySQLdExporterConfigTablestatsGroupDisabled(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TableCountTablestatsGroupLimit: -1, - TLS: true, + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + ExporterOptions: &models.ExporterOptions{}, MySQLOptions: &models.MySQLOptions{ - TLSCa: "content-of-tls-ca", - TLSCert: "content-of-tls-cert", - TLSKey: "content-of-tls-key", + TableCountTablestatsGroupLimit: -1, + TLSCa: "content-of-tls-ca", + TLSCert: "content-of-tls-cert", + TLSKey: "content-of-tls-key", }, } pmmAgentVersion := version.MustParse("2.24.0") @@ -245,11 +248,14 @@ func TestMySQLdExporterConfigDisabledCollectors(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.MySQLdExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - DisabledCollectors: []string{"heartbeat", "info_schema.clientstats", "perf_schema.eventsstatements", "custom_query.hr"}, + AgentID: "agent-id", + AgentType: models.MySQLdExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"heartbeat", "info_schema.clientstats", "perf_schema.eventsstatements", "custom_query.hr"}, + }, + MySQLOptions: &models.MySQLOptions{}, } pmmAgentVersion := version.MustParse("2.24.0") diff --git a/managed/services/agents/node.go b/managed/services/agents/node.go index 4aac0b2422..b81c763ad2 100644 --- a/managed/services/agents/node.go +++ b/managed/services/agents/node.go @@ -18,8 +18,6 @@ package agents import ( "sort" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -36,11 +34,7 @@ var ( func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion *version.Parsed) (*agentv1.SetStateRequest_AgentProcess, error) { listenAddress := getExporterListenAddress(node, exporter) - - tdp := models.TemplateDelimsPair( - pointer.GetString(exporter.MetricsPath), - ) - + tdp := models.TemplateDelimsPair(exporter.ExporterOptions.MetricsPath) args := []string{ "--collector.textfile.directory.lr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/low-resolution", "--collector.textfile.directory.mr=" + pathsBase(agentVersion, tdp.Left, tdp.Right) + "/collectors/textfile-collector/medium-resolution", @@ -127,10 +121,10 @@ func nodeExporterConfig(node *models.Node, exporter *models.Agent, agentVersion ) } - args = collectors.FilterOutCollectors("--collector.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collector.", args, exporter.ExporterOptions.DisabledCollectors) - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } args = withLogLevel(args, exporter.LogLevel, agentVersion, false) diff --git a/managed/services/agents/node_test.go b/managed/services/agents/node_test.go index 01bbdaf279..8b70d371d3 100644 --- a/managed/services/agents/node_test.go +++ b/managed/services/agents/node_test.go @@ -34,8 +34,9 @@ func TestAuthWebConfig(t *testing.T) { node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.26.1") @@ -58,8 +59,9 @@ func TestAuthWebConfig(t *testing.T) { node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.28.0") @@ -83,8 +85,9 @@ func TestAuthWebConfig(t *testing.T) { node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("3.0.0") @@ -114,8 +117,9 @@ func TestNodeExporterConfig(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.15.1") @@ -206,9 +210,11 @@ func TestNodeExporterConfig(t *testing.T) { t.Parallel() node := &models.Node{} exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, - DisabledCollectors: []string{"cpu", "netstat", "netstat.fields", "vmstat", "meminfo"}, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"cpu", "netstat", "netstat.fields", "vmstat", "meminfo"}, + }, } agentVersion := version.MustParse("2.15.1") @@ -291,8 +297,9 @@ func TestNodeExporterConfig(t *testing.T) { Distro: "darwin", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.NodeExporterType, + AgentID: "agent-id", + AgentType: models.NodeExporterType, + ExporterOptions: &models.ExporterOptions{}, } agentVersion := version.MustParse("2.15.1") diff --git a/managed/services/agents/postgresql.go b/managed/services/agents/postgresql.go index 7f2c035f90..37c78edc9d 100644 --- a/managed/services/agents/postgresql.go +++ b/managed/services/agents/postgresql.go @@ -89,13 +89,13 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter autoDiscovery := false if !pmmAgentVersion.Less(postgresExporterAutodiscoveryVersion) { switch { - case exporter.PostgreSQLOptions == nil: + case exporter.PostgreSQLOptions.AutoDiscoveryLimit == nil: autoDiscovery = true - case exporter.PostgreSQLOptions.AutoDiscoveryLimit == 0: // server defined + case pointer.GetInt32(exporter.PostgreSQLOptions.AutoDiscoveryLimit) == 0: // server defined autoDiscovery = exporter.PostgreSQLOptions.DatabaseCount <= defaultAutoDiscoveryDatabaseLimit - case exporter.PostgreSQLOptions.AutoDiscoveryLimit < 0: // always disabled + case pointer.GetInt32(exporter.PostgreSQLOptions.AutoDiscoveryLimit) < 0: // always disabled default: - autoDiscovery = exporter.PostgreSQLOptions.DatabaseCount <= exporter.PostgreSQLOptions.AutoDiscoveryLimit + autoDiscovery = exporter.PostgreSQLOptions.DatabaseCount <= pointer.GetInt32(exporter.PostgreSQLOptions.AutoDiscoveryLimit) } } if autoDiscovery { @@ -105,19 +105,18 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter } if !pmmAgentVersion.Less(postgresMaxExporterConnsVersion) && - exporter.PostgreSQLOptions != nil && exporter.PostgreSQLOptions.MaxExporterConnections != 0 { args = append(args, "--max-connections="+strconv.Itoa(int(exporter.PostgreSQLOptions.MaxExporterConnections))) } - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "--web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "--web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } - args = collectors.FilterOutCollectors("--collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("--collect.", args, exporter.ExporterOptions.DisabledCollectors) if !pmmAgentVersion.Less(postgresExporterCollectorsVersion) { - disableCollectorArgs := collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultPostgresExporterCollectors, exporter.DisabledCollectors) + disableCollectorArgs := collectors.DisableDefaultEnabledCollectors("--no-collector.", defaultPostgresExporterCollectors, exporter.ExporterOptions.DisabledCollectors) //nolint:lll args = append(args, disableCollectorArgs...) } @@ -131,7 +130,7 @@ func postgresExporterConfig(node *models.Node, service *models.Service, exporter PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } - if exporter.AzureOptions != nil { + if exporter.AzureOptions.ClientID != "" { dnsParams.DialTimeout = 5 * time.Second } @@ -165,11 +164,12 @@ func qanPostgreSQLPgStatementsAgentConfig(service *models.Service, agent *models Database: service.DatabaseName, PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATEMENTS_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), - MaxQueryLength: agent.MaxQueryLength, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, @@ -186,12 +186,13 @@ func qanPostgreSQLPgStatMonitorAgentConfig(service *models.Service, agent *model Database: service.DatabaseName, PostgreSQLSupportsSSLSNI: !pmmAgentVersion.Less(postgresSSLSniVersion), } + return &agentv1.SetStateRequest_BuiltinAgent{ Type: inventoryv1.AgentType_AGENT_TYPE_QAN_POSTGRESQL_PGSTATMONITOR_AGENT, Dsn: agent.DSN(service, dnsParams, nil, pmmAgentVersion), - DisableQueryExamples: agent.QueryExamplesDisabled, - MaxQueryLength: agent.MaxQueryLength, - DisableCommentsParsing: agent.CommentsParsingDisabled, + DisableQueryExamples: agent.QANOptions.QueryExamplesDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, TextFiles: &agentv1.TextFiles{ Files: agent.Files(), TemplateLeftDelim: tdp.Left, diff --git a/managed/services/agents/postgresql_test.go b/managed/services/agents/postgresql_test.go index f51b858fb7..a1d115f127 100644 --- a/managed/services/agents/postgresql_test.go +++ b/managed/services/agents/postgresql_test.go @@ -78,6 +78,10 @@ func (s *PostgresExporterConfigTestSuite) SetupTest() { } func (s *PostgresExporterConfigTestSuite) TestConfig() { + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -92,6 +96,10 @@ func (s *PostgresExporterConfigTestSuite) TestDatabaseName() { s.postgresql.DatabaseName = "db1" s.expected.Env[0] = "DATA_SOURCE_NAME=postgres://username:s3cur3%20p%40$$w0r4.@1.2.3.4:5432/db1?connect_timeout=1&sslmode=disable" + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -102,6 +110,10 @@ func (s *PostgresExporterConfigTestSuite) TestDatabaseName() { s.postgresql.DatabaseName = "" s.Require().PanicsWithValue("database name not set", func() { + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} + _, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") }) @@ -110,6 +122,9 @@ func (s *PostgresExporterConfigTestSuite) TestDatabaseName() { func (s *PostgresExporterConfigTestSuite) TestEmptyPassword() { s.exporter.Password = nil + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -119,6 +134,9 @@ func (s *PostgresExporterConfigTestSuite) TestEmptyPassword() { func (s *PostgresExporterConfigTestSuite) TestEmptyUsername() { s.exporter.Username = nil + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -129,6 +147,9 @@ func (s *PostgresExporterConfigTestSuite) TestEmptyUsername() { func (s *PostgresExporterConfigTestSuite) TestEmptyUsernameAndPassword() { s.exporter.Username = nil s.exporter.Password = nil + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -142,6 +163,9 @@ func (s *PostgresExporterConfigTestSuite) TestSocket() { s.postgresql.Address = nil s.postgresql.Port = nil s.postgresql.Socket = pointer.ToString("/var/run/postgres") + s.exporter.ExporterOptions = &models.ExporterOptions{} + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -154,7 +178,11 @@ func (s *PostgresExporterConfigTestSuite) TestDisabledCollectors() { s.postgresql.Address = nil s.postgresql.Port = nil s.postgresql.Socket = pointer.ToString("/var/run/postgres") - s.exporter.DisabledCollectors = []string{"custom_query.hr", "custom_query.hr.directory", "locks"} + s.exporter.ExporterOptions = &models.ExporterOptions{ + DisabledCollectors: []string{"custom_query.hr", "custom_query.hr.directory", "locks"}, + } + s.exporter.AzureOptions = &models.AzureOptions{} + s.exporter.PostgreSQLOptions = &models.PostgreSQLOptions{} actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, exposeSecrets, s.pmmAgentVersion) s.NoError(err, "Failed to create exporter config") @@ -194,10 +222,13 @@ func TestAutoDiscovery(t *testing.T) { DatabaseName: "postgres", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } expected := &agentv1.SetStateRequest_AgentProcess{ @@ -238,7 +269,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Database count more than limit - disabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 5, + AutoDiscoveryLimit: pointer.ToInt32(5), DatabaseCount: 10, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -249,7 +280,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Database count equal to limit - enabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 5, + AutoDiscoveryLimit: pointer.ToInt32(5), DatabaseCount: 5, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -260,7 +291,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Database count less than limit - enabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 5, + AutoDiscoveryLimit: pointer.ToInt32(5), DatabaseCount: 3, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -271,7 +302,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Negative limit - disabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: -1, + AutoDiscoveryLimit: pointer.ToInt32(-1), DatabaseCount: 3, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -282,7 +313,7 @@ func TestAutoDiscovery(t *testing.T) { t.Run("Default - enabled", func(t *testing.T) { exporter.PostgreSQLOptions = &models.PostgreSQLOptions{ - AutoDiscoveryLimit: 0, + AutoDiscoveryLimit: pointer.ToInt32(0), DatabaseCount: 3, } res, err := postgresExporterConfig(node, postgresql, exporter, redactSecrets, pmmAgentVersion) @@ -306,10 +337,12 @@ func TestMaxConnections(t *testing.T) { DatabaseName: "postgres", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, PostgreSQLOptions: &models.PostgreSQLOptions{ MaxExporterConnections: 10, }, @@ -379,10 +412,11 @@ func (s *PostgresExporterConfigTestSuite) TestAzureTimeout() { DatabaseName: "postgres", } s.exporter = &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, AzureOptions: &models.AzureOptions{ SubscriptionID: "subscription_id", ClientID: "client_id", @@ -390,6 +424,7 @@ func (s *PostgresExporterConfigTestSuite) TestAzureTimeout() { TenantID: "tenant_id", ResourceGroup: "resource_group", }, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) @@ -431,11 +466,14 @@ func (s *PostgresExporterConfigTestSuite) TestPrometheusWebConfig() { DatabaseName: "postgres", } s.exporter = &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) @@ -480,11 +518,14 @@ func (s *PostgresExporterConfigTestSuite) TestSSLSni() { DatabaseName: "postgres", } s.exporter = &models.Agent{ - AgentID: "agent-id", - AgentType: models.PostgresExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - TLS: true, + AgentID: "agent-id", + AgentType: models.PostgresExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + TLS: true, + ExporterOptions: &models.ExporterOptions{}, + AzureOptions: &models.AzureOptions{}, + PostgreSQLOptions: &models.PostgreSQLOptions{}, } actual, err := postgresExporterConfig(s.node, s.postgresql, s.exporter, redactSecrets, s.pmmAgentVersion) diff --git a/managed/services/agents/proxysql.go b/managed/services/agents/proxysql.go index 2de7a16fdc..8f786c25e1 100644 --- a/managed/services/agents/proxysql.go +++ b/managed/services/agents/proxysql.go @@ -20,8 +20,6 @@ import ( "sort" "time" - "github.com/AlekSi/pointer" - agentv1 "github.com/percona/pmm/api/agent/v1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" "github.com/percona/pmm/managed/models" @@ -57,11 +55,11 @@ func proxysqlExporterConfig(node *models.Node, service *models.Service, exporter args = append(args, "-collect.runtime_mysql_servers") } - if pointer.GetString(exporter.MetricsPath) != "" { - args = append(args, "-web.telemetry-path="+*exporter.MetricsPath) + if exporter.ExporterOptions.MetricsPath != "" { + args = append(args, "-web.telemetry-path="+exporter.ExporterOptions.MetricsPath) } - args = collectors.FilterOutCollectors("-collect.", args, exporter.DisabledCollectors) + args = collectors.FilterOutCollectors("-collect.", args, exporter.ExporterOptions.DisabledCollectors) args = withLogLevel(args, exporter.LogLevel, pmmAgentVersion, true) diff --git a/managed/services/agents/proxysql_test.go b/managed/services/agents/proxysql_test.go index e3e2b46ddb..ba33c65943 100644 --- a/managed/services/agents/proxysql_test.go +++ b/managed/services/agents/proxysql_test.go @@ -39,11 +39,12 @@ func TestProxySQLExporterConfig(t *testing.T) { Address: "1.2.3.4", } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.ProxySQLExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), - AgentPassword: pointer.ToString("agent-password"), + AgentID: "agent-id", + AgentType: models.ProxySQLExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentPassword: pointer.ToString("agent-password"), + ExporterOptions: &models.ExporterOptions{}, } actual := proxysqlExporterConfig(node, proxysql, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -80,7 +81,7 @@ func TestProxySQLExporterConfig(t *testing.T) { }) t.Run("DisabledCollector", func(t *testing.T) { - exporter.DisabledCollectors = []string{"mysql_connection_list", "stats_memory_metrics"} + exporter.ExporterOptions.DisabledCollectors = []string{"mysql_connection_list", "stats_memory_metrics"} actual := proxysqlExporterConfig(node, proxysql, exporter, exposeSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ Type: inventoryv1.AgentType_AGENT_TYPE_PROXYSQL_EXPORTER, @@ -104,10 +105,11 @@ func TestProxySQLExporterConfig(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.ProxySQLExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.ProxySQLExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, } actual := proxysqlExporterConfig(node, proxysql, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ @@ -144,10 +146,11 @@ func TestProxySQLExporterConfig(t *testing.T) { Port: pointer.ToUint16(3306), } exporter := &models.Agent{ - AgentID: "agent-id", - AgentType: models.ProxySQLExporterType, - Username: pointer.ToString("username"), - Password: pointer.ToString("s3cur3 p@$$w0r4."), + AgentID: "agent-id", + AgentType: models.ProxySQLExporterType, + Username: pointer.ToString("username"), + Password: pointer.ToString("s3cur3 p@$$w0r4."), + ExporterOptions: &models.ExporterOptions{}, } actual := proxysqlExporterConfig(node, proxysql, exporter, redactSecrets, pmmAgentVersion) expected := &agentv1.SetStateRequest_AgentProcess{ diff --git a/managed/services/agents/rds.go b/managed/services/agents/rds.go index 158925f07c..1be5fe70ee 100644 --- a/managed/services/agents/rds.go +++ b/managed/services/agents/rds.go @@ -80,11 +80,11 @@ func rdsExporterConfig(pairs map[*models.Node]*models.Agent, redactMode redactMo config.Instances = append(config.Instances, rdsInstance{ Region: pointer.GetString(node.Region), Instance: node.Address, - AWSAccessKey: pointer.GetString(exporter.AWSAccessKey), - AWSSecretKey: pointer.GetString(exporter.AWSSecretKey), + AWSAccessKey: exporter.AWSOptions.AWSAccessKey, + AWSSecretKey: exporter.AWSOptions.AWSSecretKey, Labels: labels, - DisableBasicMetrics: exporter.RDSBasicMetricsDisabled, - DisableEnhancedMetrics: exporter.RDSEnhancedMetricsDisabled, + DisableBasicMetrics: exporter.AWSOptions.RDSBasicMetricsDisabled, + DisableEnhancedMetrics: exporter.AWSOptions.RDSEnhancedMetricsDisabled, }) if redactMode != exposeSecrets { diff --git a/managed/services/agents/rds_test.go b/managed/services/agents/rds_test.go index 5a398d30a4..3cecdef1c6 100644 --- a/managed/services/agents/rds_test.go +++ b/managed/services/agents/rds_test.go @@ -45,12 +45,14 @@ func TestRDSExporterConfig(t *testing.T) { }) require.NoError(t, err) agent1 := &models.Agent{ - AgentID: "agent1", - AgentType: models.RDSExporterType, - NodeID: &node1.NodeID, - AWSAccessKey: pointer.ToString("access_key1"), - AWSSecretKey: pointer.ToString("secret_key1"), - RDSBasicMetricsDisabled: true, + AgentID: "agent1", + AgentType: models.RDSExporterType, + NodeID: &node1.NodeID, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: "access_key1", + AWSSecretKey: "secret_key1", + RDSBasicMetricsDisabled: true, + }, } node2 := &models.Node{ @@ -67,11 +69,13 @@ func TestRDSExporterConfig(t *testing.T) { }) require.NoError(t, err) agent2 := &models.Agent{ - AgentID: "agent2", - AgentType: models.RDSExporterType, - NodeID: &node2.NodeID, - AWSAccessKey: pointer.ToString("access_key2"), - AWSSecretKey: pointer.ToString("secret_key2"), + AgentID: "agent2", + AgentType: models.RDSExporterType, + NodeID: &node2.NodeID, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: "access_key2", + AWSSecretKey: "secret_key2", + }, } pairs := map[*models.Node]*models.Agent{ diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index de86982afc..3b63fc19dc 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -321,9 +321,11 @@ func (r *Registry) addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNod } if len(vmAgent) == 0 { if _, err := models.CreateAgent(q, models.VMAgentType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - PushMetrics: true, - NodeID: runsOnNodeID, + PMMAgentID: pmmAgentID, + NodeID: runsOnNodeID, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, }); err != nil { return errors.Wrapf(err, "Can't create 'vmAgent' for pmm-agent with ID %q", pmmAgentID) } @@ -349,9 +351,9 @@ func removeVMAgentFromPMMAgent(q *reform.Querier, pmmAgentID string) error { return errors.Wrapf(err, "Can't find agents for pmm-agent with ID %q", pmmAgentID) } for _, agent := range agents { - if agent.PushMetrics { + if agent.ExporterOptions.PushMetrics { logrus.Warnf("disabling push_metrics for agent with unsupported version ID %q with pmm-agent ID %q", agent.AgentID, pmmAgentID) - agent.PushMetrics = false + agent.ExporterOptions.PushMetrics = false if err := q.Update(agent); err != nil { return errors.Wrapf(err, "Can't set push_metrics=false for agent %q at pmm-agent with ID %q", agent.AgentID, pmmAgentID) } diff --git a/managed/services/agents/roster_test.go b/managed/services/agents/roster_test.go index 1b67f0f458..f7b8b5fcf4 100644 --- a/managed/services/agents/roster_test.go +++ b/managed/services/agents/roster_test.go @@ -18,7 +18,6 @@ package agents import ( "testing" - "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/reform.v1" @@ -56,9 +55,11 @@ func TestRoster(t *testing.T) { } awsAccessKey := "aws" exporters[node] = &models.Agent{ - AgentID: "agent1", - AgentType: models.RDSExporterType, - AWSAccessKey: pointer.ToString(awsAccessKey), + AgentID: "agent1", + AgentType: models.RDSExporterType, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: awsAccessKey, + }, } const expected = "pmm-server:rds/aws" @@ -94,9 +95,11 @@ func TestRoster(t *testing.T) { } awsAccessKey := "aws" exporters[node] = &models.Agent{ - AgentID: "agent1", - AgentType: models.RDSExporterType, - AWSAccessKey: pointer.ToString(awsAccessKey), + AgentID: "agent1", + AgentType: models.RDSExporterType, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: awsAccessKey, + }, } const expectedGroupID = "pmm-server:rds/aws" diff --git a/managed/services/agents/service_info_broker.go b/managed/services/agents/service_info_broker.go index 6802de66f8..5927fb20ce 100644 --- a/managed/services/agents/service_info_broker.go +++ b/managed/services/agents/service_info_broker.go @@ -180,7 +180,7 @@ func (c *ServiceInfoBroker) GetInfoFromService(ctx context.Context, q *reform.Qu stype := service.ServiceType switch stype { case models.MySQLServiceType: - agent.TableCount = &sInfo.TableCount + agent.MySQLOptions.TableCount = &sInfo.TableCount l.Debugf("Updating table count: %d.", sInfo.TableCount) encryptedAgent := models.EncryptAgent(*agent) if err = q.Update(&encryptedAgent); err != nil { @@ -189,10 +189,6 @@ func (c *ServiceInfoBroker) GetInfoFromService(ctx context.Context, q *reform.Qu return updateServiceVersion(ctx, q, resp, service) case models.PostgreSQLServiceType: - if agent.PostgreSQLOptions == nil { - agent.PostgreSQLOptions = &models.PostgreSQLOptions{} - } - databaseList := sInfo.DatabaseList databaseCount := len(databaseList) excludedDatabaseList := postgresExcludedDatabases() diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index afc92816a0..c10c71b567 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -268,7 +268,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI // Iterate over the rdsExporters map for node, exporter := range rdsExporters { - awsAccessKey := pointer.GetString(exporter.AWSAccessKey) + awsAccessKey := exporter.AWSOptions.AWSAccessKey if _, ok := groupedRdsExporters[awsAccessKey]; !ok { groupedRdsExporters[awsAccessKey] = make(map[*models.Node]*models.Agent) diff --git a/managed/services/converters.go b/managed/services/converters.go index 10589cc07f..a83be2abf8 100644 --- a/managed/services/converters.go +++ b/managed/services/converters.go @@ -218,6 +218,7 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro } serviceID = service.ServiceID } + processExecPath := pointer.GetString(agent.ProcessExecPath) switch agent.AgentType { case models.PMMAgentType: @@ -236,12 +237,12 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.MySQLdExporterType: @@ -256,15 +257,15 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - TablestatsGroupTableLimit: agent.TableCountTablestatsGroupLimit, + TablestatsGroupTableLimit: agent.MySQLOptions.TableCountTablestatsGroupLimit, TablestatsGroupDisabled: !agent.IsMySQLTablestatsGroupEnabled(), - TableCount: pointer.GetInt32(agent.TableCount), - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + TableCount: pointer.GetInt32(agent.MySQLOptions.TableCount), + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.MongoDBExporterType: @@ -279,18 +280,18 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), - } - if agent.MongoDBOptions != nil { - exporter.StatsCollections = agent.MongoDBOptions.StatsCollections - exporter.CollectionsLimit = agent.MongoDBOptions.CollectionsLimit - exporter.EnableAllCollectors = agent.MongoDBOptions.EnableAllCollectors + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } + + exporter.StatsCollections = agent.MongoDBOptions.StatsCollections + exporter.CollectionsLimit = agent.MongoDBOptions.CollectionsLimit + exporter.EnableAllCollectors = agent.MongoDBOptions.EnableAllCollectors + return exporter, nil case models.PostgresExporterType: @@ -305,17 +306,17 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), - } - if agent.PostgreSQLOptions != nil { - exporter.AutoDiscoveryLimit = agent.PostgreSQLOptions.AutoDiscoveryLimit - exporter.MaxExporterConnections = agent.PostgreSQLOptions.MaxExporterConnections + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), } + + exporter.AutoDiscoveryLimit = pointer.GetInt32(agent.PostgreSQLOptions.AutoDiscoveryLimit) + exporter.MaxExporterConnections = agent.PostgreSQLOptions.MaxExporterConnections + return exporter, nil case models.QANMySQLPerfSchemaAgentType: return &inventoryv1.QANMySQLPerfSchemaAgent{ @@ -328,9 +329,9 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - MaxQueryLength: agent.MaxQueryLength, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), }, nil @@ -346,9 +347,9 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, - MaxSlowlogFileSize: agent.MaxQueryLogSize, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, + MaxSlowlogFileSize: agent.QANOptions.MaxQueryLogSize, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), }, nil @@ -364,7 +365,7 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - MaxQueryLength: agent.MaxQueryLength, + MaxQueryLength: agent.QANOptions.MaxQueryLength, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-4650 @@ -382,12 +383,12 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - PushMetricsEnabled: agent.PushMetrics, - DisabledCollectors: agent.DisabledCollectors, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, + DisabledCollectors: agent.ExporterOptions.DisabledCollectors, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - ExposeExporter: agent.ExposeExporter, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + ExposeExporter: agent.ExporterOptions.ExposeExporter, + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.QANPostgreSQLPgStatementsAgentType: @@ -399,8 +400,8 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro Disabled: agent.Disabled, Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), CustomLabels: labels, - MaxQueryLength: agent.MaxQueryLength, - DisableCommentsParsing: agent.CommentsParsingDisabled, + MaxQueryLength: agent.QANOptions.MaxQueryLength, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, ProcessExecPath: processExecPath, @@ -416,11 +417,11 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro Disabled: agent.Disabled, Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), CustomLabels: labels, - MaxQueryLength: agent.MaxQueryLength, + MaxQueryLength: agent.QANOptions.MaxQueryLength, Tls: agent.TLS, TlsSkipVerify: agent.TLSSkipVerify, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - DisableCommentsParsing: agent.CommentsParsingDisabled, + QueryExamplesDisabled: agent.QANOptions.QueryExamplesDisabled, + DisableCommentsParsing: agent.QANOptions.CommentsParsingDisabled, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), }, nil @@ -431,16 +432,16 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro PmmAgentId: pointer.GetString(agent.PMMAgentID), NodeId: nodeID, Disabled: agent.Disabled, - AwsAccessKey: pointer.GetString(agent.AWSAccessKey), + AwsAccessKey: agent.AWSOptions.AWSAccessKey, Status: inventoryv1.AgentStatus(inventoryv1.AgentStatus_value[agent.Status]), ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, - BasicMetricsDisabled: agent.RDSBasicMetricsDisabled, - EnhancedMetricsDisabled: agent.RDSEnhancedMetricsDisabled, - PushMetricsEnabled: agent.PushMetrics, + BasicMetricsDisabled: agent.AWSOptions.RDSBasicMetricsDisabled, + EnhancedMetricsDisabled: agent.AWSOptions.RDSEnhancedMetricsDisabled, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.ExternalExporterType: @@ -457,13 +458,13 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro ServiceId: pointer.GetString(agent.ServiceID), Username: pointer.GetString(agent.Username), Disabled: agent.Disabled, - Scheme: pointer.GetString(agent.MetricsScheme), - MetricsPath: pointer.GetString(agent.MetricsPath), + Scheme: agent.ExporterOptions.MetricsScheme, + MetricsPath: agent.ExporterOptions.MetricsPath, ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), CustomLabels: labels, - PushMetricsEnabled: agent.PushMetrics, + PushMetricsEnabled: agent.ExporterOptions.PushMetrics, ProcessExecPath: processExecPath, - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.AzureDatabaseExporterType: @@ -478,7 +479,7 @@ func ToAPIAgent(q *reform.Querier, agent *models.Agent) (inventoryv1.Agent, erro CustomLabels: labels, ProcessExecPath: processExecPath, LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MetricsResolutions: ConvertMetricsResolutions(agent.MetricsResolutions), + MetricsResolutions: ConvertMetricsResolutions(agent.ExporterOptions.MetricsResolutions), }, nil case models.VMAgentType: diff --git a/managed/services/encryption/encryption_rotation.go b/managed/services/encryption/encryption_rotation.go index f73f6d96ed..b522998fa9 100644 --- a/managed/services/encryption/encryption_rotation.go +++ b/managed/services/encryption/encryption_rotation.go @@ -130,7 +130,7 @@ func pmmServerStatusWithRetries(status string) bool { func rotateEncryptionKey(db *reform.DB, dbName string) error { return db.InTransaction(func(tx *reform.TX) error { logrus.Infof("DB %s is being decrypted", dbName) - err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumnsV3) if err != nil { return err } @@ -144,7 +144,7 @@ func rotateEncryptionKey(db *reform.DB, dbName string) error { logrus.Infof("New encryption key generated") logrus.Infof("DB %s is being encrypted", dbName) - err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumnsV3) if err != nil { if e := encryption.RestoreOldEncryptionKey(); e != nil { return errors.Wrap(err, e.Error()) diff --git a/managed/services/encryption/encryption_rotation_test.go b/managed/services/encryption/encryption_rotation_test.go index 99a1579916..72254f8e22 100644 --- a/managed/services/encryption/encryption_rotation_test.go +++ b/managed/services/encryption/encryption_rotation_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/AlekSi/pointer" "github.com/pkg/errors" "github.com/stretchr/testify/require" @@ -40,7 +39,7 @@ const ( ) func TestEncryptionRotation(t *testing.T) { - db := testdb.Open(t, models.SkipFixtures, pointer.ToInt(88)) + db := testdb.Open(t, models.SkipFixtures, nil) defer db.Close() //nolint:errcheck err := createOriginEncryptionKey() @@ -76,7 +75,7 @@ func createOriginEncryptionKey() error { //nolint:dupword func insertTestData(db *sql.DB) error { _, err := models.UpdateSettings(db, &models.ChangeSettingsParams{ - EncryptedItems: []string{"pmm-managed-dev.agents.username", "pmm-managed-dev.agents.password", "pmm-managed-dev.agents.aws_access_key", "pmm-managed-dev.agents.aws_secret_key", "pmm-managed-dev.agents.mongo_db_tls_options", "pmm-managed-dev.agents.azure_options", "pmm-managed-dev.agents.mysql_options", "pmm-managed-dev.agents.postgresql_options", "pmm-managed-dev.agents.agent_password"}, + EncryptedItems: []string{"pmm-managed-dev.agents.username", "pmm-managed-dev.agents.password", "pmm-managed-dev.agents.agent_password", "pmm-managed-dev.agents.aws_options", "pmm-managed-dev.agents.azure_options", "pmm-managed-dev.agents.mongo_options", "pmm-managed-dev.agents.mysql_options", "pmm-managed-dev.agents.postgresql_options"}, }) if err != nil { return err @@ -98,8 +97,8 @@ func insertTestData(db *sql.DB) error { return err } _, err = db.Exec( - "INSERT INTO agents (agent_id, agent_type, username, password, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ - "VALUES ('1', 'pmm-agent', $1, $2, '1', NULL, false, '', $3, $4, false, false, 0, false, true, 0, 0, true, true, false, false)", + `INSERT INTO agents (agent_id, agent_type, username, password, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, qan_options, mysql_options, aws_options, exporter_options) `+ + `VALUES ('1', 'pmm-agent', $1, $2, '1', NULL, false, '', $3, $4, false, false, '{"max_query_length": 0, "query_examples_disabled": false, "comments_parsing_disabled": true, "max_query_log_size": 0}', '{"table_count_tablestats_group_limit": 0}', '{"rds_basic_metrics_disabled": true, "rds_enhanced_metrics_disabled": true}', '{"push_metrics": false, "expose_exporter": false}')`, originUsernameHash, originPasswordHash, now, now) if err != nil { return err diff --git a/managed/services/inventory/agents.go b/managed/services/inventory/agents.go index 1b96b51081..9f6a39baaf 100644 --- a/managed/services/inventory/agents.go +++ b/managed/services/inventory/agents.go @@ -262,21 +262,25 @@ func (as *AgentsService) ChangeNodeExporter(ctx context.Context, agentID string, func (as *AgentsService) AddMySQLdExporter(ctx context.Context, p *inventoryv1.AddMySQLdExporterParams) (*inventoryv1.AddAgentResponse, error) { var row *models.Agent var agent *inventoryv1.MySQLdExporter + + mysqlOptions := models.MySQLOptionsFromRequest(p) + mysqlOptions.TableCountTablestatsGroupLimit = p.TablestatsGroupTableLimit e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(p), - TableCountTablestatsGroupLimit: p.TablestatsGroupTableLimit, - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, + MySQLOptions: mysqlOptions, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), } var err error row, err = models.CreateAgent(tx.Querier, models.MySQLdExporterType, params) @@ -350,18 +354,20 @@ func (as *AgentsService) AddMongoDBExporter(ctx context.Context, p *inventoryv1. var agent *inventoryv1.MongoDBExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MongoDBOptions: models.MongoDBOptionsFromRequest(p), - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + MongoDBOptions: models.MongoDBOptionsFromRequest(p), + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.MongoDBExporterType, params) if err != nil { @@ -434,18 +440,20 @@ func (as *AgentsService) AddQANMySQLPerfSchemaAgent(ctx context.Context, p *inve var agent *inventoryv1.QANMySQLPerfSchemaAgent e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(p), - MaxQueryLength: p.MaxQueryLength, - QueryExamplesDisabled: p.DisableQueryExamples, - CommentsParsingDisabled: p.DisableCommentsParsing, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + QueryExamplesDisabled: p.DisableQueryExamples, + CommentsParsingDisabled: p.DisableCommentsParsing, + }, + MySQLOptions: models.MySQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANMySQLPerfSchemaAgentType, params) if err != nil { @@ -518,19 +526,21 @@ func (as *AgentsService) AddQANMySQLSlowlogAgent(ctx context.Context, p *invento } params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(p), - MaxQueryLength: p.MaxQueryLength, - QueryExamplesDisabled: p.DisableQueryExamples, - CommentsParsingDisabled: p.DisableCommentsParsing, - MaxQueryLogSize: maxSlowlogFileSize, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + QueryExamplesDisabled: p.DisableQueryExamples, + CommentsParsingDisabled: p.DisableCommentsParsing, + MaxQueryLogSize: maxSlowlogFileSize, + }, + MySQLOptions: models.MySQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANMySQLSlowlogAgentType, params) if err != nil { @@ -597,16 +607,18 @@ func (as *AgentsService) AddPostgresExporter(ctx context.Context, p *inventoryv1 var agent *inventoryv1.PostgresExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), } @@ -681,17 +693,19 @@ func (as *AgentsService) AddQANMongoDBProfilerAgent(ctx context.Context, p *inve e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-4650 - done, but not included in params. + }, MongoDBOptions: models.MongoDBOptionsFromRequest(p), - MaxQueryLength: p.MaxQueryLength, LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), - // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-4650 } row, err := models.CreateAgent(tx.Querier, models.QANMongoDBProfilerAgentType, params) if err != nil { @@ -760,17 +774,19 @@ func (as *AgentsService) AddProxySQLExporter(ctx context.Context, p *inventoryv1 var agent *inventoryv1.ProxySQLExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - AgentPassword: p.AgentPassword, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PushMetrics: p.PushMetrics, - DisableCollectors: p.DisableCollectors, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + AgentPassword: p.AgentPassword, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + DisabledCollectors: p.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.ProxySQLExporterType, params) if err != nil { @@ -842,17 +858,19 @@ func (as *AgentsService) AddQANPostgreSQLPgStatementsAgent(ctx context.Context, var agent *inventoryv1.QANPostgreSQLPgStatementsAgent e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - CustomLabels: p.CustomLabels, - MaxQueryLength: p.MaxQueryLength, - CommentsParsingDisabled: p.DisableCommentsParsing, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + CommentsParsingDisabled: p.DisableCommentsParsing, + }, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatementsAgentType, params) if err != nil { @@ -919,18 +937,20 @@ func (as *AgentsService) AddQANPostgreSQLPgStatMonitorAgent(ctx context.Context, var agent *inventoryv1.QANPostgreSQLPgStatMonitorAgent e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - ServiceID: p.ServiceId, - Username: p.Username, - Password: p.Password, - MaxQueryLength: p.MaxQueryLength, - QueryExamplesDisabled: p.DisableQueryExamples, - CommentsParsingDisabled: p.DisableCommentsParsing, - CustomLabels: p.CustomLabels, - TLS: p.Tls, - TLSSkipVerify: p.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + ServiceID: p.ServiceId, + Username: p.Username, + Password: p.Password, + CustomLabels: p.CustomLabels, + TLS: p.Tls, + TLSSkipVerify: p.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: p.MaxQueryLength, + QueryExamplesDisabled: p.DisableQueryExamples, + CommentsParsingDisabled: p.DisableCommentsParsing, + }, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(p), + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatMonitorAgentType, params) if err != nil { @@ -997,15 +1017,19 @@ func (as *AgentsService) AddRDSExporter(ctx context.Context, p *inventoryv1.AddR var agent *inventoryv1.RDSExporter e := as.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { params := &models.CreateAgentParams{ - PMMAgentID: p.PmmAgentId, - NodeID: p.NodeId, - AWSAccessKey: p.AwsAccessKey, - AWSSecretKey: p.AwsSecretKey, - CustomLabels: p.CustomLabels, - RDSBasicMetricsDisabled: p.DisableBasicMetrics, - RDSEnhancedMetricsDisabled: p.DisableEnhancedMetrics, - PushMetrics: p.PushMetrics, - LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: p.PmmAgentId, + NodeID: p.NodeId, + CustomLabels: p.CustomLabels, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + }, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: p.AwsAccessKey, + AWSSecretKey: p.AwsSecretKey, + RDSBasicMetricsDisabled: p.DisableBasicMetrics, + RDSEnhancedMetricsDisabled: p.DisableEnhancedMetrics, + }, + LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.RDSExporterType, params) if err != nil { @@ -1148,9 +1172,11 @@ func (as *AgentsService) AddAzureDatabaseExporter(ctx context.Context, p *invent params := &models.CreateAgentParams{ PMMAgentID: p.PmmAgentId, NodeID: p.NodeId, - AzureOptions: models.AzureOptionsFromRequest(p), CustomLabels: p.CustomLabels, - PushMetrics: p.PushMetrics, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: p.PushMetrics, + }, + AzureOptions: models.AzureOptionsFromRequest(p), LogLevel: services.SpecifyLogLevel(p.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), } row, err := models.CreateAgent(tx.Querier, models.AzureDatabaseExporterType, params) diff --git a/managed/services/management/agent.go b/managed/services/management/agent.go index 6d4bf61963..85077bcc67 100644 --- a/managed/services/management/agent.go +++ b/managed/services/management/agent.go @@ -123,42 +123,50 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive } ua := &managementv1.UniversalAgent{ - AgentId: agent.AgentID, - AgentType: string(agent.AgentType), - AwsAccessKey: pointer.GetString(agent.AWSAccessKey), - CreatedAt: timestamppb.New(agent.CreatedAt), - CustomLabels: labels, - Disabled: agent.Disabled, - DisabledCollectors: agent.DisabledCollectors, - IsConnected: s.r.IsConnected(agent.AgentID), - IsAgentPasswordSet: pointer.GetString(agent.AgentPassword) != "", - IsAwsSecretKeySet: pointer.GetString(agent.AWSSecretKey) != "", - IsPasswordSet: pointer.GetString(agent.Password) != "", - ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), - LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), - MaxQueryLength: agent.MaxQueryLength, - MaxQueryLogSize: agent.MaxQueryLogSize, - MetricsPath: pointer.GetString(agent.MetricsPath), - MetricsScheme: pointer.GetString(agent.MetricsScheme), - NodeId: pointer.GetString(agent.NodeID), - PmmAgentId: pointer.GetString(agent.PMMAgentID), - ProcessExecPath: pointer.GetString(agent.ProcessExecPath), - PushMetrics: agent.PushMetrics, - ExposeExporter: agent.ExposeExporter, - QueryExamplesDisabled: agent.QueryExamplesDisabled, - CommentsParsingDisabled: agent.CommentsParsingDisabled, - RdsBasicMetricsDisabled: agent.RDSBasicMetricsDisabled, - RdsEnhancedMetricsDisabled: agent.RDSEnhancedMetricsDisabled, - RunsOnNodeId: pointer.GetString(agent.RunsOnNodeID), - ServiceId: pointer.GetString(agent.ServiceID), - Status: agent.Status, - TableCount: pointer.GetInt32(agent.TableCount), - TableCountTablestatsGroupLimit: agent.TableCountTablestatsGroupLimit, - Tls: agent.TLS, - TlsSkipVerify: agent.TLSSkipVerify, - Username: pointer.GetString(agent.Username), - UpdatedAt: timestamppb.New(agent.UpdatedAt), - Version: pointer.GetString(agent.Version), + AgentId: agent.AgentID, + AgentType: string(agent.AgentType), + CreatedAt: timestamppb.New(agent.CreatedAt), + CustomLabels: labels, + Disabled: agent.Disabled, + IsConnected: s.r.IsConnected(agent.AgentID), + IsAgentPasswordSet: pointer.GetString(agent.AgentPassword) != "", + IsPasswordSet: pointer.GetString(agent.Password) != "", + ListenPort: uint32(pointer.GetUint16(agent.ListenPort)), + LogLevel: inventoryv1.LogLevelAPIValue(agent.LogLevel), + NodeId: pointer.GetString(agent.NodeID), + PmmAgentId: pointer.GetString(agent.PMMAgentID), + ProcessExecPath: pointer.GetString(agent.ProcessExecPath), + RunsOnNodeId: pointer.GetString(agent.RunsOnNodeID), + ServiceId: pointer.GetString(agent.ServiceID), + Status: agent.Status, + Tls: agent.TLS, + TlsSkipVerify: agent.TLSSkipVerify, + Username: pointer.GetString(agent.Username), + UpdatedAt: timestamppb.New(agent.UpdatedAt), + Version: pointer.GetString(agent.Version), + } + + if agent.ExporterOptions != nil { + ua.DisabledCollectors = agent.ExporterOptions.DisabledCollectors + ua.MetricsPath = agent.ExporterOptions.MetricsPath + ua.MetricsScheme = agent.ExporterOptions.MetricsScheme + ua.PushMetrics = agent.ExporterOptions.PushMetrics + ua.ExposeExporter = agent.ExporterOptions.ExposeExporter + } + + if agent.QANOptions != nil { + ua.MaxQueryLength = agent.QANOptions.MaxQueryLength + ua.MaxQueryLogSize = agent.QANOptions.MaxQueryLogSize + ua.QueryExamplesDisabled = agent.QANOptions.QueryExamplesDisabled + ua.CommentsParsingDisabled = agent.QANOptions.CommentsParsingDisabled + } + + if agent.AWSOptions != nil { + ua.IsAwsSecretKeySet = agent.AWSOptions.AWSAccessKey != "" + ua.AwsAccessKey = agent.AWSOptions.AWSAccessKey + ua.RdsBasicMetricsDisabled = agent.AWSOptions.RDSBasicMetricsDisabled + ua.RdsEnhancedMetricsDisabled = agent.AWSOptions.RDSEnhancedMetricsDisabled + } if agent.AzureOptions != nil { @@ -171,32 +179,35 @@ func (s *ManagementService) agentToAPI(agent *models.Agent) (*managementv1.Unive } } + if agent.MongoDBOptions != nil { + ua.MongoDbOptions = &managementv1.UniversalAgent_MongoDBOptions{ + AuthenticationMechanism: agent.MongoDBOptions.AuthenticationMechanism, + AuthenticationDatabase: agent.MongoDBOptions.AuthenticationDatabase, + CollectionsLimit: agent.MongoDBOptions.CollectionsLimit, + EnableAllCollectors: agent.MongoDBOptions.EnableAllCollectors, + StatsCollections: agent.MongoDBOptions.StatsCollections, + IsTlsCertificateKeySet: agent.MongoDBOptions.TLSCertificateKey != "", + IsTlsCertificateKeyFilePasswordSet: agent.MongoDBOptions.TLSCertificateKeyFilePassword != "", + } + } + if agent.MySQLOptions != nil { ua.MysqlOptions = &managementv1.UniversalAgent_MySQLOptions{ IsTlsKeySet: agent.MySQLOptions.TLSKey != "", } + ua.TableCount = pointer.GetInt32(agent.MySQLOptions.TableCount) + ua.TableCountTablestatsGroupLimit = agent.MySQLOptions.TableCountTablestatsGroupLimit + } if agent.PostgreSQLOptions != nil { ua.PostgresqlOptions = &managementv1.UniversalAgent_PostgreSQLOptions{ IsSslKeySet: agent.PostgreSQLOptions.SSLKey != "", - AutoDiscoveryLimit: agent.PostgreSQLOptions.AutoDiscoveryLimit, + AutoDiscoveryLimit: pointer.GetInt32(agent.PostgreSQLOptions.AutoDiscoveryLimit), MaxExporterConnections: agent.PostgreSQLOptions.MaxExporterConnections, } } - if agent.MongoDBOptions != nil { - ua.MongoDbOptions = &managementv1.UniversalAgent_MongoDBOptions{ - AuthenticationMechanism: agent.MongoDBOptions.AuthenticationMechanism, - AuthenticationDatabase: agent.MongoDBOptions.AuthenticationDatabase, - CollectionsLimit: agent.MongoDBOptions.CollectionsLimit, - EnableAllCollectors: agent.MongoDBOptions.EnableAllCollectors, - StatsCollections: agent.MongoDBOptions.StatsCollections, - IsTlsCertificateKeySet: agent.MongoDBOptions.TLSCertificateKey != "", - IsTlsCertificateKeyFilePasswordSet: agent.MongoDBOptions.TLSCertificateKeyFilePassword != "", - } - } - return ua, nil } diff --git a/managed/services/management/agent_test.go b/managed/services/management/agent_test.go index e6d0461f73..40705bc901 100644 --- a/managed/services/management/agent_test.go +++ b/managed/services/management/agent_test.go @@ -148,36 +148,78 @@ func TestAgentService(t *testing.T) { expected := []*agentv1.UniversalAgent{ { - AgentId: pgExporterID, - AgentType: "postgres_exporter", - PmmAgentId: models.PMMServerAgentID, - IsConnected: false, - CreatedAt: timestamppb.New(now), - UpdatedAt: timestamppb.New(now), - Username: "postgres", - PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ - IsSslKeySet: false, - }, + AgentId: pgExporterID, + AgentType: "postgres_exporter", + PmmAgentId: models.PMMServerAgentID, + IsConnected: false, + CreatedAt: timestamppb.New(now), + UpdatedAt: timestamppb.New(now), + Username: "postgres", ServiceId: "00000000-0000-4000-8000-000000000002", Status: "AGENT_STATUS_UNKNOWN", Tls: true, CommentsParsingDisabled: true, - }, - { - AgentId: pgStatStatementID, - AgentType: "qan-postgresql-pgstatements-agent", - PmmAgentId: models.PMMServerAgentID, - IsConnected: false, - CreatedAt: timestamppb.New(now), - UpdatedAt: timestamppb.New(now), - Username: "postgres", + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ - IsSslKeySet: false, + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, }, + }, + { + AgentId: pgStatStatementID, + AgentType: "qan-postgresql-pgstatements-agent", + PmmAgentId: models.PMMServerAgentID, + IsConnected: false, + CreatedAt: timestamppb.New(now), + UpdatedAt: timestamppb.New(now), + Username: "postgres", ServiceId: "00000000-0000-4000-8000-000000000002", Status: "AGENT_STATUS_UNKNOWN", Tls: true, CommentsParsingDisabled: true, + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, { AgentId: models.PMMServerAgentID, @@ -186,6 +228,30 @@ func TestAgentService(t *testing.T) { IsConnected: true, CreatedAt: timestamppb.New(now), UpdatedAt: timestamppb.New(now), + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, } @@ -237,6 +303,30 @@ func TestAgentService(t *testing.T) { UpdatedAt: timestamppb.New(now), ServiceId: "00000000-0000-4000-8000-000000000006", Status: "AGENT_STATUS_UNKNOWN", + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, } assert.Equal(t, expected, response.Agents) @@ -287,6 +377,30 @@ func TestAgentService(t *testing.T) { UpdatedAt: timestamppb.New(now), ServiceId: "00000000-0000-4000-8000-000000000006", Status: "AGENT_STATUS_UNKNOWN", + AzureOptions: &agentv1.UniversalAgent_AzureOptions{ + ClientId: "", + IsClientSecretSet: false, + ResourceGroup: "", + SubscriptionId: "", + TenantId: "", + }, + MongoDbOptions: &agentv1.UniversalAgent_MongoDBOptions{ + IsTlsCertificateKeySet: false, + IsTlsCertificateKeyFilePasswordSet: false, + AuthenticationMechanism: "", + AuthenticationDatabase: "", + StatsCollections: nil, + CollectionsLimit: 0, + EnableAllCollectors: false, + }, + MysqlOptions: &agentv1.UniversalAgent_MySQLOptions{ + IsTlsKeySet: false, + }, + PostgresqlOptions: &agentv1.UniversalAgent_PostgreSQLOptions{ + IsSslKeySet: false, + AutoDiscoveryLimit: 0, + MaxExporterConnections: 0, + }, }, } assert.Equal(t, expected, response.Agents) diff --git a/managed/services/management/azure_database.go b/managed/services/management/azure_database.go index 0fea413810..d98af5d1aa 100644 --- a/managed/services/management/azure_database.go +++ b/managed/services/management/azure_database.go @@ -271,13 +271,15 @@ func (s *ManagementService) AddAzureDatabase(ctx context.Context, req *managemen } metricsExporter, err := models.CreateAgent(tx.Querier, exporterType, &models.CreateAgentParams{ - PMMAgentID: models.PMMServerAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + PMMAgentID: models.PMMServerAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MySQLOptions: &models.MySQLOptions{ + TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + }, }) if err != nil { return err @@ -295,13 +297,15 @@ func (s *ManagementService) AddAzureDatabase(ctx context.Context, req *managemen if req.Qan { qanAgent, err := models.CreateAgent(tx.Querier, qanAgentType, &models.CreateAgentParams{ - PMMAgentID: models.PMMServerAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - QueryExamplesDisabled: req.DisableQueryExamples, + PMMAgentID: models.PMMServerAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + QueryExamplesDisabled: req.DisableQueryExamples, + }, }) if err != nil { return err diff --git a/managed/services/management/mongodb.go b/managed/services/management/mongodb.go index fe7782c194..34b7e9d7ca 100644 --- a/managed/services/management/mongodb.go +++ b/managed/services/management/mongodb.go @@ -63,18 +63,20 @@ func (s *ManagementService) addMongoDB(ctx context.Context, req *managementv1.Ad } row, err := models.CreateAgent(tx.Querier, models.MongoDBExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MongoDBOptions: models.MongoDBOptionsFromRequest(req), - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MongoDBOptions: models.MongoDBOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, }) if err != nil { return err @@ -98,16 +100,18 @@ func (s *ManagementService) addMongoDB(ctx context.Context, req *managementv1.Ad if req.QanMongodbProfiler { row, err = models.CreateAgent(tx.Querier, models.QANMongoDBProfilerAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-7860 + }, MongoDBOptions: models.MongoDBOptionsFromRequest(req), - MaxQueryLength: req.MaxQueryLength, LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), - // TODO QueryExamplesDisabled https://jira.percona.com/browse/PMM-7860 }) if err != nil { return err diff --git a/managed/services/management/mysql.go b/managed/services/management/mysql.go index 8ef37eb4d0..112de34e5a 100644 --- a/managed/services/management/mysql.go +++ b/managed/services/management/mysql.go @@ -86,20 +86,24 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM return err } + mysqlOptions := models.MySQLOptionsFromRequest(req) + mysqlOptions.TableCountTablestatsGroupLimit = tablestatsGroupTableLimit + row, err := models.CreateAgent(tx.Querier, models.MySQLdExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(req), - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MySQLOptions: mysqlOptions, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), }) if err != nil { return err @@ -113,7 +117,7 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM return err } // GetInfoFromService updates the table count in row so, let's also update the response - mysql.TableCount = *row.TableCount + mysql.TableCount = *row.MySQLOptions.TableCount } agent, err := services.ToAPIAgent(tx.Querier, row) @@ -124,17 +128,19 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM if req.QanMysqlPerfschema { row, err = models.CreateAgent(tx.Querier, models.QANMySQLPerfSchemaAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(req), - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, + MySQLOptions: models.MySQLOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err @@ -149,18 +155,20 @@ func (s *ManagementService) addMySQL(ctx context.Context, req *managementv1.AddM if req.QanMysqlSlowlog { row, err = models.CreateAgent(tx.Querier, models.QANMySQLSlowlogAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - MySQLOptions: models.MySQLOptionsFromRequest(req), - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - MaxQueryLogSize: maxSlowlogFileSize, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + MySQLOptions: models.MySQLOptionsFromRequest(req), + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + MaxQueryLogSize: maxSlowlogFileSize, + }, + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err diff --git a/managed/services/management/postgresql.go b/managed/services/management/postgresql.go index e1f1a473da..63bb85171d 100644 --- a/managed/services/management/postgresql.go +++ b/managed/services/management/postgresql.go @@ -65,16 +65,18 @@ func (s *ManagementService) addPostgreSQL(ctx context.Context, req *managementv1 } row, err := models.CreateAgent(tx.Querier, models.PostgresExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_ERROR), }) @@ -107,17 +109,19 @@ func (s *ManagementService) addPostgreSQL(ctx context.Context, req *managementv1 if req.QanPostgresqlPgstatementsAgent { row, err = models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatementsAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err @@ -132,17 +136,19 @@ func (s *ManagementService) addPostgreSQL(ctx context.Context, req *managementv1 if req.QanPostgresqlPgstatmonitorAgent { row, err = models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatMonitorAgentType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - MaxQueryLength: req.MaxQueryLength, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + MaxQueryLength: req.MaxQueryLength, + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, + PostgreSQLOptions: models.PostgreSQLOptionsFromRequest(req), + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err diff --git a/managed/services/management/proxysql.go b/managed/services/management/proxysql.go index cf8ac0d5da..d1f5eb6bce 100644 --- a/managed/services/management/proxysql.go +++ b/managed/services/management/proxysql.go @@ -63,17 +63,19 @@ func (s *ManagementService) addProxySQL(ctx context.Context, req *managementv1.A } row, err := models.CreateAgent(tx.Querier, models.ProxySQLExporterType, &models.CreateAgentParams{ - PMMAgentID: req.PmmAgentId, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - AgentPassword: req.AgentPassword, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - PushMetrics: isPushMode(req.MetricsMode), - ExposeExporter: req.ExposeExporter, - DisableCollectors: req.DisableCollectors, - LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), + PMMAgentID: req.PmmAgentId, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + AgentPassword: req.AgentPassword, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + ExposeExporter: req.ExposeExporter, + PushMetrics: isPushMode(req.MetricsMode), + DisabledCollectors: req.DisableCollectors, + }, + LogLevel: services.SpecifyLogLevel(req.LogLevel, inventoryv1.LogLevel_LOG_LEVEL_FATAL), }) if err != nil { return err diff --git a/managed/services/management/rds.go b/managed/services/management/rds.go index a7ce3035e8..6e3edfea89 100644 --- a/managed/services/management/rds.go +++ b/managed/services/management/rds.go @@ -274,13 +274,17 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add RDSExporter Agent if req.RdsExporter { rdsExporter, err := models.CreateAgent(tx.Querier, models.RDSExporterType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - NodeID: node.NodeID, - AWSAccessKey: req.AwsAccessKey, - AWSSecretKey: req.AwsSecretKey, - RDSBasicMetricsDisabled: req.DisableBasicMetrics, - RDSEnhancedMetricsDisabled: req.DisableEnhancedMetrics, - PushMetrics: isPushMode(metricsMode), + PMMAgentID: pmmAgentID, + NodeID: node.NodeID, + AWSOptions: &models.AWSOptions{ + AWSAccessKey: req.AwsAccessKey, + AWSSecretKey: req.AwsSecretKey, + RDSBasicMetricsDisabled: req.DisableBasicMetrics, + RDSEnhancedMetricsDisabled: req.DisableEnhancedMetrics, + }, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: isPushMode(metricsMode), + }, }) if err != nil { return err @@ -316,14 +320,18 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add MySQL Exporter mysqldExporter, err := models.CreateAgent(tx.Querier, models.MySQLdExporterType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, - PushMetrics: isPushMode(metricsMode), + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: isPushMode(metricsMode), + }, + MySQLOptions: &models.MySQLOptions{ + TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + }, }) if err != nil { return err @@ -346,14 +354,16 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add MySQL PerfSchema QAN Agent if req.QanMysqlPerfschema { qanAgent, err := models.CreateAgent(tx.Querier, models.QANMySQLPerfSchemaAgentType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, }) if err != nil { return err @@ -391,18 +401,23 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add PostgreSQL Exporter postgresExporter, err := models.CreateAgent(tx.Querier, models.PostgresExporterType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: isPushMode(metricsMode), + }, + MySQLOptions: &models.MySQLOptions{ + TableCountTablestatsGroupLimit: tablestatsGroupTableLimit, + }, + PostgreSQLOptions: &models.PostgreSQLOptions{ - AutoDiscoveryLimit: req.AutoDiscoveryLimit, + AutoDiscoveryLimit: pointer.ToInt32(req.AutoDiscoveryLimit), MaxExporterConnections: req.MaxPostgresqlExporterConnections, }, - PushMetrics: isPushMode(metricsMode), }) if err != nil { return err @@ -425,14 +440,16 @@ func (s *ManagementService) addRDS(ctx context.Context, req *managementv1.AddRDS // add PostgreSQL Pgstatements QAN Agent if req.QanPostgresqlPgstatements { qanAgent, err := models.CreateAgent(tx.Querier, models.QANPostgreSQLPgStatementsAgentType, &models.CreateAgentParams{ - PMMAgentID: pmmAgentID, - ServiceID: service.ServiceID, - Username: req.Username, - Password: req.Password, - TLS: req.Tls, - TLSSkipVerify: req.TlsSkipVerify, - QueryExamplesDisabled: req.DisableQueryExamples, - CommentsParsingDisabled: req.DisableCommentsParsing, + PMMAgentID: pmmAgentID, + ServiceID: service.ServiceID, + Username: req.Username, + Password: req.Password, + TLS: req.Tls, + TLSSkipVerify: req.TlsSkipVerify, + QANOptions: &models.QANOptions{ + QueryExamplesDisabled: req.DisableQueryExamples, + CommentsParsingDisabled: req.DisableCommentsParsing, + }, }) if err != nil { return err diff --git a/managed/services/victoriametrics/prometheus.go b/managed/services/victoriametrics/prometheus.go index 35e1f6b366..a4f957cc6c 100644 --- a/managed/services/victoriametrics/prometheus.go +++ b/managed/services/victoriametrics/prometheus.go @@ -108,15 +108,15 @@ func AddScrapeConfigs(l *logrus.Entry, cfg *config.Config, q *reform.Querier, // } mr := *globalResolutions // copy global resolutions - if agent.MetricsResolutions != nil { - if agent.MetricsResolutions.MR != 0 { - mr.MR = agent.MetricsResolutions.MR + if agent.ExporterOptions.MetricsResolutions != nil { + if agent.ExporterOptions.MetricsResolutions.MR != 0 { + mr.MR = agent.ExporterOptions.MetricsResolutions.MR } - if agent.MetricsResolutions.HR != 0 { - mr.HR = agent.MetricsResolutions.HR + if agent.ExporterOptions.MetricsResolutions.HR != 0 { + mr.HR = agent.ExporterOptions.MetricsResolutions.HR } - if agent.MetricsResolutions.LR != 0 { - mr.LR = agent.MetricsResolutions.LR + if agent.ExporterOptions.MetricsResolutions.LR != 0 { + mr.LR = agent.ExporterOptions.MetricsResolutions.LR } } diff --git a/managed/services/victoriametrics/scrape_configs.go b/managed/services/victoriametrics/scrape_configs.go index fcc000e22b..16504177ef 100644 --- a/managed/services/victoriametrics/scrape_configs.go +++ b/managed/services/victoriametrics/scrape_configs.go @@ -214,7 +214,7 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC "hwmon", "textfile.mr", } - mrCollect = collectors.FilterOutCollectors("", mrCollect, params.agent.DisabledCollectors) + mrCollect = collectors.FilterOutCollectors("", mrCollect, params.agent.ExporterOptions.DisabledCollectors) mr, err = scrapeConfigForStandardExporter("mr", params.metricsResolution.MR, params, mrCollect) if err != nil { return nil, err @@ -227,7 +227,7 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC "uname", "os", } - lrCollect = collectors.FilterOutCollectors("", lrCollect, params.agent.DisabledCollectors) + lrCollect = collectors.FilterOutCollectors("", lrCollect, params.agent.ExporterOptions.DisabledCollectors) lr, err = scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, lrCollect) if err != nil { return nil, err @@ -255,7 +255,7 @@ func scrapeConfigsForNodeExporter(params *scrapeConfigParams) ([]*config.ScrapeC "meminfo", "netdev", "time") - hrCollect = collectors.FilterOutCollectors("", hrCollect, params.agent.DisabledCollectors) + hrCollect = collectors.FilterOutCollectors("", hrCollect, params.agent.ExporterOptions.DisabledCollectors) hr, err = scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrCollect) if err != nil { @@ -285,7 +285,8 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap "standard.go", "standard.process", } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.DisabledCollectors) + + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { @@ -307,7 +308,7 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap mrOptions = append(mrOptions, "perf_schema.tablelocks") } - mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.DisabledCollectors) + mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.ExporterOptions.DisabledCollectors) mr, err := scrapeConfigForStandardExporter("mr", params.metricsResolution.MR, params, mrOptions) if err != nil { return nil, err @@ -335,7 +336,7 @@ func scrapeConfigsForMySQLdExporter(params *scrapeConfigParams) ([]*config.Scrap "perf_schema.tableiowaits") } - lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.DisabledCollectors) + lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.ExporterOptions.DisabledCollectors) lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, lrOptions) if err != nil { @@ -375,7 +376,7 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra "replicasetstatus", "topmetrics", } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.DisabledCollectors) + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { return nil, err @@ -385,7 +386,7 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra if hr != nil { r = append(r, hr) } - if params.agent.MongoDBOptions != nil && params.agent.MongoDBOptions.EnableAllCollectors { + if params.agent.MongoDBOptions.EnableAllCollectors { defaultCollectors := []string{ "dbstats", "indexstats", @@ -404,7 +405,7 @@ func scrapeConfigsForMongoDBExporter(params *scrapeConfigParams) ([]*config.Scra defaultCollectors = append(defaultCollectors, "pbm") } - defaultCollectors = collectors.FilterOutCollectors("", defaultCollectors, params.agent.DisabledCollectors) + defaultCollectors = collectors.FilterOutCollectors("", defaultCollectors, params.agent.ExporterOptions.DisabledCollectors) lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, defaultCollectors) if err != nil { return nil, err @@ -425,7 +426,8 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr "standard.process", "postgres", } - hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.DisabledCollectors) + + hrOptions = collectors.FilterOutCollectors("", hrOptions, params.agent.ExporterOptions.DisabledCollectors) hr, err := scrapeConfigForStandardExporter("hr", params.metricsResolution.HR, params, hrOptions) if err != nil { return nil, err @@ -434,7 +436,7 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr mrOptions := []string{ "custom_query.mr", } - mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.DisabledCollectors) + mrOptions = collectors.FilterOutCollectors("", mrOptions, params.agent.ExporterOptions.DisabledCollectors) mr, err := scrapeConfigForStandardExporter("mr", params.metricsResolution.MR, params, mrOptions) if err != nil { return nil, err @@ -443,7 +445,7 @@ func scrapeConfigsForPostgresExporter(params *scrapeConfigParams) ([]*config.Scr lrOptions := []string{ "custom_query.lr", } - lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.DisabledCollectors) + lrOptions = collectors.FilterOutCollectors("", lrOptions, params.agent.ExporterOptions.DisabledCollectors) lr, err := scrapeConfigForStandardExporter("lr", params.metricsResolution.LR, params, lrOptions) if err != nil { return nil, err @@ -532,14 +534,14 @@ func scrapeConfigsForExternalExporter(s *models.MetricsResolutions, params *scra if err != nil { return nil, err } - interval := s.MR + cfg := &config.ScrapeConfig{ JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: pointer.GetString(params.agent.MetricsScheme), - MetricsPath: pointer.GetString(params.agent.MetricsPath), + Scheme: params.agent.ExporterOptions.MetricsScheme, + MetricsPath: params.agent.ExporterOptions.MetricsPath, } if pointer.GetString(params.agent.Username) != "" { @@ -569,14 +571,14 @@ func scrapeConfigsForVMAgent(s *models.MetricsResolutions, params *scrapeConfigP if err != nil { return nil, err } - interval := s.MR + cfg := &config.ScrapeConfig{ JobName: jobName(params.agent, "mr"), ScrapeInterval: config.Duration(interval), ScrapeTimeout: scrapeTimeout(interval), - Scheme: pointer.GetString(params.agent.MetricsScheme), - MetricsPath: pointer.GetString(params.agent.MetricsPath), + Scheme: params.agent.ExporterOptions.MetricsScheme, + MetricsPath: params.agent.ExporterOptions.MetricsPath, } if pointer.GetString(params.agent.Username) != "" { diff --git a/managed/services/victoriametrics/scrape_configs_test.go b/managed/services/victoriametrics/scrape_configs_test.go index 73b98560e3..28ad444d38 100644 --- a/managed/services/victoriametrics/scrape_configs_test.go +++ b/managed/services/victoriametrics/scrape_configs_test.go @@ -47,11 +47,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_node_label": "foo"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.NodeExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"cpu", "entropy"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.NodeExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"cpu", "entropy"}, + }, } expected := []*config.ScrapeConfig{{ @@ -182,11 +184,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_node_label": "foo"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.NodeExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"cpu", "time"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.NodeExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"cpu", "time"}, + }, } expected := []*config.ScrapeConfig{{ @@ -253,10 +257,12 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, + MySQLOptions: &models.MySQLOptions{}, } expected := []*config.ScrapeConfig{{ @@ -407,11 +413,14 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"global_status", "info_schema.innodb_cmp", "info_schema.query_response_time", "perf_schema.eventsstatements", "heartbeat"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"global_status", "info_schema.innodb_cmp", "info_schema.query_response_time", "perf_schema.eventsstatements", "heartbeat"}, + }, + MySQLOptions: &models.MySQLOptions{}, } expected := []*config.ScrapeConfig{{ @@ -555,11 +564,14 @@ func TestScrapeConfig(t *testing.T) { Address: pointer.ToString("5.6.7.8"), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MySQLdExporterType, - ListenPort: pointer.ToUint16(12345), - TableCount: pointer.ToInt32(100500), - TableCountTablestatsGroupLimit: 1000, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MySQLdExporterType, + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, + MySQLOptions: &models.MySQLOptions{ + TableCount: pointer.ToInt32(100500), + TableCountTablestatsGroupLimit: 1000, + }, } expected := []*config.ScrapeConfig{{ @@ -684,8 +696,9 @@ func TestScrapeConfig(t *testing.T) { node := &models.Node{} service := &models.Service{} agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForMySQLdExporter(&scrapeConfigParams{ @@ -714,11 +727,12 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.MongoDBExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.MongoDBExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, + MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, } expected := []*config.ScrapeConfig{ @@ -804,8 +818,9 @@ func TestScrapeConfig(t *testing.T) { node := &models.Node{} service := &models.Service{} agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForMongoDBExporter(&scrapeConfigParams{ @@ -835,11 +850,13 @@ func TestScrapeConfig(t *testing.T) { CustomLabels: []byte(`{"_some_service_label": "bar"}`), } agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.PostgresExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), - DisabledCollectors: []string{"standard.process", "custom_query.lr"}, + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.PostgresExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + DisabledCollectors: []string{"standard.process", "custom_query.lr"}, + }, } expected := []*config.ScrapeConfig{{ @@ -953,8 +970,9 @@ func TestScrapeConfig(t *testing.T) { node := &models.Node{} service := &models.Service{} agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForPostgresExporter(&scrapeConfigParams{ @@ -1171,10 +1189,11 @@ func TestScrapeConfig(t *testing.T) { } t.Run("Normal", func(t *testing.T) { agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.ExternalExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - ListenPort: pointer.ToUint16(12345), + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.ExternalExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } expected := []*config.ScrapeConfig{{ @@ -1215,14 +1234,16 @@ func TestScrapeConfig(t *testing.T) { t.Run("WithExtraParams", func(t *testing.T) { agent := &models.Agent{ - AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", - AgentType: models.ExternalExporterType, - CustomLabels: []byte(`{"_some_agent_label": "baz"}`), - Username: pointer.ToString("username"), - Password: pointer.ToString("password"), - ListenPort: pointer.ToUint16(12345), - MetricsPath: pointer.ToString("/some-metric-path"), - MetricsScheme: pointer.ToString("https"), + AgentID: "75bb30d3-ef4a-4147-97a8-621a996611dd", + AgentType: models.ExternalExporterType, + CustomLabels: []byte(`{"_some_agent_label": "baz"}`), + Username: pointer.ToString("username"), + Password: pointer.ToString("password"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{ + MetricsPath: "/some-metric-path", + MetricsScheme: "https", + }, } expected := []*config.ScrapeConfig{{ @@ -1271,8 +1292,9 @@ func TestScrapeConfig(t *testing.T) { t.Run("BadCustomLabels", func(t *testing.T) { agent := &models.Agent{ - CustomLabels: []byte("{"), - ListenPort: pointer.ToUint16(12345), + CustomLabels: []byte("{"), + ListenPort: pointer.ToUint16(12345), + ExporterOptions: &models.ExporterOptions{}, } _, err := scrapeConfigsForMongoDBExporter(&scrapeConfigParams{ diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index d85e082aaf..f2b47c2bd9 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -230,7 +230,9 @@ func TestVictoriaMetrics(t *testing.T) { CustomLabels: []byte(`{"_agent_label": "mongodb-baz-push"}`), ListenPort: pointer.ToUint16(12346), MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, - PushMetrics: true, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: true, + }, }, // Agent with pull model @@ -242,7 +244,9 @@ func TestVictoriaMetrics(t *testing.T) { CustomLabels: []byte(`{"_agent_label": "mongodb-baz-pull"}`), ListenPort: pointer.ToUint16(12346), MongoDBOptions: &models.MongoDBOptions{EnableAllCollectors: true}, - PushMetrics: false, + ExporterOptions: &models.ExporterOptions{ + PushMetrics: false, + }, }, } { check.NoError(db.Insert(str), "%+v", str)