Skip to content

Commit

Permalink
PMM-5086-12634 Agent model refactor. (#3338)
Browse files Browse the repository at this point in the history 
* PMM-5086-12634 Structure draft.

* PMM-5086-12634 Some changes.

* PMM-5086-12634 Another changes.

* PMM-5086-12634 Another changes.

* PMM-5086-12634 Another changes.

* PMM-5086-12634 Revert changed makefile.

* PMM-5086-12634 Changes.

* PMM-5086-12634 Fix for encryption.

* PMM-5086-12634 Database changes.

* PMM-5086-12634 Fix DB JSONB conversion.

* PMM-5086-12634 Missed type.

* PMM-5086-12634 Typo in mongo options renaming.

* PMM-5086-12634 Changes.

* PMM-5086-12634 Nil check.

* PMM-5086-12634 Migrations, nil checks.

* PMM-5086-12634 Two models of encryption, db changes.

* PMM-5086-12634 Format.

* PMM-5086-12634 Fix double decrypt bug.

* PMM-508612634 Helpers push metrics fix.

* PMM-12634 Expect false in exporter options.

* Revert "PMM-5086-12634 Fix double decrypt bug."

This reverts commit e72961b.

* PMM-5086-12634 Handle nil in scrape configs.

* PMM-5086-12634 Fix encryption rotation test.

* PMM-5086-12634 Mongo options nil.

* PMM-5086-12634 Converters.

* PMM-5086-12634 MySQL fix.

* PMM-5086-12634 Fix proxysql nil.

* PMM-5086-12634 Fix roster test.

* PMM-5086-12634 Fix for TextFiles and templateParams.

* PMM-5086-12634 Fix PG test.

* PMM-5086-12634 Fix wrong annotation.

* PMM-5086-12634 Fix lint about receivers.

* PMM-5086-12634 Migration changes.

* PMM-5086-12634 Correct push_metrics fields naming.

* PMM-5086-12634 Lint.

* PMM-5086-12634 Fix migrations, helpers test.

* PMM-5086-12634 Fix listen address test.

* PMM-5086-12634 Fix agent management tests.

* PMM-5086-12634 Mongo nil handling.

* PMM-5086-12634 MySQL nil handling.

* PMM-5086 Fix Node nil handling.

* PMM-5086-12634 Fix Postgres nil handling.

* PMM-5086-12634 Fix RDS nil handling.

* PMM-5086-12634 Fix scraped configs.

* PMM-5086-12634 Lint.

* PMM-5086-12634 Fix another tests.

* PMM-5086-12634 Format.

* PMM-5086-12634 Fix for another tests.

* PMM-5086-12634 CreateAgent options preparation.

* PMM-5086-12634 Fix another tests.

* PMM-5086-12634 PG test.

* PMM-5086-12634 Fix PG test suite.

* PMM-5086-12634 Fix proxy exporter.

* PMM-5086-12634 Another MySQL fix.

* PMM-5086-12634 Cleanup.

* PMM-5086-12634 API tests/logic fix.

* PMM-5086-12634 Another cleanup.

* PMM-5086-12634 Remove migration test to avoid two encryption models.

* PMM-5086-12634 Remove options assignment in case of nil option.

* PMM-5086-12634 Remove the forgotten comments.
  • Loading branch information
@JiriCtvrtka
JiriCtvrtka authored Dec 16, 2024
1 parent a55a663 commit fba3928
Show file tree
Hide file tree
Showing 43 changed files with 1,799 additions and 1,365 deletions.
208 changes: 110 additions & 98 deletions managed/models/agent_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ import (
"github.com/percona/pmm/version"
)

const (
pushMetricsTrue = "((exporter_options ? 'push_metrics') AND (exporter_options->>'push_metrics')::boolean = true)"
pushMetricsFalse = "(NOT (exporter_options ? 'push_metrics') OR (exporter_options->>'push_metrics')::boolean = false)"
)

// MySQLOptionsParams contains methods to create MySQLOptions object.
type MySQLOptionsParams interface {
GetTlsCa() string
Expand All @@ -47,7 +52,7 @@ func MySQLOptionsFromRequest(params MySQLOptionsParams) *MySQLOptions {
TLSKey: params.GetTlsKey(),
}
}
return nil
return &MySQLOptions{}
}

// PostgreSQLOptionsParams contains methods to create PostgreSQLOptions object.
Expand All @@ -74,7 +79,7 @@ func PostgreSQLOptionsFromRequest(params PostgreSQLOptionsParams) *PostgreSQLOpt

// PostgreSQL exporter has these parameters but they are not needed for QAN agent.
if extendedOptions, ok := params.(PostgreSQLExtendedOptionsParams); ok && extendedOptions != nil {
res.AutoDiscoveryLimit = extendedOptions.GetAutoDiscoveryLimit()
res.AutoDiscoveryLimit = pointer.ToInt32(extendedOptions.GetAutoDiscoveryLimit())
res.MaxExporterConnections = extendedOptions.GetMaxExporterConnections()
}

Expand All @@ -99,7 +104,7 @@ type MongoDBExtendedOptionsParams interface {

// MongoDBOptionsFromRequest creates MongoDBOptionsParams object from request.
func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions {
var mdbOptions *MongoDBOptions
mdbOptions := &MongoDBOptions{}

if params.GetTlsCertificateKey() != "" || params.GetTlsCertificateKeyFilePassword() != "" || params.GetTlsCa() != "" {
mdbOptions = &MongoDBOptions{}
Expand All @@ -109,19 +114,13 @@ func MongoDBOptionsFromRequest(params MongoDBOptionsParams) *MongoDBOptions {
}

if params.GetAuthenticationMechanism() != "" || params.GetAuthenticationDatabase() != "" {
if mdbOptions == nil {
mdbOptions = &MongoDBOptions{}
}
mdbOptions.AuthenticationMechanism = params.GetAuthenticationMechanism()
mdbOptions.AuthenticationDatabase = params.GetAuthenticationDatabase()
}

// MongoDB exporter has these parameters but they are not needed for QAN agent.
if extendedOptions, ok := params.(MongoDBExtendedOptionsParams); ok {
if extendedOptions != nil {
if mdbOptions == nil {
mdbOptions = &MongoDBOptions{}
}
mdbOptions.StatsCollections = extendedOptions.GetStatsCollections()
mdbOptions.CollectionsLimit = extendedOptions.GetCollectionsLimit()
mdbOptions.EnableAllCollectors = extendedOptions.GetEnableAllCollectors()
Expand Down Expand Up @@ -221,7 +220,7 @@ func FindAgents(q *reform.Querier, filters AgentFilters) ([]*Agent, error) {
idx++
}
if filters.AWSAccessKey != "" {
conditions = append(conditions, fmt.Sprintf("aws_access_key = %s", q.Placeholder(idx)))
conditions = append(conditions, fmt.Sprintf("(aws_options ? 'aws_access_key' AND aws_options->>'aws_access_key' = %s)", q.Placeholder(idx)))
args = append(args, filters.AWSAccessKey)
}

Expand Down Expand Up @@ -474,9 +473,9 @@ func FindAgentsForScrapeConfig(q *reform.Querier, pmmAgentID *string, pushMetric
}

if pushMetrics {
conditions = append(conditions, "push_metrics")
conditions = append(conditions, pushMetricsTrue)
} else {
conditions = append(conditions, "NOT push_metrics")
conditions = append(conditions, pushMetricsFalse)
}

conditions = append(conditions, "NOT disabled", "listen_port IS NOT NULL")
Expand All @@ -496,7 +495,7 @@ func FindAgentsForScrapeConfig(q *reform.Querier, pmmAgentID *string, pushMetric

// FindPMMAgentsIDsWithPushMetrics returns pmm-agents-ids with agent, that use push_metrics mode.
func FindPMMAgentsIDsWithPushMetrics(q *reform.Querier) ([]string, error) {
structs, err := q.SelectAllFrom(AgentTable, "WHERE NOT disabled AND pmm_agent_id IS NOT NULL AND push_metrics ORDER BY agent_id")
structs, err := q.SelectAllFrom(AgentTable, fmt.Sprintf("WHERE NOT disabled AND pmm_agent_id IS NOT NULL AND %s ORDER BY agent_id", pushMetricsTrue))
if err != nil {
return nil, status.Error(codes.FailedPrecondition, "Couldn't get agents")
}
Expand Down Expand Up @@ -640,15 +639,17 @@ func CreateNodeExporter(q *reform.Querier,
" it doesn't support it, minimum supported version=%q", pointer.GetString(pmmAgent.Version), PMMAgentWithPushMetricsSupport.String())
}
row := &Agent{
AgentID: id,
AgentType: NodeExporterType,
PMMAgentID: &pmmAgentID,
NodeID: pmmAgent.RunsOnNodeID,
PushMetrics: pushMetrics,
DisabledCollectors: disableCollectors,
AgentPassword: agentPassword,
LogLevel: pointer.ToStringOrNil(logLevel),
ExposeExporter: exposeExporter,
AgentID: id,
AgentType: NodeExporterType,
PMMAgentID: &pmmAgentID,
NodeID: pmmAgent.RunsOnNodeID,
AgentPassword: agentPassword,
ExporterOptions: &ExporterOptions{
ExposeExporter: exposeExporter,
PushMetrics: pushMetrics,
DisabledCollectors: disableCollectors,
},
LogLevel: pointer.ToStringOrNil(logLevel),
}
if err := row.SetCustomLabels(customLabels); err != nil {
return nil, err
Expand Down Expand Up @@ -725,17 +726,19 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar
metricsPath = "/metrics"
}
row := &Agent{
PMMAgentID: pmmAgentID,
AgentID: id,
AgentType: ExternalExporterType,
RunsOnNodeID: runsOnNodeID,
ServiceID: pointer.ToStringOrNil(params.ServiceID),
Username: pointer.ToStringOrNil(params.Username),
Password: pointer.ToStringOrNil(params.Password),
MetricsScheme: &scheme,
MetricsPath: &metricsPath,
ListenPort: pointer.ToUint16(uint16(params.ListenPort)),
PushMetrics: params.PushMetrics,
PMMAgentID: pmmAgentID,
AgentID: id,
AgentType: ExternalExporterType,
RunsOnNodeID: runsOnNodeID,
ServiceID: pointer.ToStringOrNil(params.ServiceID),
Username: pointer.ToStringOrNil(params.Username),
Password: pointer.ToStringOrNil(params.Password),
ListenPort: pointer.ToUint16(uint16(params.ListenPort)),
ExporterOptions: &ExporterOptions{
PushMetrics: params.PushMetrics,
MetricsPath: metricsPath,
MetricsScheme: scheme,
},
}
if err := row.SetCustomLabels(params.CustomLabels); err != nil {
return nil, err
Expand All @@ -752,33 +755,23 @@ func CreateExternalExporter(q *reform.Querier, params *CreateExternalExporterPar

// CreateAgentParams params for add common exporter.
type CreateAgentParams struct {
PMMAgentID string
NodeID string
ServiceID string
Username string
Password string
AgentPassword string
CustomLabels map[string]string
TLS bool
TLSSkipVerify bool
MySQLOptions *MySQLOptions
MongoDBOptions *MongoDBOptions
PostgreSQLOptions *PostgreSQLOptions
TableCountTablestatsGroupLimit int32
MaxQueryLength int32
QueryExamplesDisabled bool
CommentsParsingDisabled bool
MaxQueryLogSize int64
AWSAccessKey string
AWSSecretKey string
RDSBasicMetricsDisabled bool
RDSEnhancedMetricsDisabled bool
AzureOptions *AzureOptions
PushMetrics bool
ExposeExporter bool
DisableCollectors []string
LogLevel string
MetricsResolutions *MetricsResolutions
PMMAgentID string
NodeID string
ServiceID string
Username string
Password string
AgentPassword string
CustomLabels map[string]string
TLS bool
TLSSkipVerify bool
LogLevel string
ExporterOptions *ExporterOptions
QANOptions *QANOptions
AWSOptions *AWSOptions
AzureOptions *AzureOptions
MongoDBOptions *MongoDBOptions
MySQLOptions *MySQLOptions
PostgreSQLOptions *PostgreSQLOptions
}

func compatibleNodeAndAgent(nodeType NodeType, agentType AgentType) bool {
Expand Down Expand Up @@ -859,19 +852,47 @@ func compatibleServiceAndAgent(serviceType ServiceType, agentType AgentType) boo
return false
}

func prepareOptionsParams(params *CreateAgentParams) *CreateAgentParams {
if params.ExporterOptions == nil {
params.ExporterOptions = &ExporterOptions{}
}
if params.QANOptions == nil {
params.QANOptions = &QANOptions{}
}
if params.AWSOptions == nil {
params.AWSOptions = &AWSOptions{}
}
if params.AzureOptions == nil {
params.AzureOptions = &AzureOptions{}
}
if params.MongoDBOptions == nil {
params.MongoDBOptions = &MongoDBOptions{}
}
if params.MySQLOptions == nil {
params.MySQLOptions = &MySQLOptions{}
}
if params.PostgreSQLOptions == nil {
params.PostgreSQLOptions = &PostgreSQLOptions{}
}

return params
}

// CreateAgent creates Agent with given type.
func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentParams) (*Agent, error) { //nolint:unparam
id := uuid.New().String()
if err := checkUniqueAgentID(q, id); err != nil {
return nil, err
}

params = prepareOptionsParams(params)

pmmAgent, err := FindAgentByID(q, params.PMMAgentID)
if err != nil {
return nil, err
}
// check version for agent, if it exists.
if params.PushMetrics {
if params.ExporterOptions.PushMetrics {
// special case for vmAgent, it always supports push metrics.
if agentType != VMAgentType && !IsPushMetricsSupported(pmmAgent.Version) {
return nil, status.Errorf(codes.FailedPrecondition, "cannot use push_metrics_enabled with pmm_agent version=%q,"+
Expand Down Expand Up @@ -902,33 +923,24 @@ func CreateAgent(q *reform.Querier, agentType AgentType, params *CreateAgentPara
}

row := &Agent{
AgentID: id,
AgentType: agentType,
PMMAgentID: &params.PMMAgentID,
ServiceID: pointer.ToStringOrNil(params.ServiceID),
NodeID: pointer.ToStringOrNil(params.NodeID),
Username: pointer.ToStringOrNil(params.Username),
Password: pointer.ToStringOrNil(params.Password),
AgentPassword: pointer.ToStringOrNil(params.AgentPassword),
TLS: params.TLS,
TLSSkipVerify: params.TLSSkipVerify,
MySQLOptions: params.MySQLOptions,
MongoDBOptions: params.MongoDBOptions,
PostgreSQLOptions: params.PostgreSQLOptions,
TableCountTablestatsGroupLimit: params.TableCountTablestatsGroupLimit,
MaxQueryLength: params.MaxQueryLength,
QueryExamplesDisabled: params.QueryExamplesDisabled,
CommentsParsingDisabled: params.CommentsParsingDisabled,
MaxQueryLogSize: params.MaxQueryLogSize,
AWSAccessKey: pointer.ToStringOrNil(params.AWSAccessKey),
AWSSecretKey: pointer.ToStringOrNil(params.AWSSecretKey),
RDSBasicMetricsDisabled: params.RDSBasicMetricsDisabled,
RDSEnhancedMetricsDisabled: params.RDSEnhancedMetricsDisabled,
AzureOptions: params.AzureOptions,
PushMetrics: params.PushMetrics,
ExposeExporter: params.ExposeExporter,
DisabledCollectors: params.DisableCollectors,
LogLevel: pointer.ToStringOrNil(params.LogLevel),
AgentID: id,
AgentType: agentType,
PMMAgentID: &params.PMMAgentID,
ServiceID: pointer.ToStringOrNil(params.ServiceID),
NodeID: pointer.ToStringOrNil(params.NodeID),
Username: pointer.ToStringOrNil(params.Username),
Password: pointer.ToStringOrNil(params.Password),
AgentPassword: pointer.ToStringOrNil(params.AgentPassword),
TLS: params.TLS,
TLSSkipVerify: params.TLSSkipVerify,
ExporterOptions: params.ExporterOptions,
QANOptions: params.QANOptions,
AWSOptions: params.AWSOptions,
AzureOptions: params.AzureOptions,
MongoDBOptions: params.MongoDBOptions,
MySQLOptions: params.MySQLOptions,
PostgreSQLOptions: params.PostgreSQLOptions,
LogLevel: pointer.ToStringOrNil(params.LogLevel),
}
if err := row.SetCustomLabels(params.CustomLabels); err != nil {
return nil, err
Expand Down Expand Up @@ -970,7 +982,7 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar
}

if params.EnablePushMetrics != nil {
row.PushMetrics = *params.EnablePushMetrics
row.ExporterOptions.PushMetrics = *params.EnablePushMetrics
if row.AgentType == ExternalExporterType {
if err := updateExternalExporterParams(q, row); err != nil {
return nil, errors.Wrap(err, "failed to update External exporterParams for PushMetrics")
Expand All @@ -990,22 +1002,22 @@ func ChangeAgent(q *reform.Querier, agentID string, params *ChangeCommonAgentPar
}
}

if row.MetricsResolutions == nil {
row.MetricsResolutions = &MetricsResolutions{}
if row.ExporterOptions.MetricsResolutions == nil {
row.ExporterOptions.MetricsResolutions = &MetricsResolutions{}
}
if params.MetricsResolutions.LR != nil {
row.MetricsResolutions.LR = *params.MetricsResolutions.LR
row.ExporterOptions.MetricsResolutions.LR = *params.MetricsResolutions.LR
}
if params.MetricsResolutions.MR != nil {
row.MetricsResolutions.MR = *params.MetricsResolutions.MR
row.ExporterOptions.MetricsResolutions.MR = *params.MetricsResolutions.MR
}
if params.MetricsResolutions.HR != nil {
row.MetricsResolutions.HR = *params.MetricsResolutions.HR
row.ExporterOptions.MetricsResolutions.HR = *params.MetricsResolutions.HR
}

// If all resolutions are empty, then drop whole MetricsResolution field.
if row.MetricsResolutions.HR == 0 && row.MetricsResolutions.MR == 0 && row.MetricsResolutions.LR == 0 {
row.MetricsResolutions = nil
if row.ExporterOptions.MetricsResolutions.HR == 0 && row.ExporterOptions.MetricsResolutions.MR == 0 && row.ExporterOptions.MetricsResolutions.LR == 0 {
row.ExporterOptions.MetricsResolutions = nil
}

if err = q.Update(row); err != nil {
Expand Down Expand Up @@ -1057,7 +1069,7 @@ func RemoveAgent(q *reform.Querier, id string, mode RemoveMode) (*Agent, error)
// for external exporter, is needed for push_metrics mode.
func updateExternalExporterParams(q *reform.Querier, row *Agent) error {
// with push metrics, external exporter must have PMMAgent id without RunsOnNodeID
if row.PushMetrics && row.PMMAgentID == nil {
if row.ExporterOptions.PushMetrics && row.PMMAgentID == nil {
pmmAgent, err := FindPMMAgentsRunningOnNode(q, pointer.GetString(row.RunsOnNodeID))
if err != nil {
return err
Expand All @@ -1074,7 +1086,7 @@ func updateExternalExporterParams(q *reform.Querier, row *Agent) error {
row.PMMAgentID = pointer.ToString(pmmAgent[0].AgentID)
}
// without push metrics, external exporter must have RunsOnNodeID without PMMAgentID
if !row.PushMetrics && row.RunsOnNodeID == nil {
if !row.ExporterOptions.PushMetrics && row.RunsOnNodeID == nil {
pmmAgent, err := FindAgentByID(q, pointer.GetString(row.PMMAgentID))
if err != nil {
return err
Expand Down
Loading

0 comments on commit fba3928

Please sign in to comment.