diff --git a/.golangci.yml b/.golangci.yml index 36c29c1d237..0ff29b401a7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -51,6 +51,7 @@ linters-settings: - github.com/charmbracelet/bubbletea.Model - github.com/percona/pmm/admin/commands.Result - github.com/percona/pmm/agent/runner/actions.Action + - github.com/percona/pmm/managed/services/telemetry.DataSource lll: line-length: 170 diff --git a/managed/services/config/config.go b/managed/services/config/config.go index da63fa58e79..fd4cedf7b59 100644 --- a/managed/services/config/config.go +++ b/managed/services/config/config.go @@ -73,7 +73,7 @@ func (s *Service) Load() error { var cfg Config if _, err := os.Stat(configPath); err == nil { - s.l.Trace("config exist, reading file") + s.l.Trace("config exists, reading file") buf, err := os.ReadFile(configPath) //nolint:gosec if err != nil { return errors.Wrapf(err, "error while reading config [%s]", configPath) diff --git a/managed/services/config/pmm-managed.yaml b/managed/services/config/pmm-managed.yaml index d9341b020f3..c9dee75d745 100644 --- a/managed/services/config/pmm-managed.yaml +++ b/managed/services/config/pmm-managed.yaml @@ -20,6 +20,8 @@ services: enabled: true timeout: 5s db_file: /srv/grafana/grafana.db + ENV_VARS: + enabled: true reporting: send: true send_on_start: false diff --git a/managed/services/telemetry/config.default.yml b/managed/services/telemetry/config.default.yml index 74924104e06..a7803f81092 100644 --- a/managed/services/telemetry/config.default.yml +++ b/managed/services/telemetry/config.default.yml @@ -936,3 +936,36 @@ telemetry: data: - metric_name: "postgresql_db_count" value: 1 + + # Note: use these for testing and PMM-12462 + # - id: PMMServerFeatureToggles + # source: ENV_VARS + # summary: "Use of feature toggles in PMM Server" + # data: + # - metric_name: "pmm_server_disable_telemetry" + # column: "DISABLE_TELEMETRY" + # - metric_name: "pmm_server_enable_alerting" + # column: "ENABLE_ALERTING" + # - metric_name: "pmm_server_enable_backup_management" + # column: "ENABLE_BACKUP_MANAGEMENT" + # - metric_name: "pmm_server_enable_debug" + # column: "ENABLE_DEBUG" + # - metric_name: "pmm_server_enable_rbac" + # column: "ENABLE_RBAC" + + # - id: PMMServerFeatureTogglesStripValues + # source: ENV_VARS + # summary: "Use of feature toggles in PMM Server" + # transform: + # type: StripValues + # data: + # - metric_name: "pmm_server_disable_telemetry" + # column: "DISABLE_TELEMETRY" + # - metric_name: "pmm_server_enable_alerting" + # column: "ENABLE_ALERTING" + # - metric_name: "pmm_server_enable_backup_management" + # column: "ENABLE_BACKUP_MANAGEMENT" + # - metric_name: "pmm_server_enable_debug" + # column: "ENABLE_DEBUG" + # - metric_name: "pmm_server_enable_rbac" + # column: "ENABLE_RBAC" diff --git a/managed/services/telemetry/config.go b/managed/services/telemetry/config.go index 40372b297fa..97ea351a8c0 100644 --- a/managed/services/telemetry/config.go +++ b/managed/services/telemetry/config.go @@ -37,19 +37,31 @@ const ( envReportingRetryBackoff = "PERCONA_TEST_TELEMETRY_RETRY_BACKOFF" ) +const ( + dsVM = DataSourceName("VM") + dsQANDBSelect = DataSourceName("QANDB_SELECT") + dsPMMDBSelect = DataSourceName("PMMDB_SELECT") + dsGrafanaDBSelect = DataSourceName("GRAFANADB_SELECT") + dsEnvVars = DataSourceName("ENV_VARS") +) + +// DataSources holds all possible data source types. +type DataSources struct { + VM *DataSourceVictoriaMetrics `yaml:"VM"` + QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` + PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` + GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"` + EnvVars *DSConfigEnvVars `yaml:"ENV_VARS"` +} + // ServiceConfig telemetry config. type ServiceConfig struct { l *logrus.Entry - Enabled bool `yaml:"enabled"` - telemetry []Config `yaml:"-"` - SaasHostname string `yaml:"saas_hostname"` - DataSources struct { - VM *DataSourceVictoriaMetrics `yaml:"VM"` - QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` - PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` - GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"` - } `yaml:"datasources"` - Reporting ReportingConfig `yaml:"reporting"` + Enabled bool `yaml:"enabled"` + telemetry []Config `yaml:"-"` + SaasHostname string `yaml:"saas_hostname"` + DataSources DataSources `yaml:"datasources"` + Reporting ReportingConfig `yaml:"reporting"` } // FileConfig top level telemetry config element. @@ -100,7 +112,11 @@ type DSConfigPMMDB struct { //nolint:musttag } `yaml:"separate_credentials"` } -// Config telemetry config. +type DSConfigEnvVars struct { + Enabled bool `yaml:"enabled"` +} + +// Config is a telemetry config. type Config struct { ID string `yaml:"id"` Source string `yaml:"source"` @@ -111,21 +127,23 @@ type Config struct { Data []ConfigData } -// ConfigTransform telemetry config transformation. +// ConfigTransform is a telemetry config transformation. type ConfigTransform struct { Type ConfigTransformType `yaml:"type"` Metric string `yaml:"metric"` } -// ConfigTransformType config transform type. +// ConfigTransformType is a config transform type. type ConfigTransformType string const ( - // JSONTransformType JSON type. - JSONTransformType = ConfigTransformType("JSON") + // JSONTransform converts multiple metrics in one formatted as JSON. + JSONTransform = ConfigTransformType("JSON") + // StripValuesTransform strips values from metrics, replacing them with 0/1 depending on presence. + StripValuesTransform = ConfigTransformType("StripValues") ) -// ConfigData telemetry config. +// ConfigData is a telemetry data config. type ConfigData struct { MetricName string `yaml:"metric_name"` Label string `yaml:"label"` diff --git a/managed/services/telemetry/config_test.go b/managed/services/telemetry/config_test.go index 5d497688d0a..7a7da4ab7e1 100644 --- a/managed/services/telemetry/config_test.go +++ b/managed/services/telemetry/config_test.go @@ -48,6 +48,8 @@ datasources: enabled: true timeout: 2s db_file: /srv/grafana/grafana.db + ENV_VARS: + enabled: true reporting: send: true @@ -71,12 +73,7 @@ reporting: RetryCount: 2, SendTimeout: time.Second * 10, }, - DataSources: struct { - VM *DataSourceVictoriaMetrics `yaml:"VM"` - QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` - PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` - GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"` - }{ + DataSources: DataSources{ VM: &DataSourceVictoriaMetrics{ Enabled: true, Timeout: time.Second * 2, @@ -103,6 +100,9 @@ reporting: Timeout: time.Second * 2, DBFile: "/srv/grafana/grafana.db", }, + EnvVars: &DSConfigEnvVars{ + Enabled: true, + }, }, } assert.Equal(t, actual, expected) diff --git a/managed/services/telemetry/datasource_envvars.go b/managed/services/telemetry/datasource_envvars.go new file mode 100644 index 00000000000..4dc320a51c8 --- /dev/null +++ b/managed/services/telemetry/datasource_envvars.go @@ -0,0 +1,84 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package telemetry provides telemetry functionality. +package telemetry + +import ( + "context" + "os" + + pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm" + "github.com/sirupsen/logrus" +) + +type dsEnvvars struct { + l *logrus.Entry + config DSConfigEnvVars +} + +// check interfaces. +var ( + _ DataSource = (*dsEnvvars)(nil) +) + +// NewDataSourceEnvVars makes a new data source for collecting envvars. +func NewDataSourceEnvVars(config DSConfigEnvVars, l *logrus.Entry) DataSource { + return &dsEnvvars{ + l: l, + config: config, + } +} + +// Enabled flag that determines if data source is enabled. +func (d *dsEnvvars) Enabled() bool { + return d.config.Enabled +} + +func (d *dsEnvvars) Init(_ context.Context) error { + return nil +} + +func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) { + var metrics []*pmmv1.ServerMetric_Metric + + check := make(map[string]bool, len(config.Data)) + + for _, col := range config.Data { + if col.Column == "" { + d.l.Warnf("no column defined or empty column name in config %s", config.ID) + continue + } + if value, ok := os.LookupEnv(col.Column); ok && value != "" { + if _, alreadyHasItem := check[col.MetricName]; alreadyHasItem { + d.l.Warnf("repeated metric key %s found in config %s, the last will win", col.MetricName, config.ID) + continue + } + + check[col.MetricName] = true + + metrics = append(metrics, &pmmv1.ServerMetric_Metric{ + Key: col.MetricName, + Value: value, + }) + } + } + + return metrics, nil +} + +func (d *dsEnvvars) Dispose(_ context.Context) error { + return nil +} diff --git a/managed/services/telemetry/datasource_envvars_test.go b/managed/services/telemetry/datasource_envvars_test.go new file mode 100644 index 00000000000..5139d68c486 --- /dev/null +++ b/managed/services/telemetry/datasource_envvars_test.go @@ -0,0 +1,151 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package telemetry provides telemetry functionality. +package telemetry + +import ( + "context" + "os" + "testing" + + pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnvVarsDatasource(t *testing.T) { + // NOTE: t.Parallel() is not possible when using a different set of envvars for each test. + t.Parallel() + + type testEnvVars map[string]string + + ctx, cancel := context.WithCancel(context.Background()) + logger := logrus.StandardLogger() + logger.SetLevel(logrus.DebugLevel) + logEntry := logrus.NewEntry(logger) + + setup := func(t *testing.T, envVars testEnvVars) (DataSource, func()) { + t.Helper() + for key, val := range envVars { + os.Setenv(key, val) //nolint:errcheck + } + + evConf := &DSConfigEnvVars{ + Enabled: true, + } + dsEnvVars := NewDataSourceEnvVars(*evConf, logEntry) + + return dsEnvVars, func() { + for key := range envVars { + os.Unsetenv(key) //nolint:errcheck + } + err := dsEnvVars.Dispose(ctx) + require.NoError(t, err) + } + } + + t.Cleanup(func() { + cancel() + }) + + t.Run("Basic", func(t *testing.T) { + t.Parallel() + + envVars := testEnvVars{ + "TEST_ENV_VAR1": "1", + "TEST_ENV_VAR2": "test", + "TEST_ENV_VAR3": "true", + "TEST_ENV_VAR4": "1.1", + "TEST_ENV_VAR5": "", + } + config := &Config{ + ID: "test", + Source: "ENV_VARS", + Summary: "EnvVar test query", + Data: []ConfigData{ + {MetricName: "test_env_var1", Column: "TEST_ENV_VAR1"}, + {MetricName: "test_env_var2", Column: "TEST_ENV_VAR2"}, + {MetricName: "test_env_var3", Column: "TEST_ENV_VAR3"}, + {MetricName: "test_env_var4", Column: "TEST_ENV_VAR4"}, + {MetricName: "test_env_var5", Column: "TEST_ENV_VAR5"}, + }, + } + + dsEnvVars, dispose := setup(t, envVars) + t.Cleanup(func() { dispose() }) + + err := dsEnvVars.Init(ctx) + require.NoError(t, err) + + metrics, err := dsEnvVars.FetchMetrics(ctx, *config) + require.NoError(t, err) + + expected := []*pmmv1.ServerMetric_Metric{ + {Key: "test_env_var1", Value: "1"}, + {Key: "test_env_var2", Value: "test"}, + {Key: "test_env_var3", Value: "true"}, + {Key: "test_env_var4", Value: "1.1"}, + } + assert.Equal(t, expected, metrics) + }) + + t.Run("StripValues", func(t *testing.T) { + t.Parallel() + + envVars := testEnvVars{ + "TEST_ENV_VAR6": "1", + "TEST_ENV_VAR7": "test", + "TEST_ENV_VAR8": "true", + "TEST_ENV_VAR9": "1.1", + "TEST_ENV_VAR10": "", + } + config := &Config{ + ID: "test", + Source: "ENV_VARS", + Summary: "EnvVar test query", + Transform: &ConfigTransform{ + Type: "StripValues", + }, + Data: []ConfigData{ + {MetricName: "test_env_var6", Column: "TEST_ENV_VAR6"}, + {MetricName: "test_env_var7", Column: "TEST_ENV_VAR7"}, + {MetricName: "test_env_var8", Column: "TEST_ENV_VAR8"}, + {MetricName: "test_env_var9", Column: "TEST_ENV_VAR9"}, + {MetricName: "test_env_var10", Column: "TEST_ENV_VAR10"}, + }, + } + + dsEnvVars, dispose := setup(t, envVars) + t.Cleanup(func() { dispose() }) + + err := dsEnvVars.Init(ctx) + require.NoError(t, err) + + metrics, err := dsEnvVars.FetchMetrics(ctx, *config) + require.NoError(t, err) + + expected := []*pmmv1.ServerMetric_Metric{ + {Key: "test_env_var6", Value: "1"}, + {Key: "test_env_var7", Value: "1"}, + {Key: "test_env_var8", Value: "1"}, + {Key: "test_env_var9", Value: "1"}, + } + metrics, err = transformExportValues(config, metrics) + require.NoError(t, err) + assert.Equal(t, expected, metrics) + }) +} diff --git a/managed/services/telemetry/datasource_grafana_sqlitedb.go b/managed/services/telemetry/datasource_grafana_sqlitedb.go index 689a3decfdb..1c451410886 100644 --- a/managed/services/telemetry/datasource_grafana_sqlitedb.go +++ b/managed/services/telemetry/datasource_grafana_sqlitedb.go @@ -47,7 +47,7 @@ func (d *dsGrafanaSelect) Enabled() bool { } // NewDataSourceGrafanaSqliteDB makes new data source for grafana sqlite database metrics. -func NewDataSourceGrafanaSqliteDB(config DSGrafanaSqliteDB, l *logrus.Entry) DataSource { //nolint:ireturn +func NewDataSourceGrafanaSqliteDB(config DSGrafanaSqliteDB, l *logrus.Entry) DataSource { return &dsGrafanaSelect{ l: l, config: config, diff --git a/managed/services/telemetry/datasource_grafana_sqlitedb_test.go b/managed/services/telemetry/datasource_grafana_sqlitedb_test.go index 4877c9263d7..ec140ce5507 100644 --- a/managed/services/telemetry/datasource_grafana_sqlitedb_test.go +++ b/managed/services/telemetry/datasource_grafana_sqlitedb_test.go @@ -27,7 +27,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestDatasource(t *testing.T) { +func TestGrafanaSqliteDatasource(t *testing.T) { t.Parallel() logger := logrus.StandardLogger() logger.SetLevel(logrus.DebugLevel) diff --git a/managed/services/telemetry/datasource_pmmdb_select.go b/managed/services/telemetry/datasource_pmmdb_select.go index 08e6485e362..8633784e7c4 100644 --- a/managed/services/telemetry/datasource_pmmdb_select.go +++ b/managed/services/telemetry/datasource_pmmdb_select.go @@ -44,7 +44,7 @@ func (d *dsPmmDBSelect) Enabled() bool { } // NewDsPmmDBSelect make new PMM DB Select data source. -func NewDsPmmDBSelect(config DSConfigPMMDB, l *logrus.Entry) (DataSource, error) { //nolint:ireturn +func NewDsPmmDBSelect(config DSConfigPMMDB, l *logrus.Entry) (DataSource, error) { db, err := openPMMDBConnection(config, l) if err != nil { return nil, err diff --git a/managed/services/telemetry/datasource_qandb_select.go b/managed/services/telemetry/datasource_qandb_select.go index 3f800532821..d0e90494151 100644 --- a/managed/services/telemetry/datasource_qandb_select.go +++ b/managed/services/telemetry/datasource_qandb_select.go @@ -42,7 +42,7 @@ func (d *dsQanDBSelect) Enabled() bool { } // NewDsQanDBSelect make new QAN DB Select data source. -func NewDsQanDBSelect(config DSConfigQAN, l *logrus.Entry) (DataSource, error) { //nolint:ireturn +func NewDsQanDBSelect(config DSConfigQAN, l *logrus.Entry) (DataSource, error) { db, err := openQANDBConnection(config.DSN, config.Enabled, l) if err != nil { return nil, err diff --git a/managed/services/telemetry/datasources.go b/managed/services/telemetry/datasources.go index 8eefc3c38e6..af9a0c36d77 100644 --- a/managed/services/telemetry/datasources.go +++ b/managed/services/telemetry/datasources.go @@ -54,13 +54,16 @@ func NewDataSourceRegistry(config ServiceConfig, l *logrus.Entry) (DataSourceLoc grafanaDB := NewDataSourceGrafanaSqliteDB(*config.DataSources.GrafanaDBSelect, l) + envVars := NewDataSourceEnvVars(*config.DataSources.EnvVars, l) + return &dataSourceRegistry{ l: l, dataSources: map[DataSourceName]DataSource{ - "VM": vmDB, - "PMMDB_SELECT": pmmDB, - "QANDB_SELECT": qanDB, - "GRAFANADB_SELECT": grafanaDB, + dsVM: vmDB, + dsPMMDBSelect: pmmDB, + dsQANDBSelect: qanDB, + dsGrafanaDBSelect: grafanaDB, + dsEnvVars: envVars, }, }, nil } diff --git a/managed/services/telemetry/telemetry.go b/managed/services/telemetry/telemetry.go index 99b491b0e99..94c17d8a990 100644 --- a/managed/services/telemetry/telemetry.go +++ b/managed/services/telemetry/telemetry.go @@ -56,7 +56,7 @@ type Service struct { sDistributionMethod serverpb.DistributionMethod tDistributionMethod pmmv1.DistributionMethod sendCh chan *pmmv1.ServerMetric - dataSourcesMap map[string]DataSource + dataSourcesMap map[DataSourceName]DataSource extensions map[ExtensionType]Extension @@ -101,7 +101,7 @@ func NewService(db *reform.DB, portalClient *platform.Client, pmmVersion string, } // LocateTelemetryDataSource retrieves DataSource by name. -func (s *Service) LocateTelemetryDataSource(name string) (DataSource, error) { //nolint:ireturn +func (s *Service) LocateTelemetryDataSource(name string) (DataSource, error) { return s.dsRegistry.LocateTelemetryDataSource(name) } @@ -117,7 +117,7 @@ func (s *Service) Run(ctx context.Context) { doSend := func() { var settings *models.Settings - err := s.db.InTransaction(func(tx *reform.TX) error { + err := s.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error { var e error if settings, e = models.GetSettings(tx); e != nil { return e @@ -140,12 +140,12 @@ func (s *Service) Run(ctx context.Context) { if s.config.Reporting.Send { s.sendCh <- report } else { - s.l.Info("Telemetry sent is disabled.") + s.l.Info("Sending telemetry is disabled.") } } if s.config.Reporting.SendOnStart { - s.l.Debug("Telemetry on start is enabled, sending...") + s.l.Debug("Sending telemetry on start is enabled, in progress...") doSend() } @@ -213,7 +213,7 @@ func (s *Service) processSendCh(ctx context.Context) { } func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { - initializedDataSources := make(map[string]DataSource) + initializedDataSources := make(map[DataSourceName]DataSource) telemetryMetric, _ := s.makeMetric(ctx) var totalTime time.Duration @@ -246,7 +246,7 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { } // locate DS in initialized state - ds := initializedDataSources[telemetry.Source] + ds := initializedDataSources[DataSourceName(telemetry.Source)] if ds == nil { s.l.Debugf("cannot find initialized telemetry datasource: %s", telemetry.Source) continue @@ -268,15 +268,23 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { } if telemetry.Transform != nil { - if telemetry.Transform.Type == JSONTransformType { + switch telemetry.Transform.Type { + case JSONTransform: telemetryCopy := telemetry // G601: Implicit memory aliasing in for loop. (gosec) metrics, err = transformToJSON(&telemetryCopy, metrics) if err != nil { s.l.Debugf("failed to transform to JSON: %s", err) continue } - } else { - s.l.Errorf("Unsupported transform type: %s", telemetry.Transform.Type) + case StripValuesTransform: + telemetryCopy := telemetry // G601: Implicit memory aliasing in for loop. (gosec) + metrics, err = transformExportValues(&telemetryCopy, metrics) + if err != nil { + s.l.Debugf("failed to strip values: %s", err) + continue + } + default: + s.l.Errorf("unsupported transform type: %s", telemetry.Transform.Type) } } @@ -287,7 +295,7 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { for sourceName, dataSource := range initializedDataSources { err := dataSource.Dispose(ctx) if err != nil { - s.l.Debugf("Dispose of %s datasource failed: %v", sourceName, err) + s.l.Debugf("Disposing of %s datasource failed: %v", sourceName, err) continue } } @@ -299,15 +307,15 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { return telemetryMetric } -func (s *Service) locateDataSources(telemetryConfig []Config) map[string]DataSource { - dataSources := make(map[string]DataSource) +func (s *Service) locateDataSources(telemetryConfig []Config) map[DataSourceName]DataSource { + dataSources := make(map[DataSourceName]DataSource) for _, telemetry := range telemetryConfig { ds, err := s.LocateTelemetryDataSource(telemetry.Source) if err != nil { s.l.Debugf("failed to lookup telemetry datasource for [%s]:[%s]", telemetry.Source, telemetry.ID) continue } - dataSources[telemetry.Source] = ds + dataSources[DataSourceName(telemetry.Source)] = ds } return dataSources diff --git a/managed/services/telemetry/telemetry_test.go b/managed/services/telemetry/telemetry_test.go index 726e80cbaa4..501147eb660 100644 --- a/managed/services/telemetry/telemetry_test.go +++ b/managed/services/telemetry/telemetry_test.go @@ -193,12 +193,7 @@ func getServiceConfig(pgPortHost string, qanDSN string, vmDSN string) ServiceCon RetryCount: 2, SendTimeout: time.Second * 10, }, - DataSources: struct { - VM *DataSourceVictoriaMetrics `yaml:"VM"` - QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` - PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` - GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"` - }{ + DataSources: DataSources{ VM: &DataSourceVictoriaMetrics{ Enabled: true, Timeout: time.Second * 2, @@ -237,6 +232,9 @@ func getServiceConfig(pgPortHost string, qanDSN string, vmDSN string) ServiceCon Timeout: time.Second * 2, DBFile: "/srv/grafana/grafana.db", }, + EnvVars: &DSConfigEnvVars{ + Enabled: true, + }, }, } return serviceConfig @@ -302,12 +300,7 @@ func getTestConfig(sendOnStart bool, testSourceName string, reportingInterval ti }, }, SaasHostname: "", - DataSources: struct { - VM *DataSourceVictoriaMetrics `yaml:"VM"` - QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` - PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` - GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"` - }{}, + DataSources: DataSources{}, Reporting: ReportingConfig{ Send: true, SendOnStart: sendOnStart, diff --git a/managed/services/telemetry/transform.go b/managed/services/telemetry/transform.go index b9ab4c7ac60..2616c7317be 100644 --- a/managed/services/telemetry/transform.go +++ b/managed/services/telemetry/transform.go @@ -33,8 +33,8 @@ func transformToJSON(config *Config, metrics []*pmmv1.ServerMetric_Metric) ([]*p return nil, errors.Errorf("no transformation config is set") } - if config.Transform.Type != JSONTransformType { - return nil, errors.Errorf("not supported transformation type [%s], it must be [%s]", config.Transform.Type, JSONTransformType) + if config.Transform.Type != JSONTransform { + return nil, errors.Errorf("unsupported transformation type [%s], it must be [%s]", config.Transform.Type, JSONTransform) } if len(config.Data) == 0 || config.Data[0].MetricName == "" { @@ -86,6 +86,27 @@ func transformToJSON(config *Config, metrics []*pmmv1.ServerMetric_Metric) ([]*p }, nil } +func transformExportValues(config *Config, metrics []*pmmv1.ServerMetric_Metric) ([]*pmmv1.ServerMetric_Metric, error) { + if len(metrics) == 0 { + return metrics, nil + } + + if config.Transform.Type != StripValuesTransform { + return nil, errors.Errorf("unspported transformation type [%s], it must be [%s]", config.Transform.Type, StripValuesTransform) + } + + if config.Source != string(dsEnvVars) { + return nil, errors.Errorf("this transform can only be used for %s data source", dsEnvVars) + } + + for _, metric := range metrics { + // Here we replace the metric value with "1", which stands for "present". + metric.Value = "1" + } + + return metrics, nil +} + func removeEmpty(metrics []*pmmv1.ServerMetric_Metric) []*pmmv1.ServerMetric_Metric { result := make([]*pmmv1.ServerMetric_Metric, 0, len(metrics)) diff --git a/managed/services/telemetry/transform_test.go b/managed/services/telemetry/transform_test.go index 6e4e8f2cbc1..65a59fe6bd3 100644 --- a/managed/services/telemetry/transform_test.go +++ b/managed/services/telemetry/transform_test.go @@ -16,18 +16,18 @@ package telemetry import ( - "fmt" "testing" pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm" "github.com/stretchr/testify/assert" ) -func Test_transformToJSON(t *testing.T) { +func TestTransformToJSON(t *testing.T) { type args struct { config *Config metrics []*pmmv1.ServerMetric_Metric } + noMetrics := []*pmmv1.ServerMetric_Metric{} tests := []struct { @@ -39,7 +39,7 @@ func Test_transformToJSON(t *testing.T) { { name: "nil metrics", args: args{ - config: config(), + config: configJSON(), metrics: nil, }, want: nil, @@ -48,7 +48,7 @@ func Test_transformToJSON(t *testing.T) { { name: "empty metrics", args: args{ - config: config(), + config: configJSON(), metrics: noMetrics, }, want: noMetrics, @@ -57,7 +57,7 @@ func Test_transformToJSON(t *testing.T) { { name: "no Transform in config", args: args{ - config: config().noTransform(), + config: configJSON().noTransform(), metrics: noMetrics, }, want: noMetrics, @@ -66,7 +66,7 @@ func Test_transformToJSON(t *testing.T) { { name: "no Metrics config", args: args{ - config: config().noFirstMetricConfig(), + config: configJSON().noFirstMetricConfig(), metrics: noMetrics, }, want: noMetrics, @@ -75,7 +75,7 @@ func Test_transformToJSON(t *testing.T) { { name: "no Metric Name config", args: args{ - config: config().noFirstMetricNameConfig(), + config: configJSON().noFirstMetricNameConfig(), metrics: noMetrics, }, want: noMetrics, @@ -84,7 +84,7 @@ func Test_transformToJSON(t *testing.T) { { name: "invalid seq", args: args{ - config: config(), + config: configJSON(), metrics: []*pmmv1.ServerMetric_Metric{ {Key: "my-metric", Value: "v1"}, {Key: "b", Value: "v1"}, @@ -98,7 +98,7 @@ func Test_transformToJSON(t *testing.T) { { name: "correct seq", args: args{ - config: config(), + config: configJSON(), metrics: []*pmmv1.ServerMetric_Metric{ {Key: "my-metric", Value: "v1"}, {Key: "b", Value: "v1"}, @@ -107,29 +107,32 @@ func Test_transformToJSON(t *testing.T) { }, }, want: []*pmmv1.ServerMetric_Metric{ - {Key: config().Transform.Metric, Value: `{"v":[{"b":"v1","my-metric":"v1"},{"b":"v1","my-metric":"v1"}]}`}, + {Key: configJSON().Transform.Metric, Value: `{"v":[{"b":"v1","my-metric":"v1"},{"b":"v1","my-metric":"v1"}]}`}, }, wantErr: assert.NoError, }, { name: "happy path", args: args{ - config: config(), + config: configJSON(), metrics: []*pmmv1.ServerMetric_Metric{ - {Key: config().Data[0].MetricName, Value: "v1"}, - {Key: config().Data[0].MetricName, Value: "v2"}, + {Key: configJSON().Data[0].MetricName, Value: "v1"}, + {Key: configJSON().Data[0].MetricName, Value: "v2"}, }, }, want: []*pmmv1.ServerMetric_Metric{ - {Key: config().Transform.Metric, Value: `{"v":[{"my-metric":"v1"},{"my-metric":"v2"}]}`}, + {Key: configJSON().Transform.Metric, Value: `{"v":[{"my-metric":"v1"},{"my-metric":"v2"}]}`}, }, wantErr: assert.NoError, }, } + for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { got, err := transformToJSON(tt.args.config, tt.args.metrics) - if !tt.wantErr(t, err, fmt.Sprintf("transformToJSON(%v, %v)", tt.args.config, tt.args.metrics)) { + if !tt.wantErr(t, err) { + t.Logf("config: %v", tt.args.config) return } assert.Equalf(t, tt.want, got, "transformToJSON(%v, %v)", tt.args.config, tt.args.metrics) @@ -137,11 +140,103 @@ func Test_transformToJSON(t *testing.T) { } } -func config() *Config { +func TestTransformExportValues(t *testing.T) { + type args struct { + config *Config + metrics []*pmmv1.ServerMetric_Metric + } + + noMetrics := []*pmmv1.ServerMetric_Metric{} + + tests := []struct { + name string + args args + want []*pmmv1.ServerMetric_Metric + wantErr assert.ErrorAssertionFunc + }{ + { + name: "nil metrics", + args: args{ + config: configEnvVars(), + metrics: nil, + }, + want: nil, + wantErr: assert.NoError, + }, + { + name: "empty metrics", + args: args{ + config: configEnvVars(), + metrics: noMetrics, + }, + want: noMetrics, + wantErr: assert.NoError, + }, + { + name: "no Transform in config", + args: args{ + config: configEnvVars().noTransform(), + metrics: noMetrics, + }, + want: noMetrics, + wantErr: assert.NoError, + }, + { + name: "no Metrics config", + args: args{ + config: configEnvVars().noFirstMetricConfig(), + metrics: noMetrics, + }, + want: noMetrics, + wantErr: assert.NoError, + }, + { + name: "invalid data source", + args: args{ + config: configEnvVars().changeDataSource(dsPMMDBSelect), + metrics: []*pmmv1.ServerMetric_Metric{ + {Key: "metric-a", Value: "v1"}, + {Key: "metric-b", Value: "v2"}, + }, + }, + want: nil, + wantErr: assert.Error, + }, + { + name: "happy path", + args: args{ + config: configEnvVars(), + metrics: []*pmmv1.ServerMetric_Metric{ + {Key: "metric-a", Value: "v1"}, + {Key: "metric-b", Value: "v2"}, + }, + }, + want: []*pmmv1.ServerMetric_Metric{ + {Key: "metric-a", Value: "1"}, + {Key: "metric-b", Value: "1"}, + }, + wantErr: assert.NoError, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + got, err := transformExportValues(tt.args.config, tt.args.metrics) + if !tt.wantErr(t, err) { + t.Logf("config: %v", tt.args.config) + return + } + assert.Equalf(t, tt.want, got, "transformExportValues(%v, %v)", tt.args.config, tt.args.metrics) + }) + } +} + +func configJSON() *Config { return &Config{ Transform: &ConfigTransform{ Metric: "metric", - Type: JSONTransformType, + Type: JSONTransform, }, Data: []ConfigData{ {MetricName: "my-metric", Label: "label"}, @@ -149,6 +244,15 @@ func config() *Config { } } +func configEnvVars() *Config { + return &Config{ + Source: "ENV_VARS", + Transform: &ConfigTransform{ + Type: StripValuesTransform, + }, + } +} + func (c *Config) noTransform() *Config { c.Transform = nil return c @@ -164,7 +268,12 @@ func (c *Config) noFirstMetricNameConfig() *Config { return c } -func Test_removeEmpty(t *testing.T) { +func (c *Config) changeDataSource(s DataSourceName) *Config { + c.Source = string(s) + return c +} + +func TestRemoveEmpty(t *testing.T) { type args struct { metrics []*pmmv1.ServerMetric_Metric }