Skip to content

Commit

Permalink
PMM-12463 Add telemetry envvar datasource (percona#2532)
Browse files Browse the repository at this point in the history
* PMM-12463 allow ireturn on DataSources

* PMM-12463 fix some command hints

* PMM-12463 add ENV_VAR dataSource

* PMM-12463 add a provisionary envvar metrics extraction

* PMM-12463 add a test for EnvVar datasource

* PMM-12463 revert staticckeck linter

* PMM-12463 fix linter warnings

* PMM-12463 revert changes to main.go

* PMM-12463 refactor the output format

* PMM-12463 add a transform to strip values

* PMM-12463 fix linter warnings

* PMM-12463 refactor transform, add tests

* PMM-12463 run format

* PMM-12463 follow up on review comments

* PMM-12185 remove the debug statement

* PMM-12185 don't be too noisy when a ds is not initialized
  • Loading branch information
ademidoff authored and u5surf committed Oct 16, 2023
1 parent 297f92c commit 7351ea5
Show file tree
Hide file tree
Showing 17 changed files with 500 additions and 77 deletions.
1 change: 1 addition & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ linters-settings:
- github.com/charmbracelet/bubbletea.Model
- github.com/percona/pmm/admin/commands.Result
- github.com/percona/pmm/agent/runner/actions.Action
- github.com/percona/pmm/managed/services/telemetry.DataSource

lll:
line-length: 170
Expand Down
2 changes: 1 addition & 1 deletion managed/services/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func (s *Service) Load() error {
var cfg Config

if _, err := os.Stat(configPath); err == nil {
s.l.Trace("config exist, reading file")
s.l.Trace("config exists, reading file")
buf, err := os.ReadFile(configPath) //nolint:gosec
if err != nil {
return errors.Wrapf(err, "error while reading config [%s]", configPath)
Expand Down
2 changes: 2 additions & 0 deletions managed/services/config/pmm-managed.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ services:
enabled: true
timeout: 5s
db_file: /srv/grafana/grafana.db
ENV_VARS:
enabled: true
reporting:
send: true
send_on_start: false
Expand Down
33 changes: 33 additions & 0 deletions managed/services/telemetry/config.default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -936,3 +936,36 @@ telemetry:
data:
- metric_name: "postgresql_db_count"
value: 1

# Note: use these for testing and PMM-12462
# - id: PMMServerFeatureToggles
# source: ENV_VARS
# summary: "Use of feature toggles in PMM Server"
# data:
# - metric_name: "pmm_server_disable_telemetry"
# column: "DISABLE_TELEMETRY"
# - metric_name: "pmm_server_enable_alerting"
# column: "ENABLE_ALERTING"
# - metric_name: "pmm_server_enable_backup_management"
# column: "ENABLE_BACKUP_MANAGEMENT"
# - metric_name: "pmm_server_enable_debug"
# column: "ENABLE_DEBUG"
# - metric_name: "pmm_server_enable_rbac"
# column: "ENABLE_RBAC"

# - id: PMMServerFeatureTogglesStripValues
# source: ENV_VARS
# summary: "Use of feature toggles in PMM Server"
# transform:
# type: StripValues
# data:
# - metric_name: "pmm_server_disable_telemetry"
# column: "DISABLE_TELEMETRY"
# - metric_name: "pmm_server_enable_alerting"
# column: "ENABLE_ALERTING"
# - metric_name: "pmm_server_enable_backup_management"
# column: "ENABLE_BACKUP_MANAGEMENT"
# - metric_name: "pmm_server_enable_debug"
# column: "ENABLE_DEBUG"
# - metric_name: "pmm_server_enable_rbac"
# column: "ENABLE_RBAC"
50 changes: 34 additions & 16 deletions managed/services/telemetry/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,31 @@ const (
envReportingRetryBackoff = "PERCONA_TEST_TELEMETRY_RETRY_BACKOFF"
)

const (
dsVM = DataSourceName("VM")
dsQANDBSelect = DataSourceName("QANDB_SELECT")
dsPMMDBSelect = DataSourceName("PMMDB_SELECT")
dsGrafanaDBSelect = DataSourceName("GRAFANADB_SELECT")
dsEnvVars = DataSourceName("ENV_VARS")
)

// DataSources holds all possible data source types.
type DataSources struct {
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
EnvVars *DSConfigEnvVars `yaml:"ENV_VARS"`
}

// ServiceConfig telemetry config.
type ServiceConfig struct {
l *logrus.Entry
Enabled bool `yaml:"enabled"`
telemetry []Config `yaml:"-"`
SaasHostname string `yaml:"saas_hostname"`
DataSources struct {
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
} `yaml:"datasources"`
Reporting ReportingConfig `yaml:"reporting"`
Enabled bool `yaml:"enabled"`
telemetry []Config `yaml:"-"`
SaasHostname string `yaml:"saas_hostname"`
DataSources DataSources `yaml:"datasources"`
Reporting ReportingConfig `yaml:"reporting"`
}

// FileConfig top level telemetry config element.
Expand Down Expand Up @@ -100,7 +112,11 @@ type DSConfigPMMDB struct { //nolint:musttag
} `yaml:"separate_credentials"`
}

// Config telemetry config.
type DSConfigEnvVars struct {
Enabled bool `yaml:"enabled"`
}

// Config is a telemetry config.
type Config struct {
ID string `yaml:"id"`
Source string `yaml:"source"`
Expand All @@ -111,21 +127,23 @@ type Config struct {
Data []ConfigData
}

// ConfigTransform telemetry config transformation.
// ConfigTransform is a telemetry config transformation.
type ConfigTransform struct {
Type ConfigTransformType `yaml:"type"`
Metric string `yaml:"metric"`
}

// ConfigTransformType config transform type.
// ConfigTransformType is a config transform type.
type ConfigTransformType string

const (
// JSONTransformType JSON type.
JSONTransformType = ConfigTransformType("JSON")
// JSONTransform converts multiple metrics in one formatted as JSON.
JSONTransform = ConfigTransformType("JSON")
// StripValuesTransform strips values from metrics, replacing them with 0/1 depending on presence.
StripValuesTransform = ConfigTransformType("StripValues")
)

// ConfigData telemetry config.
// ConfigData is a telemetry data config.
type ConfigData struct {
MetricName string `yaml:"metric_name"`
Label string `yaml:"label"`
Expand Down
12 changes: 6 additions & 6 deletions managed/services/telemetry/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ datasources:
enabled: true
timeout: 2s
db_file: /srv/grafana/grafana.db
ENV_VARS:
enabled: true
reporting:
send: true
Expand All @@ -71,12 +73,7 @@ reporting:
RetryCount: 2,
SendTimeout: time.Second * 10,
},
DataSources: struct {
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
}{
DataSources: DataSources{
VM: &DataSourceVictoriaMetrics{
Enabled: true,
Timeout: time.Second * 2,
Expand All @@ -103,6 +100,9 @@ reporting:
Timeout: time.Second * 2,
DBFile: "/srv/grafana/grafana.db",
},
EnvVars: &DSConfigEnvVars{
Enabled: true,
},
},
}
assert.Equal(t, actual, expected)
Expand Down
84 changes: 84 additions & 0 deletions managed/services/telemetry/datasource_envvars.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright (C) 2023 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

// Package telemetry provides telemetry functionality.
package telemetry

import (
"context"
"os"

pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
"github.com/sirupsen/logrus"
)

type dsEnvvars struct {
l *logrus.Entry
config DSConfigEnvVars
}

// check interfaces.
var (
_ DataSource = (*dsEnvvars)(nil)
)

// NewDataSourceEnvVars makes a new data source for collecting envvars.
func NewDataSourceEnvVars(config DSConfigEnvVars, l *logrus.Entry) DataSource {
return &dsEnvvars{
l: l,
config: config,
}
}

// Enabled flag that determines if data source is enabled.
func (d *dsEnvvars) Enabled() bool {
return d.config.Enabled
}

func (d *dsEnvvars) Init(_ context.Context) error {
return nil
}

func (d *dsEnvvars) FetchMetrics(_ context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
var metrics []*pmmv1.ServerMetric_Metric

check := make(map[string]bool, len(config.Data))

for _, col := range config.Data {
if col.Column == "" {
d.l.Warnf("no column defined or empty column name in config %s", config.ID)
continue
}
if value, ok := os.LookupEnv(col.Column); ok && value != "" {
if _, alreadyHasItem := check[col.MetricName]; alreadyHasItem {
d.l.Warnf("repeated metric key %s found in config %s, the last will win", col.MetricName, config.ID)
continue
}

check[col.MetricName] = true

metrics = append(metrics, &pmmv1.ServerMetric_Metric{
Key: col.MetricName,
Value: value,
})
}
}

return metrics, nil
}

func (d *dsEnvvars) Dispose(_ context.Context) error {
return nil
}
Loading

0 comments on commit 7351ea5

Please sign in to comment.