PMM-9374 external victoria metrics (#1916)

* PMM-9374 support of external victoria metrics.

* PMM-9374 update version number of victoria metrics.

* PMM-9374 vmalert.

* PMM-9374 use pmm-clients vmagent instead of a separate one.

* PMM-9374 Fix linters.

* PMM-9374 Fix tests.

* PMM-9374 Improve work with VM Proxy.

* PMM-9374 Fix tests.

* PMM-9374 Fix tests.

* PMM-9374 Fix tests.

* PMM-9374 Fix linters.

* PMM-9374 Add new tests.

* PMM-9374 use interfaces instead of real object.

* PMM-9374 Fix the test.

* PMM-9374 Fix the linter.

* PMM-9374 Fix the linter.

* Update pmm_config.go

* Update pmm-db_disabled.ini

* PMM-9374 fix supervisord for external VM.

* PMM-9374 Fix linters.

* PMM-9374 Don't run victoria metrics if it's external.

docker-compose.yml

@@ -100,20 +100,45 @@ services:
     #     /opt/entrypoint.sh
     #   "
 
-  # PMM with external ClickHouse DB
+  # PMM with external DBs
   ch:
     profiles:
-      - pmm-ch
+      - pmm-external-dbs
     image: ${CH_IMAGE:-clickhouse/clickhouse-server:22.6.9.11-alpine}
     platform: linux/amd64
     hostname: ${CH_HOSTNAME:-ch}
     ports:
       - ${CH_PORT:-9000}:9000
-  pmm-server-ch:
+    networks:
+      - ${NETWORK:-default}
+  victoriametrics:
+    profiles:
+      - pmm-external-dbs
+    hostname: ${VM_HOSTNAME:-victoriametrics}
+    image: victoriametrics/victoria-metrics:v1.88.1
+    ports:
+      - 8428:8428
+      - 8089:8089
+      - 8089:8089/udp
+      - 2003:2003
+      - 2003:2003/udp
+      - 4242:4242
+    volumes:
+      - vmdata:/storage
+    command:
+      - "--storageDataPath=/storage"
+      - "--graphiteListenAddr=:2003"
+      - "--opentsdbListenAddr=:4242"
+      - "--httpListenAddr=:8428"
+      - "--influxListenAddr=:8089"
+    networks:
+      - ${NETWORK:-default}
+  pmm-managed-server-ch:
     profiles:
-      - pmm-ch
+      - pmm-external-dbs
     depends_on:
       - ch
+      - victoriametrics
     image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container}
     container_name: pmm-server
     hostname: pmm-server
@@ -136,7 +161,8 @@ services:
       - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm
       - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000
       - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2
-      # - PMM_DEBUG=1
+      - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/}
+      - PMM_DEBUG=1
       - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest
 
     extra_hosts:
@@ -161,13 +187,6 @@ services:
       - ${PMM_PORT_HTTPS:-443}:443
       # For headless delve
       - ${PMM_PORT_DELVE:-2345}:2345
-      # PG
-      - ${PMM_PORT_PG:-15432}:5432
-      # VM
-      - ${PMM_PORT_VM:-9090}:9090
-      # CH
-      - ${PMM_PORT_CH_TCP:-11000}:9000
-      - ${PMM_PORT_CH_HTTP:-11123}:8123
     volumes:
       - ./:/root/go/src/github.com/percona/pmm
       #  - "../grafana/public:/usr/share/grafana/public"
@@ -207,4 +226,5 @@ services:
 
 volumes:
   go-modules:
+  vmdata: {}
   root-cache:

managed/cmd/pmm-managed/main.go

@@ -680,9 +680,9 @@ func main() { //nolint:cyclop,maintidx
 	kingpin.Version(version.FullInfo())
 	kingpin.HelpFlag.Short('h')
 
-	victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL").
-		Default("http://127.0.0.1:9090/prometheus/").String()
-	victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL").
+	victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL").Envar("PMM_VM_URL").
+		Default(models.VMBaseURL).String()
+	victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL").Envar("PMM_VM_ALERT_URL").
 		Default("http://127.0.0.1:8880/").String()
 	victoriaMetricsConfigF := kingpin.Flag("victoriametrics-config", "VictoriaMetrics scrape configuration file path").
 		Default("/etc/victoriametrics-promscrape.yml").String()
@@ -762,6 +762,7 @@ func main() { //nolint:cyclop,maintidx
 		*postgresSSLModeF = models.VerifyCaSSLMode
 	}
 	ds := cfg.Config.Services.Telemetry.DataSources
+
 	pmmdb := ds.PmmDBSelect
 	pmmdb.Credentials.Username = *postgresDBUsernameF
 	pmmdb.Credentials.Password = *postgresDBPasswordF
@@ -782,6 +783,15 @@ func main() { //nolint:cyclop,maintidx
 	qanDB := ds.QanDBSelect
 	qanDB.DSN = clickhouseDSN
 
+	ds.VM.Address = *victoriaMetricsURLF
+
+	vmParams, err := models.NewVictoriaMetricsParams(
+		models.BasePrometheusConfigPath,
+		*victoriaMetricsURLF)
+	if err != nil {
+		l.Panicf("cannot load victoriametrics params problem: %+v", err)
+	}
+
 	setupParams := models.SetupDBParams{
 		Address:     *postgresAddrF,
 		Name:        *postgresDBNameF,
@@ -814,12 +824,7 @@ func main() { //nolint:cyclop,maintidx
 
 	cleaner := clean.New(db)
 	externalRules := vmalert.NewExternalRules()
-
-	vmParams, err := models.NewVictoriaMetricsParams(victoriametrics.BasePrometheusConfigPath)
-	if err != nil {
-		l.Panicf("cannot load victoriametrics params problem: %+v", err)
-	}
-	vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, *victoriaMetricsURLF, vmParams)
+	vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, vmParams)
 	if err != nil {
 		l.Panicf("VictoriaMetrics service problem: %+v", err)
 	}
@@ -833,7 +838,7 @@ func main() { //nolint:cyclop,maintidx
 
 	qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF)
 
-	agentsRegistry := agents.NewRegistry(db)
+	agentsRegistry := agents.NewRegistry(db, vmParams)
 	pbmPITRService := backup.NewPBMPITRService()
 	backupRemovalService := backup.NewRemovalService(db, pbmPITRService)
 	backupRetentionService := backup.NewRetentionService(db, backupRemovalService)
@@ -852,7 +857,7 @@ func main() { //nolint:cyclop,maintidx
 
 	pmmUpdateCheck := supervisord.NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker"))
 
-	logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck)
+	logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck, vmParams)
 
 	supervisord := supervisord.New(
 		*supervisordConfigDirF,
@@ -897,7 +902,7 @@ func main() { //nolint:cyclop,maintidx
 	prom.MustRegister(grafanaClient)
 
 	jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService)
-	agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb)
+	agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams)
 	agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService)
 
 	actionsService := agents.NewActionsService(qanClient, agentsRegistry)

managed/models/victoriametrics.go → managed/models/victoriametrics_params.go

@@ -16,25 +16,45 @@
 package models
 
 import (
+	"net/url"
 	"os"
+	"strings"
 
 	config "github.com/percona/promconfig"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	// BasePrometheusConfigPath - basic path with prometheus config,
+	// that user can mount to container.
+	BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml"
+	VMBaseURL                = "http://127.0.0.1:9090/prometheus/"
+)
+
 // VictoriaMetricsParams - defines flags and settings for victoriametrics.
 type VictoriaMetricsParams struct {
 	// VMAlertFlags additional flags for VMAlert.
 	VMAlertFlags []string
 	// BaseConfigPath defines path for basic prometheus config.
 	BaseConfigPath string
+	// url defines url of Victoria Metrics
+	url *url.URL
 }
 
 // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics.
-func NewVictoriaMetricsParams(basePath string) (*VictoriaMetricsParams, error) {
+func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) {
+	if !strings.HasSuffix(vmURL, "/") {
+		vmURL += "/"
+	}
+
+	URL, err := url.Parse(vmURL)
+	if err != nil {
+		return nil, err
+	}
 	vmp := &VictoriaMetricsParams{
 		BaseConfigPath: basePath,
+		url:            URL,
 	}
 	if err := vmp.UpdateParams(); err != nil {
 		return vmp, err
@@ -59,7 +79,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error {
 		if !os.IsNotExist(err) {
 			return errors.Wrap(err, "cannot read baseConfigPath for VMAlertParams")
 		}
-		// fast return if users configuration doesn't exists with path
+		// fast return if users configuration doesn't exist with path
 		// /srv/prometheus/prometheus.base.yml,
 		// its maybe mounted into container by user.
 		return nil
@@ -79,3 +99,18 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error {
 
 	return nil
 }
+
+func (vmp *VictoriaMetricsParams) ExternalVM() bool {
+	return vmp.url.Hostname() != "127.0.0.1"
+}
+
+func (vmp *VictoriaMetricsParams) URL() string {
+	return vmp.url.String()
+}
+
+func (vmp *VictoriaMetricsParams) URLFor(path string) (*url.URL, error) {
+	if path == "" {
+		return vmp.url, nil
+	}
+	return vmp.url.Parse(path)
+}

managed/models/victoriametrics_test.go → managed/models/victoriametrics_params_test.go

@@ -18,17 +18,48 @@ package models
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestVictoriaMetricsParams(t *testing.T) {
 	t.Run("read non exist baseConfigFile", func(t *testing.T) {
-		_, err := NewVictoriaMetricsParams("nonExistConfigFile.yml")
+		_, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", VMBaseURL)
 		require.NoError(t, err)
 	})
 	t.Run("check params for VMAlert", func(t *testing.T) {
-		vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml")
+		vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", VMBaseURL)
 		require.NoError(t, err)
 		require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags)
 	})
+	t.Run("check external VM", func(t *testing.T) {
+		tests := []struct {
+			url  string
+			want bool
+		}{
+			{
+				"http://127.0.0.1:9090/prometheus",
+				false,
+			},
+			{
+				"http://127.0.0.1:9090/prometheus/",
+				false,
+			},
+			{
+				"http://victoriametrics:8428/",
+				true,
+			},
+			{
+				"https://example.com:9090/",
+				true,
+			},
+		}
+		for _, tt := range tests {
+			t.Run(tt.url, func(t *testing.T) {
+				vmp, err := NewVictoriaMetricsParams(BasePrometheusConfigPath, tt.url)
+				require.NoError(t, err)
+				assert.Equalf(t, tt.want, vmp.ExternalVM(), "ExternalVM()")
+			})
+		}
+	})
 }

managed/services/agents/deps.go

@@ -17,6 +17,7 @@ package agents
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/sirupsen/logrus"
 
@@ -54,3 +55,11 @@ type jobsService interface {
 	handleJobResult(ctx context.Context, l *logrus.Entry, result *agentpb.JobResult)
 	handleJobProgress(ctx context.Context, progress *agentpb.JobProgress)
 }
+
+// victoriaMetricsParams is a subset of methods of models.VMParams used by this package.
+// We use it instead of real type to avoid dependency cycle.
+type victoriaMetricsParams interface {
+	ExternalVM() bool
+	URLFor(path string) (*url.URL, error)
+	URL() string
+}

managed/services/agents/registry.go

@@ -85,10 +85,12 @@ type Registry struct {
 	mRoundTrip   prom.Summary
 	mClockDrift  prom.Summary
 	mAgents      prom.GaugeFunc
+
+	isExternalVM bool
 }
 
 // NewRegistry creates a new registry with given database connection.
-func NewRegistry(db *reform.DB) *Registry {
+func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry {
 	agents := make(map[string]*pmmAgentInfo)
 	r := &Registry{
 		db: db,
@@ -123,6 +125,8 @@ func NewRegistry(db *reform.DB) *Registry {
 			Help:       "Clock drift.",
 			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 		}),
+
+		isExternalVM: externalVMChecker.ExternalVM(),
 	}
 
 	r.mAgents = prom.NewGaugeFunc(prom.GaugeOpts{
@@ -160,7 +164,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo,
 	}
 	var node *models.Node
 	err = r.db.InTransaction(func(tx *reform.TX) error {
-		node, err = authenticate(agentMD, tx.Querier)
+		node, err = r.authenticate(agentMD, tx.Querier)
 		if err != nil {
 			return err
 		}
@@ -204,7 +208,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo,
 	return agent, nil
 }
 
-func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) {
+func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) {
 	if md.ID == "" {
 		return nil, status.Error(codes.PermissionDenied, "Empty Agent ID.")
 	}
@@ -233,7 +237,7 @@ func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.
 		return nil, status.Errorf(codes.InvalidArgument, "Can't parse 'version' for pmm-agent with ID %q.", md.ID)
 	}
 
-	if err := addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil {
+	if err := r.addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil {
 		return nil, err
 	}
 
@@ -296,18 +300,18 @@ func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error {
 // addOrRemoveVMAgent - creates vmAgent agentType if pmm-agent's version supports it and agent not exists yet,
 // otherwise ensures that vmAgent not exist for pmm-agent and pmm-agent's agents don't have push_metrics mode,
 // removes it if needed.
-func addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error {
+func (r *Registry) addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error {
 	if pmmAgentVersion.Less(models.PMMAgentWithPushMetricsSupport) {
 		// ensure that vmagent not exists and agents dont have push_metrics.
 		return removeVMAgentFromPMMAgent(q, pmmAgentID)
 	}
-	return addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID)
+	return r.addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID)
 }
 
-func addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error {
+func (r *Registry) addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error {
 	// TODO remove it after fix
 	// https://jira.percona.com/browse/PMM-4420
-	if runsOnNodeID == "pmm-server" {
+	if runsOnNodeID == "pmm-server" && !r.isExternalVM {
 		return nil
 	}
 	vmAgentType := models.VMAgentType