From ffc66f09b0af07e8f41ef52b9e1b4618453438a1 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Mon, 18 Sep 2023 13:01:55 +0300 Subject: [PATCH] PMM-9374 external victoria metrics (#1916) * PMM-9374 support of external victoria metrics. * PMM-9374 update version number of victoria metrics. * PMM-9374 vmalert. * PMM-9374 use pmm-clients vmagent instead of a separate one. * PMM-9374 Fix linters. * PMM-9374 Fix tests. * PMM-9374 Improve work with VM Proxy. * PMM-9374 Fix tests. * PMM-9374 Fix tests. * PMM-9374 Fix tests. * PMM-9374 Fix linters. * PMM-9374 Add new tests. * PMM-9374 use interfaces instead of real object. * PMM-9374 Fix the test. * PMM-9374 Fix the linter. * PMM-9374 Fix the linter. * Update pmm_config.go * Update pmm-db_disabled.ini * PMM-9374 fix supervisord for external VM. * PMM-9374 Fix linters. * PMM-9374 Don't run victoria metrics if it's external. --- docker-compose.yml | 44 ++++++++--- managed/cmd/pmm-managed/main.go | 29 +++++--- ...iametrics.go => victoriametrics_params.go} | 39 +++++++++- ...test.go => victoriametrics_params_test.go} | 35 ++++++++- managed/services/agents/deps.go | 9 +++ managed/services/agents/registry.go | 20 +++-- managed/services/agents/state.go | 18 +++-- managed/services/agents/vmagent.go | 25 +++++-- managed/services/agents/vmagent_test.go | 30 +++++++- managed/services/config/pmm-managed.yaml | 1 - managed/services/supervisord/deps.go | 25 +++++++ .../services/supervisord/devcontainer_test.go | 4 +- managed/services/supervisord/logs.go | 14 +++- managed/services/supervisord/logs_test.go | 9 ++- managed/services/supervisord/pmm_config.go | 1 - managed/services/supervisord/supervisord.go | 37 ++++------ .../services/supervisord/supervisord_test.go | 6 +- .../victoriametrics/victoriametrics.go | 73 ++++++++++++++----- .../victoriametrics/victoriametrics_test.go | 7 +- .../supervisord.d/pmm-db_disabled.ini | 1 - .../testdata/supervisord.d/pmm-db_enabled.ini | 1 - managed/testdata/supervisord.d/vmalert.ini | 8 +- managed/testdata/supervisord.d/vmproxy.ini | 2 +- managed/utils/envvars/parser.go | 12 +++ .../playbook/tasks/files/datasources.yml | 2 +- vmproxy/Makefile | 2 +- vmproxy/proxy/proxy.go | 7 ++ 27 files changed, 349 insertions(+), 112 deletions(-) rename managed/models/{victoriametrics.go => victoriametrics_params.go} (72%) rename managed/models/{victoriametrics_test.go => victoriametrics_params_test.go} (62%) create mode 100644 managed/services/supervisord/deps.go diff --git a/docker-compose.yml b/docker-compose.yml index d516be2235..566c7495d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,20 +100,45 @@ services: # /opt/entrypoint.sh # " - # PMM with external ClickHouse DB + # PMM with external DBs ch: profiles: - - pmm-ch + - pmm-external-dbs image: ${CH_IMAGE:-clickhouse/clickhouse-server:22.6.9.11-alpine} platform: linux/amd64 hostname: ${CH_HOSTNAME:-ch} ports: - ${CH_PORT:-9000}:9000 - pmm-server-ch: + networks: + - ${NETWORK:-default} + victoriametrics: + profiles: + - pmm-external-dbs + hostname: ${VM_HOSTNAME:-victoriametrics} + image: victoriametrics/victoria-metrics:v1.88.1 + ports: + - 8428:8428 + - 8089:8089 + - 8089:8089/udp + - 2003:2003 + - 2003:2003/udp + - 4242:4242 + volumes: + - vmdata:/storage + command: + - "--storageDataPath=/storage" + - "--graphiteListenAddr=:2003" + - "--opentsdbListenAddr=:4242" + - "--httpListenAddr=:8428" + - "--influxListenAddr=:8089" + networks: + - ${NETWORK:-default} + pmm-managed-server-ch: profiles: - - pmm-ch + - pmm-external-dbs depends_on: - ch + - victoriametrics image: ${PMM_CONTAINER:-perconalab/pmm-server:dev-container} container_name: pmm-server hostname: pmm-server @@ -136,7 +161,8 @@ services: - PERCONA_TEST_PMM_CLICKHOUSE_DATABASE=pmm - PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE=10000 - PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE=2 - # - PMM_DEBUG=1 + - PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/} + - PMM_DEBUG=1 - PERCONA_TEST_DBAAS_PMM_CLIENT=perconalab/pmm-client:dev-latest extra_hosts: @@ -161,13 +187,6 @@ services: - ${PMM_PORT_HTTPS:-443}:443 # For headless delve - ${PMM_PORT_DELVE:-2345}:2345 - # PG - - ${PMM_PORT_PG:-15432}:5432 - # VM - - ${PMM_PORT_VM:-9090}:9090 - # CH - - ${PMM_PORT_CH_TCP:-11000}:9000 - - ${PMM_PORT_CH_HTTP:-11123}:8123 volumes: - ./:/root/go/src/github.com/percona/pmm # - "../grafana/public:/usr/share/grafana/public" @@ -207,4 +226,5 @@ services: volumes: go-modules: + vmdata: {} root-cache: diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index cd203b2e31..c3d7bf5e86 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -680,9 +680,9 @@ func main() { //nolint:cyclop,maintidx kingpin.Version(version.FullInfo()) kingpin.HelpFlag.Short('h') - victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL"). - Default("http://127.0.0.1:9090/prometheus/").String() - victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL"). + victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL").Envar("PMM_VM_URL"). + Default(models.VMBaseURL).String() + victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL").Envar("PMM_VM_ALERT_URL"). Default("http://127.0.0.1:8880/").String() victoriaMetricsConfigF := kingpin.Flag("victoriametrics-config", "VictoriaMetrics scrape configuration file path"). Default("/etc/victoriametrics-promscrape.yml").String() @@ -762,6 +762,7 @@ func main() { //nolint:cyclop,maintidx *postgresSSLModeF = models.VerifyCaSSLMode } ds := cfg.Config.Services.Telemetry.DataSources + pmmdb := ds.PmmDBSelect pmmdb.Credentials.Username = *postgresDBUsernameF pmmdb.Credentials.Password = *postgresDBPasswordF @@ -782,6 +783,15 @@ func main() { //nolint:cyclop,maintidx qanDB := ds.QanDBSelect qanDB.DSN = clickhouseDSN + ds.VM.Address = *victoriaMetricsURLF + + vmParams, err := models.NewVictoriaMetricsParams( + models.BasePrometheusConfigPath, + *victoriaMetricsURLF) + if err != nil { + l.Panicf("cannot load victoriametrics params problem: %+v", err) + } + setupParams := models.SetupDBParams{ Address: *postgresAddrF, Name: *postgresDBNameF, @@ -814,12 +824,7 @@ func main() { //nolint:cyclop,maintidx cleaner := clean.New(db) externalRules := vmalert.NewExternalRules() - - vmParams, err := models.NewVictoriaMetricsParams(victoriametrics.BasePrometheusConfigPath) - if err != nil { - l.Panicf("cannot load victoriametrics params problem: %+v", err) - } - vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, *victoriaMetricsURLF, vmParams) + vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, vmParams) if err != nil { l.Panicf("VictoriaMetrics service problem: %+v", err) } @@ -833,7 +838,7 @@ func main() { //nolint:cyclop,maintidx qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) - agentsRegistry := agents.NewRegistry(db) + agentsRegistry := agents.NewRegistry(db, vmParams) pbmPITRService := backup.NewPBMPITRService() backupRemovalService := backup.NewRemovalService(db, pbmPITRService) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) @@ -852,7 +857,7 @@ func main() { //nolint:cyclop,maintidx pmmUpdateCheck := supervisord.NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker")) - logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck) + logs := supervisord.NewLogs(version.FullInfo(), pmmUpdateCheck, vmParams) supervisord := supervisord.New( *supervisordConfigDirF, @@ -897,7 +902,7 @@ func main() { //nolint:cyclop,maintidx prom.MustRegister(grafanaClient) jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService) - agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb) + agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams) agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) actionsService := agents.NewActionsService(qanClient, agentsRegistry) diff --git a/managed/models/victoriametrics.go b/managed/models/victoriametrics_params.go similarity index 72% rename from managed/models/victoriametrics.go rename to managed/models/victoriametrics_params.go index d8b07caf27..f033b5a4f4 100644 --- a/managed/models/victoriametrics.go +++ b/managed/models/victoriametrics_params.go @@ -16,25 +16,45 @@ package models import ( + "net/url" "os" + "strings" config "github.com/percona/promconfig" "github.com/pkg/errors" "gopkg.in/yaml.v3" ) +const ( + // BasePrometheusConfigPath - basic path with prometheus config, + // that user can mount to container. + BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml" + VMBaseURL = "http://127.0.0.1:9090/prometheus/" +) + // VictoriaMetricsParams - defines flags and settings for victoriametrics. type VictoriaMetricsParams struct { // VMAlertFlags additional flags for VMAlert. VMAlertFlags []string // BaseConfigPath defines path for basic prometheus config. BaseConfigPath string + // url defines url of Victoria Metrics + url *url.URL } // NewVictoriaMetricsParams - returns configuration params for VictoriaMetrics. -func NewVictoriaMetricsParams(basePath string) (*VictoriaMetricsParams, error) { +func NewVictoriaMetricsParams(basePath string, vmURL string) (*VictoriaMetricsParams, error) { + if !strings.HasSuffix(vmURL, "/") { + vmURL += "/" + } + + URL, err := url.Parse(vmURL) + if err != nil { + return nil, err + } vmp := &VictoriaMetricsParams{ BaseConfigPath: basePath, + url: URL, } if err := vmp.UpdateParams(); err != nil { return vmp, err @@ -59,7 +79,7 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { if !os.IsNotExist(err) { return errors.Wrap(err, "cannot read baseConfigPath for VMAlertParams") } - // fast return if users configuration doesn't exists with path + // fast return if users configuration doesn't exist with path // /srv/prometheus/prometheus.base.yml, // its maybe mounted into container by user. return nil @@ -79,3 +99,18 @@ func (vmp *VictoriaMetricsParams) loadVMAlertParams() error { return nil } + +func (vmp *VictoriaMetricsParams) ExternalVM() bool { + return vmp.url.Hostname() != "127.0.0.1" +} + +func (vmp *VictoriaMetricsParams) URL() string { + return vmp.url.String() +} + +func (vmp *VictoriaMetricsParams) URLFor(path string) (*url.URL, error) { + if path == "" { + return vmp.url, nil + } + return vmp.url.Parse(path) +} diff --git a/managed/models/victoriametrics_test.go b/managed/models/victoriametrics_params_test.go similarity index 62% rename from managed/models/victoriametrics_test.go rename to managed/models/victoriametrics_params_test.go index 02ab59b036..b0f55796f6 100644 --- a/managed/models/victoriametrics_test.go +++ b/managed/models/victoriametrics_params_test.go @@ -18,17 +18,48 @@ package models import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestVictoriaMetricsParams(t *testing.T) { t.Run("read non exist baseConfigFile", func(t *testing.T) { - _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml") + _, err := NewVictoriaMetricsParams("nonExistConfigFile.yml", VMBaseURL) require.NoError(t, err) }) t.Run("check params for VMAlert", func(t *testing.T) { - vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml") + vmp, err := NewVictoriaMetricsParams("../testdata/victoriametrics/prometheus.external.alerts.yml", VMBaseURL) require.NoError(t, err) require.Equal(t, []string{"--rule=/srv/external_rules/rul1.yml", "--rule=/srv/external_rules/rule2.yml", "--evaluationInterval=10s"}, vmp.VMAlertFlags) }) + t.Run("check external VM", func(t *testing.T) { + tests := []struct { + url string + want bool + }{ + { + "http://127.0.0.1:9090/prometheus", + false, + }, + { + "http://127.0.0.1:9090/prometheus/", + false, + }, + { + "http://victoriametrics:8428/", + true, + }, + { + "https://example.com:9090/", + true, + }, + } + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + vmp, err := NewVictoriaMetricsParams(BasePrometheusConfigPath, tt.url) + require.NoError(t, err) + assert.Equalf(t, tt.want, vmp.ExternalVM(), "ExternalVM()") + }) + } + }) } diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go index 0a321d944d..396517c7d1 100644 --- a/managed/services/agents/deps.go +++ b/managed/services/agents/deps.go @@ -17,6 +17,7 @@ package agents import ( "context" + "net/url" "github.com/sirupsen/logrus" @@ -54,3 +55,11 @@ type jobsService interface { handleJobResult(ctx context.Context, l *logrus.Entry, result *agentpb.JobResult) handleJobProgress(ctx context.Context, progress *agentpb.JobProgress) } + +// victoriaMetricsParams is a subset of methods of models.VMParams used by this package. +// We use it instead of real type to avoid dependency cycle. +type victoriaMetricsParams interface { + ExternalVM() bool + URLFor(path string) (*url.URL, error) + URL() string +} diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go index 3745044378..d786d2474f 100644 --- a/managed/services/agents/registry.go +++ b/managed/services/agents/registry.go @@ -85,10 +85,12 @@ type Registry struct { mRoundTrip prom.Summary mClockDrift prom.Summary mAgents prom.GaugeFunc + + isExternalVM bool } // NewRegistry creates a new registry with given database connection. -func NewRegistry(db *reform.DB) *Registry { +func NewRegistry(db *reform.DB, externalVMChecker victoriaMetricsParams) *Registry { agents := make(map[string]*pmmAgentInfo) r := &Registry{ db: db, @@ -123,6 +125,8 @@ func NewRegistry(db *reform.DB) *Registry { Help: "Clock drift.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }), + + isExternalVM: externalVMChecker.ExternalVM(), } r.mAgents = prom.NewGaugeFunc(prom.GaugeOpts{ @@ -160,7 +164,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, } var node *models.Node err = r.db.InTransaction(func(tx *reform.TX) error { - node, err = authenticate(agentMD, tx.Querier) + node, err = r.authenticate(agentMD, tx.Querier) if err != nil { return err } @@ -204,7 +208,7 @@ func (r *Registry) register(stream agentpb.Agent_ConnectServer) (*pmmAgentInfo, return agent, nil } -func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) { +func (r *Registry) authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models.Node, error) { if md.ID == "" { return nil, status.Error(codes.PermissionDenied, "Empty Agent ID.") } @@ -233,7 +237,7 @@ func authenticate(md *agentpb.AgentConnectMetadata, q *reform.Querier) (*models. return nil, status.Errorf(codes.InvalidArgument, "Can't parse 'version' for pmm-agent with ID %q.", md.ID) } - if err := addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil { + if err := r.addOrRemoveVMAgent(q, md.ID, runsOnNodeID, agentVersion); err != nil { return nil, err } @@ -296,18 +300,18 @@ func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error { // addOrRemoveVMAgent - creates vmAgent agentType if pmm-agent's version supports it and agent not exists yet, // otherwise ensures that vmAgent not exist for pmm-agent and pmm-agent's agents don't have push_metrics mode, // removes it if needed. -func addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error { +func (r *Registry) addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string, pmmAgentVersion *version.Parsed) error { if pmmAgentVersion.Less(models.PMMAgentWithPushMetricsSupport) { // ensure that vmagent not exists and agents dont have push_metrics. return removeVMAgentFromPMMAgent(q, pmmAgentID) } - return addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID) + return r.addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID) } -func addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error { +func (r *Registry) addVMAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error { // TODO remove it after fix // https://jira.percona.com/browse/PMM-4420 - if runsOnNodeID == "pmm-server" { + if runsOnNodeID == "pmm-server" && !r.isExternalVM { return nil } vmAgentType := models.VMAgentType diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go index ac49fc5768..96f8f0562c 100644 --- a/managed/services/agents/state.go +++ b/managed/services/agents/state.go @@ -40,17 +40,19 @@ const ( // StateUpdater handles updating status of agents. type StateUpdater struct { - db *reform.DB - r *Registry - vmdb prometheusService + db *reform.DB + r *Registry + vmdb prometheusService + vmParams victoriaMetricsParams } // NewStateUpdater creates new agent state updater. -func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService) *StateUpdater { +func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParams victoriaMetricsParams) *StateUpdater { return &StateUpdater{ - db: db, - r: r, - vmdb: vmdb, + db: db, + r: r, + vmdb: vmdb, + vmParams: vmParams, } } @@ -180,7 +182,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI if err != nil { return errors.Wrapf(err, "cannot get agent scrape config for agent: %s", agent.id) } - agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg)) + agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg), u.vmParams) case models.NodeExporterType: node, err := models.FindNodeByID(u.db.Querier, pointer.GetString(row.NodeID)) diff --git a/managed/services/agents/vmagent.go b/managed/services/agents/vmagent.go index f12d345d36..b5586df710 100644 --- a/managed/services/agents/vmagent.go +++ b/managed/services/agents/vmagent.go @@ -16,8 +16,10 @@ package agents import ( + "fmt" "os" "sort" + "strings" "github.com/percona/pmm/api/agentpb" "github.com/percona/pmm/api/inventorypb" @@ -30,7 +32,11 @@ var ( ) // vmAgentConfig returns desired configuration of vmagent process. -func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { +func vmAgentConfig(scrapeCfg string, params victoriaMetricsParams) *agentpb.SetStateRequest_AgentProcess { + serverURL := "{{.server_url}}/victoriametrics/" + if params.ExternalVM() { + serverURL = params.URL() + } maxScrapeSize := maxScrapeSizeDefault if space := os.Getenv(maxScrapeSizeEnv); space != "" { maxScrapeSize = space @@ -39,7 +45,7 @@ func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { interfaceToBind := envvars.GetInterfaceToBind() args := []string{ - "-remoteWrite.url={{.server_url}}/victoriametrics/api/v1/write", + fmt.Sprintf("-remoteWrite.url=%sapi/v1/write", serverURL), "-remoteWrite.tlsInsecureSkipVerify={{.server_insecure}}", "-remoteWrite.tmpDataPath={{.tmp_dir}}/vmagent-temp-dir", "-promscrape.config={{.TextFiles.vmagentscrapecfg}}", @@ -50,13 +56,22 @@ func vmAgentConfig(scrapeCfg string) *agentpb.SetStateRequest_AgentProcess { "-httpListenAddr=" + interfaceToBind + ":{{.listen_port}}", // needed for login/password at client side. "-envflag.enable=true", + "-envflag.prefix=VMAGENT_", } sort.Strings(args) - envs := []string{ - "remoteWrite_basicAuth_username={{.server_username}}", - "remoteWrite_basicAuth_password={{.server_password}}", + var envs []string + if !params.ExternalVM() { + envs = []string{ + "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}", + "VMAGENT_remoteWrite_basicAuth_password={{.server_password}}", + } + } + for _, env := range os.Environ() { + if strings.HasPrefix(env, envvars.ENVvmAgentPrefix) { + envs = append(envs, env) + } } sort.Strings(envs) diff --git a/managed/services/agents/vmagent_test.go b/managed/services/agents/vmagent_test.go index 2adc5fded8..43402e56ad 100644 --- a/managed/services/agents/vmagent_test.go +++ b/managed/services/agents/vmagent_test.go @@ -19,17 +19,43 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/percona/pmm/managed/models" ) func TestMaxScrapeSize(t *testing.T) { t.Run("by default 64MiB", func(t *testing.T) { - actual := vmAgentConfig("") + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+maxScrapeSizeDefault) }) t.Run("overridden with ENV", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) newValue := "16MiB" t.Setenv(maxScrapeSizeEnv, newValue) - actual := vmAgentConfig("") + actual := vmAgentConfig("", params) assert.Contains(t, actual.Args, "-promscrape.maxScrapeSize="+newValue) }) + t.Run("VMAGENT_ ENV variables", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + t.Setenv("VMAGENT_promscrape_maxScrapeSize", "16MiB") + t.Setenv("VM_remoteWrite_basicAuth_password", "password") + actual := vmAgentConfig("", params) + assert.Contains(t, actual.Env, "VMAGENT_promscrape_maxScrapeSize=16MiB") + assert.Contains(t, actual.Env, "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}") + assert.NotContains(t, actual.Env, "VM_remoteWrite_basicAuth_password=password") + }) + t.Run("External Victoria Metrics ENV variables", func(t *testing.T) { + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, "http://victoriametrics:8428") + require.NoError(t, err) + t.Setenv("VMAGENT_promscrape_maxScrapeSize", "16MiB") + actual := vmAgentConfig("", params) + assert.Contains(t, actual.Args, "-remoteWrite.url=http://victoriametrics:8428/api/v1/write") + assert.Contains(t, actual.Env, "VMAGENT_promscrape_maxScrapeSize=16MiB") + assert.NotContains(t, actual.Env, "VMAGENT_remoteWrite_basicAuth_username={{.server_username}}") + }) } diff --git a/managed/services/config/pmm-managed.yaml b/managed/services/config/pmm-managed.yaml index af40bfc254..d9341b020f 100644 --- a/managed/services/config/pmm-managed.yaml +++ b/managed/services/config/pmm-managed.yaml @@ -6,7 +6,6 @@ services: VM: enabled: true timeout: 2s - address: http://localhost:9090/prometheus QANDB_SELECT: enabled: true timeout: 2s diff --git a/managed/services/supervisord/deps.go b/managed/services/supervisord/deps.go new file mode 100644 index 0000000000..8dea19e974 --- /dev/null +++ b/managed/services/supervisord/deps.go @@ -0,0 +1,25 @@ +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package supervisord + +import "net/url" + +// victoriaMetricsParams is a subset of methods of models.VMParams used by this package. +// We use it instead of real type to avoid dependency cycle. +type victoriaMetricsParams interface { + ExternalVM() bool + URLFor(path string) (*url.URL, error) +} diff --git a/managed/services/supervisord/devcontainer_test.go b/managed/services/supervisord/devcontainer_test.go index e3f0992f0a..0110cfb793 100644 --- a/managed/services/supervisord/devcontainer_test.go +++ b/managed/services/supervisord/devcontainer_test.go @@ -112,7 +112,9 @@ func TestDevContainer(t *testing.T) { t.Run("UpdateConfiguration", func(t *testing.T) { // logrus.SetLevel(logrus.DebugLevel) checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - vmParams := &models.VictoriaMetricsParams{} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + s := New("/etc/supervisord.d", checker, vmParams, models.PGParams{}, gRPCMessageMaxSize) require.NotEmpty(t, s.supervisorctlPath) diff --git a/managed/services/supervisord/logs.go b/managed/services/supervisord/logs.go index 7eb802fae3..7752a1b97f 100644 --- a/managed/services/supervisord/logs.go +++ b/managed/services/supervisord/logs.go @@ -59,14 +59,16 @@ type fileContent struct { type Logs struct { pmmVersion string pmmUpdateChecker *PMMUpdateChecker + vmParams victoriaMetricsParams } // NewLogs creates a new Logs service. // The number of last log lines to read is n. -func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs { +func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker, vmParams victoriaMetricsParams) *Logs { return &Logs{ pmmVersion: pmmVersion, pmmUpdateChecker: pmmUpdateChecker, + vmParams: vmParams, } } @@ -215,7 +217,7 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten }) // add VictoriaMetrics targets - b, err = readURL(ctx, "http://127.0.0.1:9090/prometheus/api/v1/targets") + b, err = l.victoriaMetricsTargets(ctx) files = append(files, fileContent{ Name: "victoriametrics_targets.json", Data: b, @@ -280,6 +282,14 @@ func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileConten return files } +func (l *Logs) victoriaMetricsTargets(ctx context.Context) ([]byte, error) { + targetsURL, err := l.vmParams.URLFor("api/v1/targets") + if err != nil { + return nil, err + } + return readURL(ctx, targetsURL.String()) +} + // readLog reads last lines (up to given number of lines and bytes) from given file, // and returns them together with modification time. func readLog(name string, maxLines int, maxBytes int64) ([]byte, time.Time, error) { diff --git a/managed/services/supervisord/logs_test.go b/managed/services/supervisord/logs_test.go index 118a42bac1..e7de37dae9 100644 --- a/managed/services/supervisord/logs_test.go +++ b/managed/services/supervisord/logs_test.go @@ -31,6 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/percona/pmm/managed/models" "github.com/percona/pmm/utils/logger" ) @@ -147,7 +148,9 @@ func TestAddAdminSummary(t *testing.T) { func TestFiles(t *testing.T) { checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - l := NewLogs("2.4.5", checker) + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) files := l.files(ctx, nil) @@ -190,7 +193,9 @@ func TestZip(t *testing.T) { t.Skip("FIXME") checker := NewPMMUpdateChecker(logrus.WithField("test", t.Name())) - l := NewLogs("2.4.5", checker) + params, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) + l := NewLogs("2.4.5", checker, params) ctx := logger.Set(context.Background(), t.Name()) var buf bytes.Buffer diff --git a/managed/services/supervisord/pmm_config.go b/managed/services/supervisord/pmm_config.go index 602b9a5546..581e6de282 100644 --- a/managed/services/supervisord/pmm_config.go +++ b/managed/services/supervisord/pmm_config.go @@ -166,7 +166,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 29490ff66d..7172bbe8fa 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -394,20 +394,10 @@ func (s *Service) UpdateLog(offset uint32) ([]string, uint32, error) { // reload asks supervisord to reload configuration. func (s *Service) reload(name string) error { - // See https://github.com/Supervisor/supervisor/issues/1264 for explanation - // why we do reread + stop/remove/add. - if _, err := s.supervisorctl("reread"); err != nil { s.l.Warn(err) } - if _, err := s.supervisorctl("stop", name); err != nil { - s.l.Warn(err) - } - if _, err := s.supervisorctl("remove", name); err != nil { - s.l.Warn(err) - } - - _, err := s.supervisorctl("add", name) + _, err := s.supervisorctl("update", name) return err } @@ -433,7 +423,10 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "DataRetentionDays": int(settings.DataRetention.Hours() / 24), "VMAlertFlags": s.vmParams.VMAlertFlags, "VMDBCacheDisable": !settings.VictoriaMetrics.CacheEnabled, + "VMURL": s.vmParams.URL(), + "ExternalVM": s.vmParams.ExternalVM(), "PerconaTestDbaas": settings.DBaaS.Enabled, + "InterfaceToBind": envvars.GetInterfaceToBind(), "ClickhouseAddr": clickhouseAddr, "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, "ClickhouseDatabase": clickhouseDatabase, @@ -614,8 +607,6 @@ func (s *Service) RestartSupervisedService(serviceName string) error { return err } -var interfaceToBind = envvars.GetInterfaceToBind() - //nolint:lll var templates = template.Must(template.New("").Option("missingkey=error").Parse(` {{define "dbaas-controller"}} @@ -652,6 +643,7 @@ redirect_stderr = true {{end}} {{define "victoriametrics"}} +{{- if not .ExternalVM }} [program:victoriametrics] priority = 7 command = @@ -659,7 +651,7 @@ command = --promscrape.config=/etc/victoriametrics-promscrape.yml --retentionPeriod={{ .DataRetentionDays }}d --storageDataPath=/srv/victoriametrics/data - --httpListenAddr=` + interfaceToBind + `:9090 + --httpListenAddr={{ .InterfaceToBind }}:9090 --search.disableCache={{ .VMDBCacheDisable }} --search.maxQueryLen=1MB --search.latencyOffset=5s @@ -684,6 +676,7 @@ stdout_logfile = /srv/logs/victoriametrics.log stdout_logfile_maxbytes = 10MB stdout_logfile_backups = 3 redirect_stderr = true +{{end -}} {{end}} {{define "vmalert"}} @@ -694,14 +687,14 @@ command = --notifier.url="{{ .AlertmanagerURL }}" --notifier.basicAuth.password='{{ .AlertManagerPassword }}' --notifier.basicAuth.username="{{ .AlertManagerUser }}" - --external.url=http://localhost:9090/prometheus - --datasource.url=http://127.0.0.1:9090/prometheus - --remoteRead.url=http://127.0.0.1:9090/prometheus + --external.url={{ .VMURL }} + --datasource.url={{ .VMURL }} + --remoteRead.url={{ .VMURL }} --remoteRead.ignoreRestoreErrors=false - --remoteWrite.url=http://127.0.0.1:9090/prometheus + --remoteWrite.url={{ .VMURL }} --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml - --httpListenAddr=` + interfaceToBind + `:8880 + --httpListenAddr={{ .InterfaceToBind }}:8880 {{- range $index, $param := .VMAlertFlags }} {{ $param }} {{- end }} @@ -723,9 +716,9 @@ redirect_stderr = true priority = 9 command = /usr/sbin/vmproxy - --target-url=http://127.0.0.1:9090/ + --target-url={{ .VMURL }} --listen-port=8430 - --listen-address=` + interfaceToBind + ` + --listen-address={{ .InterfaceToBind }} --header-name=X-Proxy-Filter user = pmm autorestart = true @@ -749,7 +742,7 @@ command = --storage.path=/srv/alertmanager/data --data.retention={{ .DataRetentionHours }}h --web.external-url=http://localhost:9093/alertmanager/ - --web.listen-address=` + interfaceToBind + `:9093 + --web.listen-address={{ .InterfaceToBind }}:9093 --cluster.listen-address="" user = pmm autorestart = true diff --git a/managed/services/supervisord/supervisord_test.go b/managed/services/supervisord/supervisord_test.go index ed6e515ff5..3263efa842 100644 --- a/managed/services/supervisord/supervisord_test.go +++ b/managed/services/supervisord/supervisord_test.go @@ -36,7 +36,8 @@ func TestConfig(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New(configDir, pmmUpdateCheck, vmParams, models.PGParams{}, gRPCMessageMaxSize) settings := &models.Settings{ DataRetention: 30 * 24 * time.Hour, @@ -68,7 +69,8 @@ func TestDBaaSController(t *testing.T) { pmmUpdateCheck := NewPMMUpdateChecker(logrus.WithField("component", "supervisord/pmm-update-checker_logs")) configDir := filepath.Join("..", "..", "testdata", "supervisord.d") - vmParams := &models.VictoriaMetricsParams{} + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + require.NoError(t, err) s := New(configDir, pmmUpdateCheck, vmParams, models.PGParams{}, gRPCMessageMaxSize) var tp *template.Template diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go index 13a468be87..7a70108d67 100644 --- a/managed/services/victoriametrics/victoriametrics.go +++ b/managed/services/victoriametrics/victoriametrics.go @@ -46,10 +46,6 @@ const ( updateBatchDelay = time.Second configurationUpdateTimeout = 3 * time.Second - // BasePrometheusConfigPath - basic path with prometheus config, - // that user can mount to container. - BasePrometheusConfigPath = "/srv/prometheus/prometheus.base.yml" - victoriametricsDir = "/srv/victoriametrics" victoriametricsDataDir = "/srv/victoriametrics/data" dirPerm = os.FileMode(0o775) @@ -64,15 +60,15 @@ type Service struct { baseURL *url.URL client *http.Client - baseConfigPath string // for testing + params *models.VictoriaMetricsParams l *logrus.Entry reloadCh chan struct{} } // NewVictoriaMetrics creates new VictoriaMetrics service. -func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, baseURL string, params *models.VictoriaMetricsParams) (*Service, error) { - u, err := url.Parse(baseURL) +func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, params *models.VictoriaMetricsParams) (*Service, error) { + u, err := url.Parse(params.URL()) if err != nil { return nil, errors.WithStack(err) } @@ -82,7 +78,7 @@ func NewVictoriaMetrics(scrapeConfigPath string, db *reform.DB, baseURL string, db: db, baseURL: u, client: &http.Client{}, // TODO instrument with utils/irt; see vmalert package https://jira.percona.com/browse/PMM-7229 - baseConfigPath: params.BaseConfigPath, + params: params, l: logrus.WithField("component", "victoriametrics"), reloadCh: make(chan struct{}, 1), }, nil @@ -145,6 +141,9 @@ func (svc *Service) RequestConfigurationUpdate() { // updateConfiguration updates VictoriaMetrics configuration. func (svc *Service) updateConfiguration(ctx context.Context) error { + if svc.params.ExternalVM() { + return nil + } start := time.Now() defer func() { if dur := time.Since(start); dur > time.Second { @@ -152,8 +151,7 @@ func (svc *Service) updateConfiguration(ctx context.Context) error { } }() - base := svc.loadBaseConfig() - cfg, err := svc.marshalConfig(base) + cfg, err := svc.buildVMConfig() if err != nil { return err } @@ -161,6 +159,11 @@ func (svc *Service) updateConfiguration(ctx context.Context) error { return svc.configAndReload(ctx, cfg) } +func (svc *Service) buildVMConfig() ([]byte, error) { + base := svc.loadBaseConfig() + return svc.marshalConfig(base) +} + // reload asks VictoriaMetrics to reload configuration. func (svc *Service) reload(ctx context.Context) error { u := *svc.baseURL @@ -189,10 +192,10 @@ func (svc *Service) reload(ctx context.Context) error { // loadBaseConfig returns parsed base configuration file, or empty configuration on error. func (svc *Service) loadBaseConfig() *config.Config { - buf, err := os.ReadFile(svc.baseConfigPath) + buf, err := os.ReadFile(svc.params.BaseConfigPath) if err != nil { if !os.IsNotExist(err) { - svc.l.Errorf("Failed to load base VictoriaMetrics config %s: %s", svc.baseConfigPath, err) + svc.l.Errorf("Failed to load base VictoriaMetrics config %s: %s", svc.params.BaseConfigPath, err) } return &config.Config{} @@ -200,7 +203,7 @@ func (svc *Service) loadBaseConfig() *config.Config { var cfg config.Config if err := yaml.Unmarshal(buf, &cfg); err != nil { - svc.l.Errorf("Failed to parse base VictoriaMetrics config %s: %s.", svc.baseConfigPath, err) + svc.l.Errorf("Failed to parse base VictoriaMetrics config %s: %s.", svc.params.BaseConfigPath, err) return &config.Config{} } @@ -330,7 +333,10 @@ func (svc *Service) populateConfig(cfg *config.Config) error { if cfg.GlobalConfig.ScrapeTimeout == 0 { cfg.GlobalConfig.ScrapeTimeout = ScrapeTimeout(s.LR) } - cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(s.HR)) + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVictoriaMetrics(svc.l, s.HR, svc.params)) + if svc.params.ExternalVM() { + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForInternalVMAgent(s.HR, svc.baseURL.Host)) + } cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scrapeConfigForVMAlert(s.HR)) AddInternalServicesToScrape(cfg, s, settings.DBaaS.Enabled) return AddScrapeConfigs(svc.l, cfg, tx.Querier, &s, nil, false) @@ -338,17 +344,41 @@ func (svc *Service) populateConfig(cfg *config.Config) error { } // scrapeConfigForVictoriaMetrics returns scrape config for Victoria Metrics in Prometheus format. -func scrapeConfigForVictoriaMetrics(interval time.Duration) *config.ScrapeConfig { +func scrapeConfigForVictoriaMetrics(l *logrus.Entry, interval time.Duration, vmParams *models.VictoriaMetricsParams) *config.ScrapeConfig { + target, err := vmParams.URLFor("metrics") + if err != nil { + l.Errorf("couldn't parse relative path to victoria metrics: %q", err) + return nil + } + return &config.ScrapeConfig{ JobName: "victoriametrics", ScrapeInterval: config.Duration(interval), ScrapeTimeout: ScrapeTimeout(interval), - MetricsPath: "/prometheus/metrics", + MetricsPath: target.Path, ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ StaticConfigs: []*config.Group{ { - Targets: []string{"127.0.0.1:9090"}, - Labels: map[string]string{"instance": "pmm-server"}, + Targets: []string{target.Host}, + Labels: map[string]string{"instance": models.PMMServerAgentID}, + }, + }, + }, + } +} + +// scrapeConfigForInternalVMAgent returns scrape config for internal VM Agent in Prometheus format. +func scrapeConfigForInternalVMAgent(interval time.Duration, target string) *config.ScrapeConfig { + return &config.ScrapeConfig{ + JobName: "vmagent", + ScrapeInterval: config.Duration(interval), + ScrapeTimeout: ScrapeTimeout(interval), + MetricsPath: "/metrics", + ServiceDiscoveryConfig: config.ServiceDiscoveryConfig{ + StaticConfigs: []*config.Group{ + { + Targets: []string{target}, + Labels: map[string]string{"instance": models.PMMServerAgentID}, }, }, }, @@ -375,6 +405,9 @@ func scrapeConfigForVMAlert(interval time.Duration) *config.ScrapeConfig { // BuildScrapeConfigForVMAgent builds scrape configuration for given pmm-agent. func (svc *Service) BuildScrapeConfigForVMAgent(pmmAgentID string) ([]byte, error) { + if pmmAgentID == models.PMMServerAgentID { + return svc.buildVMConfig() + } var cfg config.Config e := svc.db.InTransaction(func(tx *reform.TX) error { settings, err := models.GetSettings(tx) @@ -393,6 +426,10 @@ func (svc *Service) BuildScrapeConfigForVMAgent(pmmAgentID string) ([]byte, erro // IsReady verifies that VictoriaMetrics works. func (svc *Service) IsReady(ctx context.Context) error { + if svc.params.ExternalVM() { + svc.l.Debugf("External VM is used, VM healthcheck is skipped") + return nil + } u := *svc.baseURL u.Path = path.Join(u.Path, "health") req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) diff --git a/managed/services/victoriametrics/victoriametrics_test.go b/managed/services/victoriametrics/victoriametrics_test.go index 995cedf740..90607082fb 100644 --- a/managed/services/victoriametrics/victoriametrics_test.go +++ b/managed/services/victoriametrics/victoriametrics_test.go @@ -43,8 +43,9 @@ func setup(t *testing.T) (*reform.DB, *Service, []byte) { sqlDB := testdb.Open(t, models.SkipFixtures, nil) db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf)) - vmParams := &models.VictoriaMetricsParams{BaseConfigPath: "/srv/prometheus/prometheus.base.yml"} - svc, err := NewVictoriaMetrics(configPath, db, "http://127.0.0.1:9090/prometheus/", vmParams) + vmParams, err := models.NewVictoriaMetricsParams(models.BasePrometheusConfigPath, models.VMBaseURL) + check.NoError(err) + svc, err := NewVictoriaMetrics(configPath, db, vmParams) check.NoError(err) original, err := os.ReadFile(configPath) @@ -814,7 +815,7 @@ func TestBaseConfig(t *testing.T) { db, svc, original := setup(t) defer teardown(t, db, svc, original) - svc.baseConfigPath = "../../testdata/victoriametrics/promscrape.base.yml" + svc.params.BaseConfigPath = "../../testdata/victoriametrics/promscrape.base.yml" expected := strings.TrimSpace(` # Managed by pmm-managed. DO NOT EDIT. diff --git a/managed/testdata/supervisord.d/pmm-db_disabled.ini b/managed/testdata/supervisord.d/pmm-db_disabled.ini index 210d822422..b45251ab1f 100644 --- a/managed/testdata/supervisord.d/pmm-db_disabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_disabled.ini @@ -61,7 +61,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true diff --git a/managed/testdata/supervisord.d/pmm-db_enabled.ini b/managed/testdata/supervisord.d/pmm-db_enabled.ini index df0fd59059..4d4ad4afd9 100644 --- a/managed/testdata/supervisord.d/pmm-db_enabled.ini +++ b/managed/testdata/supervisord.d/pmm-db_enabled.ini @@ -85,7 +85,6 @@ priority = 14 command = /usr/sbin/pmm-managed --victoriametrics-config=/etc/victoriametrics-promscrape.yml - --victoriametrics-url=http://127.0.0.1:9090/prometheus --supervisord-config-dir=/etc/supervisord.d autorestart = true autostart = true diff --git a/managed/testdata/supervisord.d/vmalert.ini b/managed/testdata/supervisord.d/vmalert.ini index e784596bcb..40f61b1d83 100644 --- a/managed/testdata/supervisord.d/vmalert.ini +++ b/managed/testdata/supervisord.d/vmalert.ini @@ -7,11 +7,11 @@ command = --notifier.url="http://127.0.0.1:9093/alertmanager,https://external-alertmanager:6443/alerts" --notifier.basicAuth.password=',"passw!,ord"' --notifier.basicAuth.username=",external-user" - --external.url=http://localhost:9090/prometheus - --datasource.url=http://127.0.0.1:9090/prometheus - --remoteRead.url=http://127.0.0.1:9090/prometheus + --external.url=http://127.0.0.1:9090/prometheus/ + --datasource.url=http://127.0.0.1:9090/prometheus/ + --remoteRead.url=http://127.0.0.1:9090/prometheus/ --remoteRead.ignoreRestoreErrors=false - --remoteWrite.url=http://127.0.0.1:9090/prometheus + --remoteWrite.url=http://127.0.0.1:9090/prometheus/ --rule=/srv/prometheus/rules/*.yml --rule=/etc/ia/rules/*.yml --httpListenAddr=127.0.0.1:8880 diff --git a/managed/testdata/supervisord.d/vmproxy.ini b/managed/testdata/supervisord.d/vmproxy.ini index 72acc05eaf..5654b85bc4 100644 --- a/managed/testdata/supervisord.d/vmproxy.ini +++ b/managed/testdata/supervisord.d/vmproxy.ini @@ -4,7 +4,7 @@ priority = 9 command = /usr/sbin/vmproxy - --target-url=http://127.0.0.1:9090/ + --target-url=http://127.0.0.1:9090/prometheus/ --listen-port=8430 --listen-address=127.0.0.1 --header-name=X-Proxy-Filter diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 4bd4fc67f2..7847419984 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -42,6 +42,7 @@ const ( envEnableAccessControl = "ENABLE_RBAC" envPlatformAPITimeout = "PERCONA_PLATFORM_API_TIMEOUT" defaultPlatformAPITimeout = 30 * time.Second + ENVvmAgentPrefix = "VMAGENT_" ) // InvalidDurationError invalid duration error. @@ -156,6 +157,12 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs case "PMM_PUBLIC_ADDRESS": envSettings.PMMPublicAddress = v + case "PMM_VM_URL": + _, err = url.Parse(v) + if err != nil { + err = fmt.Errorf("invalid value %q for environment variable %q", v, k) + } + case "NO_PROXY", "HTTP_PROXY", "HTTPS_PROXY": continue @@ -203,6 +210,11 @@ func ParseEnvVars(envs []string) (envSettings *models.ChangeSettingsParams, errs continue } + // skip VM Agents environment variables + if strings.HasPrefix(k, ENVvmAgentPrefix) { + continue + } + // skip supervisord environment variables if strings.HasPrefix(k, "SUPERVISOR_") { continue diff --git a/update/ansible/playbook/tasks/files/datasources.yml b/update/ansible/playbook/tasks/files/datasources.yml index 79ba5ebcd0..b11fa2d0c8 100644 --- a/update/ansible/playbook/tasks/files/datasources.yml +++ b/update/ansible/playbook/tasks/files/datasources.yml @@ -8,7 +8,7 @@ datasources: orgId: 1 type: prometheus access: proxy - url: http://127.0.0.1:8430/prometheus/ + url: http://127.0.0.1:8430/ isDefault: true jsonData: httpMethod: POST diff --git a/vmproxy/Makefile b/vmproxy/Makefile index 458aea2543..de22f04d71 100644 --- a/vmproxy/Makefile +++ b/vmproxy/Makefile @@ -16,7 +16,7 @@ ifeq ($(GOBIN),) endif LD_FLAGS = -ldflags " \ - -X 'github.com/percona/pmm/version.ProjectName=pmm-admin' \ + -X 'github.com/percona/pmm/version.ProjectName=vmproxy' \ -X 'github.com/percona/pmm/version.Version=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.PMMVersion=$(PMM_RELEASE_VERSION)' \ -X 'github.com/percona/pmm/version.Timestamp=$(PMM_RELEASE_TIMESTAMP)' \ diff --git a/vmproxy/proxy/proxy.go b/vmproxy/proxy/proxy.go index 20fa20f33d..8d7f5ec762 100644 --- a/vmproxy/proxy/proxy.go +++ b/vmproxy/proxy/proxy.go @@ -24,6 +24,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "strings" "time" "github.com/pkg/errors" @@ -84,6 +85,12 @@ func director(target *url.URL, headerName string) func(*http.Request) { req.URL.Scheme = target.Scheme req.URL.Host = target.Host + rp, err := target.Parse(strings.TrimPrefix(req.URL.Path, "/")) + if err != nil { + logrus.Error(err) + } + req.URL.Path = rp.Path + // Replace extra filters if present if filters := req.Header.Get(headerName); filters != "" { q := req.URL.Query()