Merge branch 'main' into PMM-12078-HA-Phase-3-Active-Active-PoC

.github/workflows/admin.yml

@@ -82,6 +82,7 @@ jobs:
 
   cli-test:
     name: CLI Tests
+    if: false
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
@@ -126,7 +127,6 @@ jobs:
 
       - name: Setup tools
         run: |
-          sudo apt-get install -y wget jq
           sudo ln -sf /home/runner/go/bin/pmm /usr/bin
           sudo chown -R runner:docker /usr/bin/pmm
 

.github/workflows/devcontainer.yml

@@ -0,0 +1,62 @@
+name: Devcontainer
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch to build the devcontainer from"
+        default: "main"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      branch:
+        description: "The branch to build the devcontainer from"
+        default: "main"
+        required: true
+        type: string
+
+jobs:
+  devcontainer:
+    name: Build
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+    permissions:
+      packages: write
+
+    env:
+      LAB_DEVCONTAINER_IMAGE: perconalab/pmm-server:dev-container
+      GH_DEVCONTAINER_IMAGE: ghcr.io/percona/pmm:dev-container
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to ghcr.io registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to docker.io registry
+        uses: docker/login-action@v2
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push to registries
+        uses: docker/build-push-action@v4
+        with:
+          file: ./devcontainer.Dockerfile
+          push: true
+          tags: |
+            ${{ env.GH_DEVCONTAINER_IMAGE }}
+            ${{ env.LAB_DEVCONTAINER_IMAGE }}

agent/agents/mysql/perfschema/history.go

@@ -53,16 +53,6 @@ func getHistory(q *reform.Querier) (historyMap, error) {
 	return getHistoryRows(rows, q)
 }
 
-func getHistory80(q *reform.Querier) (historyMap, error) {
-	rows, err := q.SelectRows(eventsStatementsSummaryByDigestExamplesView, "WHERE DIGEST IS NOT NULL AND QUERY_SAMPLE_TEXT IS NOT NULL")
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to query events_statements_summary_by_digest")
-	}
-	defer rows.Close() //nolint:errcheck
-
-	return getHistoryRows(rows, q)
-}
-
 func getHistoryRows(rows *sql.Rows, q *reform.Querier) (historyMap, error) {
 	var err error
 	res := make(historyMap)

agent/agents/mysql/perfschema/perfschema.go

@@ -18,6 +18,7 @@ package perfschema
 import (
 	"context"
 	"database/sql"
+	"fmt"
 	"io"
 	"math"
 	"sync"
@@ -119,6 +120,39 @@ type newPerfSchemaParams struct {
 
 const queryTag = "agent='perfschema'"
 
+// getPerfschemaSummarySize returns size of rows for perfschema summary cache.
+func getPerfschemaSummarySize(q reform.Querier, l *logrus.Entry) uint {
+	var name string
+	var size uint
+
+	query := fmt.Sprintf("SHOW VARIABLES /* %s */ LIKE 'performance_schema_digests_size'", queryTag)
+	err := q.QueryRow(query).Scan(&name, &size)
+	if err != nil {
+		l.Debug(err)
+		size = summariesCacheSize
+	}
+
+	l.Infof("performance_schema_digests_size=%d", size)
+
+	return size
+}
+
+// getPerfschemaHistorySize returns size of rows for perfschema history cache.
+func getPerfschemaHistorySize(q reform.Querier, l *logrus.Entry) uint {
+	var name string
+	var size uint
+	query := fmt.Sprintf("SHOW VARIABLES /* %s */ LIKE 'performance_schema_events_statements_history_long_size'", queryTag)
+	err := q.QueryRow(query).Scan(&name, &size)
+	if err != nil {
+		l.Debug(err)
+		size = historyCacheSize
+	}
+
+	l.Infof("performance_schema_events_statements_history_long_size=%d", size)
+
+	return size
+}
+
 // New creates new PerfSchema QAN service.
 func New(params *Params, l *logrus.Entry) (*PerfSchema, error) {
 	if params.TextFiles != nil {
@@ -152,12 +186,12 @@ func New(params *Params, l *logrus.Entry) (*PerfSchema, error) {
 }
 
 func newPerfSchema(params *newPerfSchemaParams) (*PerfSchema, error) {
-	historyCache, err := newHistoryCache(historyMap{}, retainHistory, historyCacheSize, params.LogEntry)
+	historyCache, err := newHistoryCache(historyMap{}, retainHistory, getPerfschemaHistorySize(*params.Querier, params.LogEntry), params.LogEntry)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create cache")
 	}
 
-	summaryCache, err := newSummaryCache(summaryMap{}, retainSummaries, summariesCacheSize, params.LogEntry)
+	summaryCache, err := newSummaryCache(summaryMap{}, retainSummaries, getPerfschemaSummarySize(*params.Querier, params.LogEntry), params.LogEntry)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create cache")
 	}
@@ -279,16 +313,7 @@ func (m *PerfSchema) runHistoryCacheRefresher(ctx context.Context) {
 }
 
 func (m *PerfSchema) refreshHistoryCache() error {
-	mysqlVer := m.mySQLVersion()
-
-	var err error
-	var current historyMap
-	switch {
-	case mysqlVer.version >= 8 && mysqlVer.vendor == "oracle":
-		current, err = getHistory80(m.q)
-	default:
-		current, err = getHistory(m.q)
-	}
+	current, err := getHistory(m.q)
 	if err != nil {
 		return err
 	}

agent/agents/supervisor/supervisor.go

@@ -359,6 +359,12 @@ func (s *Supervisor) setBuiltinAgents(builtinAgents map[string]*agentpb.SetState
 		<-agent.done
 
 		delete(s.builtinAgents, agentID)
+
+		agentTmp := filepath.Join(s.cfg.Get().Paths.TempDir, strings.ToLower(agent.requestedState.Type.String()), agentID)
+		err := os.RemoveAll(agentTmp)
+		if err != nil {
+			s.l.Warnf("Failed to cleanup directory '%s': %s", agentTmp, err.Error())
+		}
 	}
 
 	// restart

agent/cmd/pmm-agent-entrypoint/main.go

@@ -28,6 +28,8 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"gopkg.in/alecthomas/kingpin.v2"
+
+	"github.com/percona/pmm/utils/logger"
 )
 
 var helpText = `
@@ -131,11 +133,7 @@ func main() {
 
 	var status int
 
-	logrus.SetFormatter(&logrus.TextFormatter{
-		ForceColors:     true,
-		FullTimestamp:   true,
-		TimestampFormat: "2006-01-02T15:04:05.000-07:00",
-	})
+	logger.SetupGlobalLogger()
 
 	l := logrus.WithField("component", "entrypoint")
 

agent/commands/run.go

@@ -40,6 +40,7 @@ import (
 
 // Run implements `pmm-agent run` default command.
 func Run() {
+	var cfg *config.Config
 	l := logrus.WithField("component", "main")
 	ctx, cancel := context.WithCancel(context.Background())
 	defer l.Info("Done.")
@@ -55,6 +56,9 @@ func Run() {
 		s := <-signals
 		signal.Stop(signals)
 		l.Warnf("Got %s, shutting down...", unix.SignalName(s.(unix.Signal))) //nolint:forcetypeassert
+		if cfg != nil {
+			cleanupTmp(cfg.Paths.TempDir, l)
+		}
 		cancel()
 	}()
 
@@ -64,7 +68,7 @@ func Run() {
 		l.Fatalf("Failed to load configuration: %s.", err)
 	}
 
-	cfg := configStorage.Get()
+	cfg = configStorage.Get()
 
 	cleanupTmp(cfg.Paths.TempDir, l)
 	connectionUptimeService := connectionuptime.NewService(cfg.WindowConnectedTime)

agent/config/config.go

@@ -17,6 +17,7 @@ package config
 
 import (
 	"fmt"
+	"io/fs"
 	"net"
 	"net/url"
 	"os"
@@ -25,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"gopkg.in/alecthomas/kingpin.v2"
@@ -34,7 +36,10 @@ import (
 	"github.com/percona/pmm/version"
 )
 
-const pathBaseDefault = "/usr/local/percona/pmm2"
+const (
+	pathBaseDefault = "/usr/local/percona/pmm2"
+	agentTmpPath    = "tmp" // temporary directory to keep exporters' config files, relative to pathBase
+)
 
 // Server represents PMM Server configuration.
 type Server struct {
@@ -179,7 +184,7 @@ func getFromCmdLine(cfg *Config, l *logrus.Entry) (string, error) {
 }
 
 // get is Get for unit tests: it parses args instead of command-line.
-func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err error) { //nolint:nonamedreturns
+func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err error) { //nolint:nonamedreturns,cyclop
 	// tweak configuration on exit to cover all return points
 	defer func() {
 		if cfg == nil {
@@ -212,7 +217,6 @@ func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err e
 			&cfg.Paths.RDSExporter:      "rds_exporter",
 			&cfg.Paths.AzureExporter:    "azure_exporter",
 			&cfg.Paths.VMAgent:          "vmagent",
-			&cfg.Paths.TempDir:          os.TempDir(),
 			&cfg.Paths.PTSummary:        "tools/pt-summary",
 			&cfg.Paths.PTPGSummary:      "tools/pt-pg-summary",
 			&cfg.Paths.PTMongoDBSummary: "tools/pt-mongodb-summary",
@@ -237,6 +241,16 @@ func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err e
 			cfg.Paths.ExportersBase = abs
 		}
 
+		if cfg.Paths.TempDir == "" {
+			cfg.Paths.TempDir = filepath.Join(cfg.Paths.PathsBase, agentTmpPath)
+			l.Infof("Temporary directory is not configured and will be set to %s", cfg.Paths.TempDir)
+		}
+
+		if !filepath.IsAbs(cfg.Paths.TempDir) {
+			cfg.Paths.TempDir = filepath.Join(cfg.Paths.PathsBase, cfg.Paths.TempDir)
+			l.Debugf("Temporary directory is configured as %s", cfg.Paths.TempDir)
+		}
+
 		if !filepath.IsAbs(cfg.Paths.PTSummary) {
 			cfg.Paths.PTSummary = filepath.Join(cfg.Paths.PathsBase, cfg.Paths.PTSummary)
 		}
@@ -308,7 +322,7 @@ func get(args []string, cfg *Config, l *logrus.Entry) (configFileF string, err e
 	}
 
 	*cfg = *fileCfg
-	return //nolint:nakedret
+	return configFileF, nil
 }
 
 // Application returns kingpin application that will parse command-line flags and environment variables
@@ -474,7 +488,7 @@ func Application(cfg *Config) (*kingpin.Application, *string) {
 // Other errors are returned if file exists, but configuration can't be loaded due to permission problems,
 // YAML parsing problems, etc.
 func loadFromFile(path string) (*Config, error) {
-	if _, err := os.Stat(path); os.IsNotExist(err) {
+	if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) {
 		return nil, ConfigFileDoesNotExistError(path)
 	}
 
@@ -510,8 +524,8 @@ func SaveToFile(path string, cfg *Config, comment string) error {
 func IsWritable(path string) error {
 	_, err := os.Stat(path)
 	if err != nil {
-		// File doesn't exists, check if folder is writable.
-		if os.IsNotExist(err) {
+		// File doesn't exist, check if folder is writable.
+		if errors.Is(err, fs.ErrNotExist) {
 			return unix.Access(filepath.Dir(path), unix.W_OK)
 		}
 		return err