PMM-10385 Explain tab. (#1311)

* PMM-10385 API. * PMM-10385 Gen. * Revert "Bump github.com/ClickHouse/clickhouse-go/v2 from 2.2.0 to 2.3.0 (#1176)" This reverts commit f2e63af. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Gen and changes. * PMM-10385 Changes. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Changes. * PMM-10385 Changes. * PMM-10385 Gen. * PMM-10385 Changes. * PMM-10385 Changes. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Mod. * PMM-10385 Mod. * PMM-10385 Mod. * PMM-10385 Tidy. * PMM-10385 Changes. * PMM-10385 Gen. * PMM-10385 Gen. * PMM-10385 Add new fields. * PMM-10385 Add license. * PMM-10385 Fix tests. * PMM-10385 Move package. * PMM-10385 Typo. * PMM-10385 Lint. * PMM-10385 Fix another test. * PMM-10385 Fix agent tests. * PMM-10385 Fix test. * PMM-10385 Fix. * PMM-10385 Add debug logging when cannot parse query. * PMM-10385 Merge QAN PR. * PMM-10385 Sync object details test. * PMM-10385 Sync reporter. * PMM-10385 Sync metrics. * PMM-10385 Sync slowlog. * PMM-10385 Sync slowlog. * Revert "PMM-10385 Sync slowlog." This reverts commit 3c92520. * PMM-10385 Format. * PMM-10385 Mod. * PMM-10385 Fix. * PMM-10385 Fix. * PMM-10385 Mod. * PMM-10385 Lint. * PMM-10385 Lint. * PMM-10385 Fix. * PMM-10385 Workaround for defect. * PMM-10385 Fix.
percona · Nov 29, 2022 · 404aa71 · 404aa71
1 parent f6a6ad3
commit 404aa71
Show file tree

Hide file tree

Showing 45 changed files with 3,097 additions and 1,375 deletions.
diff --git a/agent/agents/mysql/perfschema/perfschema.go b/agent/agents/mysql/perfschema/perfschema.go
@@ -32,6 +32,7 @@ import (
 
 	"github.com/percona/pmm/agent/agents"
 	"github.com/percona/pmm/agent/agents/cache"
+	"github.com/percona/pmm/agent/queryparser"
 	"github.com/percona/pmm/agent/tlshelpers"
 	"github.com/percona/pmm/agent/utils/truncate"
 	"github.com/percona/pmm/agent/utils/version"
@@ -333,13 +334,27 @@ func (m *PerfSchema) getNewBuckets(periodStart time.Time, periodLengthSecs uint3
 				b.Common.Schema = pointer.GetString(esh.CurrentSchema)
 			}
 
-			if !m.disableQueryExamples && esh.SQLText != nil {
-				example, truncated := truncate.Query(*esh.SQLText, m.maxQueryLength)
-				if truncated {
-					b.Common.IsTruncated = truncated
+			if esh.SQLText != nil {
+				explainFingerprint, placeholdersCount, err := queryparser.MySQL(*esh.SQLText)
+				if err != nil {
+					m.l.Debugf("cannot parse query: %s", *esh.SQLText)
+				} else {
+					explainFingerprint, truncated := truncate.Query(explainFingerprint, m.maxQueryLength)
+					if truncated {
+						b.Common.IsTruncated = truncated
+					}
+					b.Common.ExplainFingerprint = explainFingerprint
+					b.Common.PlaceholdersCount = placeholdersCount
+				}
+
+				if !m.disableQueryExamples {
+					example, truncated := truncate.Query(*esh.SQLText, m.maxQueryLength)
+					if truncated {
+						b.Common.IsTruncated = truncated
+					}
+					b.Common.Example = example
+					b.Common.ExampleType = agentpb.ExampleType_RANDOM
 				}
-				b.Common.Example = example
-				b.Common.ExampleType = agentpb.ExampleType_RANDOM
 			}
 		}
 

diff --git a/agent/agents/mysql/perfschema/perfschema_test.go b/agent/agents/mysql/perfschema/perfschema_test.go
@@ -336,6 +336,8 @@ func TestPerfSchema(t *testing.T) {
 		assert.InDelta(t, 0.1, actual.Common.MQueryTimeSum, 0.09)
 		expected := &agentpb.MetricsBucket{
 			Common: &agentpb.MetricsBucket_Common{
+				ExplainFingerprint:  "select /* Sleep */ sleep(:1) from dual",
+				PlaceholdersCount:   1,
 				Fingerprint:         "SELECT `sleep` (?)",
 				Schema:              "world",
 				AgentId:             "agent_id",
@@ -380,6 +382,7 @@ func TestPerfSchema(t *testing.T) {
 		assert.InDelta(t, 0, actual.Mysql.MLockTimeSum, 0.09)
 		expected := &agentpb.MetricsBucket{
 			Common: &agentpb.MetricsBucket_Common{
+				ExplainFingerprint:  "select /* AllCities */ * from city",
 				Fingerprint:         "SELECT * FROM `city`",
 				Schema:              "world",
 				AgentId:             "agent_id",

diff --git a/agent/agents/mysql/slowlog/slowlog.go b/agent/agents/mysql/slowlog/slowlog.go
@@ -36,6 +36,7 @@ import (
 
 	"github.com/percona/pmm/agent/agents"
 	"github.com/percona/pmm/agent/agents/mysql/slowlog/parser"
+	"github.com/percona/pmm/agent/queryparser"
 	"github.com/percona/pmm/agent/tlshelpers"
 	"github.com/percona/pmm/agent/utils/backoff"
 	"github.com/percona/pmm/agent/utils/truncate"
@@ -366,12 +367,12 @@ func (s *SlowLog) processFile(ctx context.Context, file string, outlierTime floa
 			s.l.Tracef("Parsed slowlog event: %+v.", e)
 			fingerprint := query.Fingerprint(e.Query)
 			digest := query.Id(fingerprint)
-			aggregator.AddEvent(e, digest, e.User, e.Host, e.Db, e.Server, fingerprint)
+			aggregator.AddEvent(e, digest, e.User, e.Host, e.Db, e.Server, e.Query)
 
 		case <-t.C:
 			lengthS := uint32(math.Round(wait.Seconds())) // round 59.9s/60.1s to 60s
 			res := aggregator.Finalize()
-			buckets := makeBuckets(s.params.AgentID, res, start, lengthS, s.params.DisableQueryExamples, s.params.MaxQueryLength)
+			buckets := makeBuckets(s.params.AgentID, res, start, lengthS, s.params.DisableQueryExamples, s.params.MaxQueryLength, s.l)
 			s.l.Debugf("Made %d buckets out of %d classes in %s+%d interval. Wait time: %s.",
 				len(buckets), len(res.Class), start.Format("15:04:05"), lengthS, time.Since(start))
 
@@ -387,14 +388,20 @@ func (s *SlowLog) processFile(ctx context.Context, file string, outlierTime floa
 }
 
 // makeBuckets is a pure function for easier testing.
-func makeBuckets(agentID string, res event.Result, periodStart time.Time, periodLengthSecs uint32, disableQueryExamples bool, maxQueryLength int32) []*agentpb.MetricsBucket { //nolint:cyclop,lll
+func makeBuckets(agentID string, res event.Result, periodStart time.Time, periodLengthSecs uint32, disableQueryExamples bool, maxQueryLength int32, l *logrus.Entry) []*agentpb.MetricsBucket {
 	buckets := make([]*agentpb.MetricsBucket, 0, len(res.Class))
 
 	for _, v := range res.Class {
 		if v.Metrics == nil {
 			continue
 		}
 
+		// In fingerprint field there is no fingerprint yet.
+		// It contains whole query without any changes.
+		// This in workaround to keep original query until field "Query" will be
+		// added here: https://github.com/percona/go-mysql/blob/PMM-2.0/event/class.go#L56
+		q := v.Fingerprint
+		v.Fingerprint = query.Fingerprint(v.Fingerprint)
 		fingerprint, isTruncated := truncate.Query(v.Fingerprint, maxQueryLength)
 		mb := &agentpb.MetricsBucket{
 			Common: &agentpb.MetricsBucket_Common{
@@ -416,13 +423,29 @@ func makeBuckets(agentID string, res event.Result, periodStart time.Time, period
 			Mysql: &agentpb.MetricsBucket_MySQL{},
 		}
 
-		if v.Example != nil && !disableQueryExamples {
-			example, truncated := truncate.Query(v.Example.Query, maxQueryLength)
-			if truncated {
-				mb.Common.IsTruncated = truncated
+		if q != "" {
+			explainFingerprint, placeholdersCount, err := queryparser.MySQL(v.Example.Query)
+			if err != nil {
+				l.Debugf("cannot parse query: %s", v.Example.Query)
+			} else {
+				explainFingerprint, truncated := truncate.Query(explainFingerprint, maxQueryLength)
+				if truncated {
+					mb.Common.IsTruncated = truncated
+				}
+				mb.Common.ExplainFingerprint = explainFingerprint
+				mb.Common.PlaceholdersCount = placeholdersCount
+			}
+		}
+
+		if v.Example != nil {
+			if !disableQueryExamples {
+				example, truncated := truncate.Query(v.Example.Query, maxQueryLength)
+				if truncated {
+					mb.Common.IsTruncated = truncated
+				}
+				mb.Common.Example = example
+				mb.Common.ExampleType = agentpb.ExampleType_RANDOM
 			}
-			mb.Common.Example = example
-			mb.Common.ExampleType = agentpb.ExampleType_RANDOM
 		}
 
 		// If key has suffix _time or _wait than field is TimeMetrics.

diff --git a/agent/agents/mysql/slowlog/slowlog_expected.json b/agent/agents/mysql/slowlog/slowlog_expected.json
@@ -2,6 +2,8 @@
 	{
 		"common": {
 			"queryid": "C684EA1D78348D23",
+			"explain_fingerprint": "select `name`, subsystem, type, `comment`, `count` from information_schema.innodb_metrics where `status` = :1",
+			"placeholders_count": 1,
 			"fingerprint": "select name, subsystem, type, comment, count from information_schema.innodb_metrics where status = ?",
 			"username": "root",
 			"agent_id": "/agent_id/73ee2f92-d5aa-45f0-8b09-6d3df605fd44",
@@ -44,6 +46,8 @@
 	{
 		"common": {
 			"queryid": "FA521F3C42DC5272",
+			"explain_fingerprint": "select pad from sbtest1 where id = :1",
+			"placeholders_count": 1,
 			"fingerprint": "select pad from sbtest1 where id=?",
 			"schema": "sbtest",
 			"username": "root",
@@ -117,6 +121,7 @@
 	{
 		"common": {
 			"queryid": "28FC5B5D583E2DA6",
+			"explain_fingerprint":  "show global status",
 			"fingerprint": "show global status",
 			"username": "root",
 			"agent_id": "/agent_id/73ee2f92-d5aa-45f0-8b09-6d3df605fd44",
@@ -159,6 +164,7 @@
 	{
 		"common": {
 			"queryid": "328AAB9660C2879E",
+			"explain_fingerprint":  "set @@lock_wait_timeout = 2",
 			"fingerprint": "set lock_wait_timeout=?",
 			"username": "root",
 			"agent_id": "/agent_id/73ee2f92-d5aa-45f0-8b09-6d3df605fd44",
@@ -189,6 +195,7 @@
 	{
 		"common": {
 			"queryid": "A212AD93263CF26F",
+			"explain_fingerprint":  "select @@version from dual",
 			"fingerprint": "select @@version",
 			"username": "root",
 			"agent_id": "/agent_id/73ee2f92-d5aa-45f0-8b09-6d3df605fd44",

diff --git a/agent/agents/mysql/slowlog/slowlog_test.go b/agent/agents/mysql/slowlog/slowlog_test.go
@@ -53,18 +53,22 @@ func TestSlowLogMakeBucketsInvalidUTF8(t *testing.T) {
 	parsingResult := event.Result{
 		Class: map[string]*event.Class{
 			"example": {
-				Metrics: &event.Metrics{},
+				Metrics:     &event.Metrics{},
+				Fingerprint: "SELECT * FROM contacts t0 WHERE t0.person_id = '߿�\xff\\ud83d\xdd'",
 				Example: &event.Example{
 					Query: "SELECT * FROM contacts t0 WHERE t0.person_id = '߿�\xff\\ud83d\xdd'",
 				},
 			},
 		},
 	}
 
-	actualBuckets := makeBuckets(agentID, parsingResult, periodStart, 60, false, truncate.GetDefaultMaxQueryLength())
+	actualBuckets := makeBuckets(agentID, parsingResult, periodStart, 60, false, truncate.GetDefaultMaxQueryLength(), logrus.NewEntry(logrus.New()))
 	expectedBuckets := []*agentpb.MetricsBucket{
 		{
 			Common: &agentpb.MetricsBucket_Common{
+				Fingerprint:         "select * from contacts t0 where t0.person_id = ?",
+				ExplainFingerprint:  "select * from contacts as t0 where t0.person_id = :1",
+				PlaceholdersCount:   1,
 				AgentId:             agentID,
 				AgentType:           inventorypb.AgentType_QAN_MYSQL_SLOWLOG_AGENT,
 				PeriodStartUnixSecs: 1557137220,
@@ -90,7 +94,7 @@ func TestSlowLogMakeBuckets(t *testing.T) {
 	parsingResult := event.Result{}
 	getDataFromFile(t, "slowlog_fixture.json", &parsingResult)
 
-	actualBuckets := makeBuckets(agentID, parsingResult, periodStart, 60, false, truncate.GetDefaultMaxQueryLength())
+	actualBuckets := makeBuckets(agentID, parsingResult, periodStart, 60, false, truncate.GetDefaultMaxQueryLength(), logrus.NewEntry(logrus.New()))
 
 	var expectedBuckets []*agentpb.MetricsBucket
 	getDataFromFile(t, "slowlog_expected.json", &expectedBuckets)

diff --git a/agent/queryparser/parser.go b/agent/queryparser/parser.go
@@ -0,0 +1,40 @@
+// Copyright 2019 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package queryparser
+
+import (
+	"github.com/pkg/errors"
+	"vitess.io/vitess/go/vt/proto/query"
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+// MySQL parse query and return fingeprint and placeholders.
+func MySQL(q string) (string, uint32, error) {
+	normalizedQuery, _, err := sqlparser.Parse2(q)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot parse query")
+	}
+
+	bv := make(map[string]*query.BindVariable)
+	err = sqlparser.Normalize(normalizedQuery, sqlparser.NewReservedVars("", sqlparser.GetBindvars(normalizedQuery)), bv)
+	if err != nil {
+		return "", 0, errors.Wrap(err, "cannot normalize query")
+	}
+
+	parsedQuery := sqlparser.NewParsedQuery(normalizedQuery)
+	bindVars := sqlparser.GetBindvars(normalizedQuery)
+
+	return parsedQuery.Query, uint32(len(bindVars)), nil
+}
diff --git a/api/agentpb/collector.pb.go b/api/agentpb/collector.pb.go
diff --git a/api/agentpb/collector.proto b/api/agentpb/collector.proto
@@ -31,6 +31,10 @@ message MetricsBucket {
   message Common {
     // md5 of digest_text/fingerprint.
     string queryid = 1;
+    // contains fingerprint prepared by sql parser, which can be different than fingerprint.
+    string explain_fingerprint = 25;
+    // ammount of variables in query.
+    uint32 placeholders_count = 26;
     // digest_text - query signature. Query without values.
     string fingerprint = 2;
     //