mysqld system metrics, with TabletManager rpc (#16850)

Signed-off-by: Shlomi Noach <[email protected]>

go/vt/mysqlctl/fakemysqldaemon.go

@@ -744,6 +744,19 @@ func (fmd *FakeMysqlDaemon) GetVersionComment(ctx context.Context) (string, erro
 	return "", nil
 }
 
+func (fmd *FakeMysqlDaemon) HostMetrics(ctx context.Context, cnf *Mycnf) (*mysqlctlpb.HostMetricsResponse, error) {
+	return &mysqlctlpb.HostMetricsResponse{
+		Metrics: map[string]*mysqlctlpb.HostMetricsResponse_Metric{
+			"loadavg": {
+				Value: 1.0,
+			},
+			"datadir-used-ratio": {
+				Value: 0.2,
+			},
+		},
+	}, nil
+}
+
 // AcquireGlobalReadLock is part of the MysqlDaemon interface.
 func (fmd *FakeMysqlDaemon) AcquireGlobalReadLock(ctx context.Context) error {
 	return errors.New("not implemented")

go/vt/mysqlctl/grpcmysqlctlserver/server.go

@@ -88,6 +88,10 @@ func (s *server) VersionString(ctx context.Context, request *mysqlctlpb.VersionS
 	return &mysqlctlpb.VersionStringResponse{Version: version}, nil
 }
 
+func (s *server) HostMetrics(ctx context.Context, request *mysqlctlpb.HostMetricsRequest) (*mysqlctlpb.HostMetricsResponse, error) {
+	return s.mysqld.HostMetrics(ctx, s.cnf)
+}
+
 // StartServer registers the Server for RPCs.
 func StartServer(s *grpc.Server, cnf *mysqlctl.Mycnf, mysqld *mysqlctl.Mysqld) {
 	mysqlctlpb.RegisterMysqlCtlServer(s, &server{cnf: cnf, mysqld: mysqld})

go/vt/mysqlctl/mysql_daemon.go

@@ -119,6 +119,9 @@ type MysqlDaemon interface {
 	// GetVersionComment returns the version comment
 	GetVersionComment(ctx context.Context) (string, error)
 
+	// HostMetrics returns some OS metrics
+	HostMetrics(ctx context.Context, cnf *Mycnf) (*mysqlctlpb.HostMetricsResponse, error)
+
 	// ExecuteSuperQuery executes a single query, no result
 	ExecuteSuperQuery(ctx context.Context, query string) error
 

go/vt/mysqlctl/mysqld.go

@@ -35,6 +35,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -1293,6 +1294,71 @@ func (mysqld *Mysqld) GetVersionComment(ctx context.Context) (string, error) {
 	return res.ToString("@@global.version_comment")
 }
 
+// hostMetrics returns several OS metrics to be used by the tablet throttler.
+func hostMetrics(ctx context.Context, cnf *Mycnf) (*mysqlctlpb.HostMetricsResponse, error) {
+	resp := &mysqlctlpb.HostMetricsResponse{
+		Metrics: make(map[string]*mysqlctlpb.HostMetricsResponse_Metric),
+	}
+	newMetric := func(name string) *mysqlctlpb.HostMetricsResponse_Metric {
+		metric := &mysqlctlpb.HostMetricsResponse_Metric{
+			Name: name,
+		}
+		resp.Metrics[name] = metric
+		return metric
+	}
+	withError := func(metric *mysqlctlpb.HostMetricsResponse_Metric, err error) error {
+		if err != nil {
+			metric.Error = &vtrpcpb.RPCError{
+				Message: err.Error(),
+				Code:    vtrpcpb.Code_FAILED_PRECONDITION,
+			}
+		}
+		return err
+	}
+
+	_ = func() error {
+		metric := newMetric("datadir-used-ratio")
+		// 0.0 for empty mount, 1.0 for completely full mount
+		var st syscall.Statfs_t
+		if err := syscall.Statfs(cnf.DataDir, &st); err != nil {
+			return withError(metric, err)
+		}
+		if st.Blocks == 0 {
+			return withError(metric, fmt.Errorf("unexpected zero blocks in %s", cnf.DataDir))
+		}
+		metric.Value = float64(st.Blocks-st.Bfree) / float64(st.Blocks)
+		return nil
+	}()
+
+	_ = func() error {
+		metric := newMetric("loadavg")
+		if runtime.GOOS != "linux" {
+			return withError(metric, fmt.Errorf("loadavg metric is only available on Linux"))
+		}
+		content, err := os.ReadFile("/proc/loadavg")
+		if err != nil {
+			return withError(metric, err)
+		}
+		fields := strings.Fields(string(content))
+		if len(fields) == 0 {
+			return withError(metric, fmt.Errorf("unexpected /proc/loadavg content"))
+		}
+		loadAvg, err := strconv.ParseFloat(fields[0], 64)
+		if err != nil {
+			return withError(metric, err)
+		}
+		metric.Value = loadAvg / float64(runtime.NumCPU())
+		return nil
+	}()
+
+	return resp, nil
+}
+
+// HostMetrics returns several OS metrics to be used by the tablet throttler.
+func (mysqld *Mysqld) HostMetrics(ctx context.Context, cnf *Mycnf) (*mysqlctlpb.HostMetricsResponse, error) {
+	return hostMetrics(ctx, cnf)
+}
+
 // ApplyBinlogFile extracts a binary log file and applies it to MySQL. It is the equivalent of:
 // $ mysqlbinlog --include-gtids binlog.file | mysql
 func (mysqld *Mysqld) ApplyBinlogFile(ctx context.Context, req *mysqlctlpb.ApplyBinlogFileRequest) error {

go/vt/mysqlctl/mysqld_test.go

@@ -307,3 +307,18 @@ func TestGetVersionComment(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, ver, str)
 }
+
+func TestHostMetrics(t *testing.T) {
+	ctx := context.Background()
+	cnf := &Mycnf{
+		DataDir: os.TempDir(),
+	}
+	resp, err := hostMetrics(ctx, cnf)
+	require.NoError(t, err)
+	assert.NotEmpty(t, resp.Metrics)
+	assert.Contains(t, resp.Metrics, "loadavg")
+	assert.Contains(t, resp.Metrics, "datadir-used-ratio")
+	metric := resp.Metrics["datadir-used-ratio"]
+	assert.Equal(t, "datadir-used-ratio", metric.Name)
+	assert.Empty(t, metric.Error)
+}