Skip to content

Commit

Permalink
PMM-10464 Telemetry data from Grafana (#1285)
Browse files Browse the repository at this point in the history
* PMM-10464 Initial commit.

* PMM-10464 Refactoring tests.

* PMM-10464 FetchMetrics for Grafana DB.

* PMM-10464 FetchMetrics for Grafana DB.

* PMM-10464 Tests and code adjustments.

* PMM-10464 Tests and code adjustments.

* PMM-10464 Code adjustments.

* PMM-10464 Code adjustments.

* PMM-10464 pre/post fetching methods in telemetry datasources.

* PMM-10464 pre/post fetching methods in telemetry datasources.

* PMM-10464 Fixing tests.

* PMM-10464 Fixing tests.

* PMM-10464 Code review adjustments.

* PMM-10464 bump sqlitedb driver

* PMM-10464 GrafanaUsersCount metrics.

* PMM-10464 Licenses fix + go.mod cleanup.

* PMM-10464 Tests fix after rebase.

* PMM-10464 Codereview adjustments.

* PMM-10464 Codereview adjustments.

* PMM-10464 Changeing sqlite driver.

* PMM-10464 Comment fixing.

* PMM-10464 Comment fixing.

* PMM-10464 Code review adjustments.

* PMM-10464 Linter fixes.
  • Loading branch information
pkadej authored Dec 14, 2022
1 parent 41e65bc commit 742da09
Show file tree
Hide file tree
Showing 15 changed files with 383 additions and 21 deletions.
12 changes: 12 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ require (
k8s.io/apimachinery v0.25.4
k8s.io/cli-runtime v0.24.8
k8s.io/client-go v0.24.8
modernc.org/sqlite v1.19.5
vitess.io/vitess v0.15.0
)

Expand All @@ -105,11 +106,13 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 // indirect
github.com/philhofer/fwd v1.0.0 // indirect
github.com/posener/complete v1.2.3 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tinylib/msgp v1.1.1 // indirect
Expand All @@ -123,6 +126,15 @@ require (
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.21.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect
sigs.k8s.io/controller-runtime v0.12.2 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/kustomize/api v0.12.1 // indirect
Expand Down
31 changes: 30 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
Expand Down Expand Up @@ -515,6 +516,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
Expand Down Expand Up @@ -575,7 +578,7 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw=
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
Expand Down Expand Up @@ -749,6 +752,8 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/ramr/go-reaper v0.2.1 h1:zww+wlQOvTjBZuk1920R/e0GFEb6O7+B0WQLV6dM924=
github.com/ramr/go-reaper v0.2.1/go.mod h1:AVypdzrcCXjSc/JYnlXl8TsB+z84WyFzxWE8Jh0MOJc=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
Expand Down Expand Up @@ -1382,6 +1387,30 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/libc v1.21.5 h1:xBkU9fnHV+hvZuPSRszN0AXDG4M7nwPLwTWwkYcvLCI=
modernc.org/libc v1.21.5/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk=
modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.19.5 h1:E3iHL55c1Vw1knqIeU9N7B0fSjuiOjHZo7iVMsO6U5U=
modernc.org/sqlite v1.19.5/go.mod h1:EsYz8rfOvLCiYTy5ZFsOYzoCcRMu98YYkwAcCw5YIYw=
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down
4 changes: 4 additions & 0 deletions managed/services/config/pmm-managed.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ services:
separate_credentials:
username: pmm
password: pmm
GRAFANADB_SELECT:
enabled: true
timeout: 5s
db_file: /srv/grafana/grafana.db
reporting:
send: true
send_on_start: false
Expand Down
14 changes: 11 additions & 3 deletions managed/services/telemetry/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ type ServiceConfig struct {
telemetry []Config `yaml:"-"`
SaasHostname string `yaml:"saas_hostname"`
DataSources struct {
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
} `yaml:"datasources"`
Reporting ReportingConfig `yaml:"reporting"`
}
Expand All @@ -70,6 +71,13 @@ type DataSourceVictoriaMetrics struct {
Address string `yaml:"address"`
}

// DSGrafanaSqliteDB telemetry config.
type DSGrafanaSqliteDB struct {
Enabled bool `yaml:"enabled"`
Timeout time.Duration `yaml:"timeout"`
DBFile string `yaml:"db_file"`
}

// DSConfigPMMDB telemetry config.
type DSConfigPMMDB struct {
Enabled bool `yaml:"enabled"`
Expand Down
17 changes: 14 additions & 3 deletions managed/services/telemetry/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ datasources:
separate_credentials:
username: pmm-managed
password: pmm-managed
GRAFANADB_SELECT:
enabled: true
timeout: 2s
db_file: /srv/grafana/grafana.db
reporting:
send: true
send_on_start: true
Expand All @@ -67,9 +72,10 @@ reporting:
SendTimeout: time.Second * 10,
},
DataSources: struct {
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
VM *DataSourceVictoriaMetrics `yaml:"VM"`
QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"`
PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"`
GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"`
}{
VM: &DataSourceVictoriaMetrics{
Enabled: true,
Expand All @@ -92,6 +98,11 @@ reporting:
Password: "pmm-managed",
},
},
GrafanaDBSelect: &DSGrafanaSqliteDB{
Enabled: true,
Timeout: time.Second * 2,
DBFile: "/srv/grafana/grafana.db",
},
},
}
assert.Equal(t, actual, expected)
Expand Down
126 changes: 126 additions & 0 deletions managed/services/telemetry/datasource_grafana_sqlitedb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

// Package telemetry provides telemetry functionality.
package telemetry

import (
"context"
"database/sql"
"io"
"os"

// Events, errors and driver for for grafana sqlite database.
pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
_ "modernc.org/sqlite"
)

type dsGrafanaSelect struct {
l *logrus.Entry
config DSGrafanaSqliteDB
db *sql.DB
tempFile string
}

// check interfaces.
var (
_ DataSource = (*dsGrafanaSelect)(nil)
)

// Enabled flag that determines if data source is enabled.
func (d *dsGrafanaSelect) Enabled() bool {
return d.config.Enabled
}

// NewDataSourceGrafanaSqliteDB makes new data source for grafana sqlite database metrics.
func NewDataSourceGrafanaSqliteDB(config DSGrafanaSqliteDB, l *logrus.Entry) DataSource {
return &dsGrafanaSelect{
l: l,
config: config,
db: nil,
tempFile: "",
}
}

func (d *dsGrafanaSelect) Init(ctx context.Context) error {
// validate source file db
sourceFileStat, err := os.Stat(d.config.DBFile)
if err != nil {
return err
}

if sourceFileStat.Size() == 0 {
return errors.Errorf("Sourcefile %s is empty.", d.config.DBFile)
}

if !sourceFileStat.Mode().IsRegular() {
return errors.Wrapf(err, "%s is not a regular file", d.config.DBFile)
}

source, err := os.Open(d.config.DBFile)
if err != nil {
return err
}

tempFile, err := os.CreateTemp(os.TempDir(), "grafana")
if err != nil {
return err
}

defer func() {
if err := source.Close(); err != nil {
d.l.Errorf("Error closing file. %s", err)
}
}()

nBytes, err := io.Copy(tempFile, source)
d.l.Debugf("grafana sqlitedb copied with total bytes: %d", nBytes)
if err != nil || nBytes == 0 {
return errors.Wrapf(err, "cannot create copy of database file %s", d.config.DBFile)
}

db, err := sql.Open("sqlite", tempFile.Name())
if err != nil {
return err
}

d.tempFile = tempFile.Name()
d.db = db

return nil
}

func (d *dsGrafanaSelect) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) {
if d.db == nil {
return nil, errors.Errorf("temporary grafana database is not initialized: %s", d.config.DBFile)
}
return fetchMetricsFromDB(ctx, d.l, d.config.Timeout, d.db, config)
}

func (d *dsGrafanaSelect) Dispose(ctx context.Context) error {
err := d.db.Close()
if err != nil {
return err
}

err = os.Remove(d.tempFile)
if err != nil {
return errors.Wrapf(err, "failed to remove sqlite database file")
}

return nil
}
Loading

0 comments on commit 742da09

Please sign in to comment.