diff --git a/metrics/noop.go b/metrics/noop.go index 6eb909ff9..b804486b6 100644 --- a/metrics/noop.go +++ b/metrics/noop.go @@ -5,7 +5,10 @@ package metrics -import "net/http" +import ( + "net/http" + "time" +) // noopMetrics implements a no operations metrics service type noopMetrics struct{} @@ -48,3 +51,5 @@ func (n noopMeters) Set(int64) {} func (n noopMeters) Observe(int64) {} func (n *noopMetrics) ObserveWithLabels(int64, map[string]string) {} + +func (n *noopMetrics) collectDiskIO(time.Duration) {} diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 50745752c..15447f6dc 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -6,8 +6,15 @@ package metrics import ( + "bufio" + "fmt" "net/http" + "os" + "runtime" + "strconv" + "strings" "sync" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -24,6 +31,8 @@ func InitializePrometheusMetrics() { // don't allow for reset if _, ok := metrics.(*prometheusMetrics); !ok { metrics = newPrometheusMetrics() + // collection disk io metrics every 5 seconds + go metrics.(*prometheusMetrics).collectDiskIO(5 * time.Second) } } @@ -123,6 +132,62 @@ func (o *prometheusMetrics) GetOrCreateGaugeVecMeter(name string, labels []strin return meter } +func getIOLineValue(line string) int64 { + fields := strings.Fields(line) + if len(fields) != 2 { + logger.Warn("this io file line is malformed", "err", line) + return 0 + } + value, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + logger.Warn("unable to parse int", "err", err) + return 0 + } + + return value +} + +func getDiskIOData() (int64, int64, error) { + pid := os.Getpid() + ioFilePath := fmt.Sprintf("/proc/%d/io", pid) + file, err := os.Open(ioFilePath) + if err != nil { + return 0, 0, err + } + + // Parse the file line by line + scanner := bufio.NewScanner(file) + var reads, writes int64 + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "syscr") { + reads = getIOLineValue(line) + } else if strings.HasPrefix(line, "syscw") { + writes = getIOLineValue(line) + } + } + + return reads, writes, nil +} + +func (o *prometheusMetrics) collectDiskIO(refresh time.Duration) { + if runtime.GOOS != "linux" { + return + } + for { + reads, writes, err := getDiskIOData() + if err == nil { + readsMeter := o.GetOrCreateGaugeMeter("disk_reads") + readsMeter.Set(reads) + + writesMeter := o.GetOrCreateGaugeMeter("disk_writes") + writesMeter.Set(writes) + } + + time.Sleep(refresh) + } +} + func (o *prometheusMetrics) newHistogramMeter(name string, buckets []int64) HistogramMeter { var floatBuckets []float64 for _, bucket := range buckets {