exporter.go

package main

import (
	"context"
	"database/sql"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

// Exporter collects Oracle DB metrics. It implements prometheus.Collector.
type Exporter struct {
	dsn             string
	duration, error prometheus.Gauge
	totalScrapes    prometheus.Counter
	scrapeErrors    *prometheus.CounterVec
	scrapeResults   []prometheus.Metric
	up              prometheus.Gauge
	db              *sql.DB
	logger          *logrus.Logger
}

// NewExporter returns a new Oracle DB exporter for the provided DSN.
func NewExporter(dsn string, logger *logrus.Logger) *Exporter {
	db, _ := connect(dsn, logger)
	return &Exporter{
		dsn: dsn,
		duration: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: exporter,
			Name:      "last_scrape_duration_seconds",
			Help:      "Duration of the last scrape of metrics from Oracle DB.",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: exporter,
			Name:      "scrapes_total",
			Help:      "Total number of times Oracle DB was scraped for metrics.",
		}),
		scrapeErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: exporter,
			Name:      "scrape_errors_total",
			Help:      "Total number of times an error occured scraping a Oracle database.",
		}, []string{"collector"}),
		error: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: exporter,
			Name:      "last_scrape_error",
			Help:      "Whether the last scrape of metrics from Oracle DB resulted in an error (1 for error, 0 for success).",
		}),
		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "up",
			Help:      "Whether the Oracle database server is up.",
		}),
		db:     db,
		logger: logger,
	}
}

// Describe describes all the metrics exported by the Oracle DB exporter.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	// We cannot know in advance what metrics the exporter will generate
	// So we use the poor man's describe method: Run a collect
	// and send the descriptors of all the collected metrics. The problem
	// here is that we need to connect to the Oracle DB. If it is currently
	// unavailable, the descriptors will be incomplete. Since this is a
	// stand-alone exporter and not used as a library within other code
	// implementing additional metrics, the worst that can happen is that we
	// don't detect inconsistent metrics created by this exporter
	// itself. Also, a change in the monitored Oracle instance may change the
	// exported metrics during the runtime of the exporter.
	defer panicLogger()

	metricCh := make(chan prometheus.Metric)
	doneCh := make(chan struct{})

	go func() {
		defer panicLogger()
		for m := range metricCh {
			ch <- m.Desc()
		}
		close(doneCh)
	}()

	e.Collect(metricCh)
	close(metricCh)
	<-doneCh

}

// Collect implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	defer panicLogger()

	if *scrapeInterval == 0 { // if we are to scrape when the request is made
		e.scrape(ch)
	} else {
		scrapeResults := e.scrapeResults // There is a risk that e.scrapeResults will be replaced while we traverse this look. This should mitigate that risk
		for idx := range scrapeResults {
			ch <- scrapeResults[idx]
		}
	}
	ch <- e.duration
	ch <- e.totalScrapes
	ch <- e.error
	e.scrapeErrors.Collect(ch)
	ch <- e.up
}

func (e *Exporter) runScheduledScrapes() {
	defer panicLogger()

	if *scrapeInterval == 0 {
		return // Do nothing as scrapes will be done on Collect requests
	}
	ticker := time.NewTicker(*scrapeInterval)
	defer ticker.Stop()
	for {
		metricCh := make(chan prometheus.Metric, 5)
		go func() {
			defer panicLogger()
			scrapeResults := []prometheus.Metric{}
			for {
				scrapeResult, more := <-metricCh
				if more {
					scrapeResults = append(scrapeResults, scrapeResult)
				} else {
					e.scrapeResults = scrapeResults
					return
				}
			}
		}()
		e.scrape(metricCh)
		close(metricCh)
		<-ticker.C
	}
}

func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
	defer panicLogger()

	e.totalScrapes.Inc()
	var err error
	defer func(begun time.Time) {
		e.duration.Set(time.Since(begun).Seconds())
		if err == nil {
			e.error.Set(0)
		} else {
			e.error.Set(1)
		}
	}(time.Now())

	if e.db == nil {
		e.logger.Info("Reconnecting to DB")
		db, err := connect(e.dsn, e.logger)
		if err != nil {
			e.logger.Error("Reconnectiong to DB Error: ", err)
			e.up.Set(0)
			return
		}
		e.db = db
	}

	if err := ping(context.Background(), e.db); err != nil {
		e.logger.Info("Reconnecting to DB")
		db, err := connect(e.dsn, e.logger)
		if err != nil {
			e.logger.Error("Reconnectiong to DB Error: ", err)
			e.up.Set(0)
			return
		}
		e.db = db
	}

	if err := ping(context.Background(), e.db); err != nil {
		e.logger.Error("Error pinging oracle: ", err)
		//e.db.Close()
		e.up.Set(0)
		return
	} else {
		e.logger.Debug("Successfully pinged Oracle database: ", maskDsn(e.dsn))
		e.up.Set(1)
	}

	if checkIfMetricsChanged(e.logger) {
		reloadMetrics(e.logger)
	}

	wg := sync.WaitGroup{}

	metricsLock.RLock()
	defer metricsLock.RUnlock()
	for _, metric := range metricsToScrap.Metric {
		wg.Add(1)
		metric := metric //https://golang.org/doc/faq#closures_and_goroutines

		go func() {
			defer panicLogger()
			defer wg.Done()

			e.logger.Debug("About to scrape metric: ")
			e.logger.Debug("- Metric MetricsDesc: ", metric.MetricsDesc)
			e.logger.Debug("- Metric Context: ", metric.Context)
			e.logger.Debug("- Metric MetricsType: ", metric.MetricsType)
			e.logger.Debug("- Metric MetricsBuckets: ", metric.MetricsBuckets, "(Ignored unless Histogram type)")
			e.logger.Debug("- Metric Labels: ", metric.Labels)
			e.logger.Debug("- Metric FieldToAppend: ", metric.FieldToAppend)
			e.logger.Debug("- Metric IgnoreZeroResult: ", metric.IgnoreZeroResult)
			e.logger.Debug("- Metric Request: ", metric.Request)

			if len(metric.Request) == 0 {
				e.logger.Error("Error scraping for ", metric.MetricsDesc, ". Did you forget to define request in your toml file?")
				return
			}

			if len(metric.MetricsDesc) == 0 {
				e.logger.Error("Error scraping for query", metric.Request, ". Did you forget to define metricsdesc  in your toml file?")
				return
			}

			for column, metricType := range metric.MetricsType {
				if metricType == "histogram" {
					_, ok := metric.MetricsBuckets[column]
					if !ok {
						e.logger.Error("Unable to find MetricsBuckets configuration key for metric. (metric=" + column + ")")
						return
					}
				}
			}

			scrapeStart := time.Now()
			if err = ScrapeMetric(e.db, ch, metric, e.logger); err != nil {
				e.logger.Error("Error scraping for", metric.Context, "_", metric.MetricsDesc, time.Since(scrapeStart), ": ", err)
				e.scrapeErrors.WithLabelValues(metric.Context).Inc()
			} else {
				e.logger.Debug("Successfully scraped metric: ", metric.Context, metric.MetricsDesc, time.Since(scrapeStart))
			}
		}()
	}
	wg.Wait()
}