Merge pull request #133 from m-lab/sandbox-histogram

Use histogram for time-in-state

cloud/bq/dedup.go

@@ -31,6 +31,8 @@ var (
 )
 
 // WaitForStableTable loops checking until table exists and has no streaming buffer.
+// NOTE: We discovered that the StreamingBuffer == nil is not reliable, and must be rechecked for a minimum
+// of 5 minutes to ensure that there is no more buffered data.  (We saw reverts after as long as 4m50s).
 // TODO - refactor this to make it easier to understand.
 // TODO - move these functions to go/bqext package
 func WaitForStableTable(ctx context.Context, tt bqiface.Table) error {

metrics/metrics.go

@@ -10,6 +10,7 @@ func init() {
 	prometheus.MustRegister(StartedCount)
 	prometheus.MustRegister(CompletedCount)
 	prometheus.MustRegister(StateTimeSummary)
+	prometheus.MustRegister(StateTimeHistogram)
 	prometheus.MustRegister(StateDate)
 	prometheus.MustRegister(FilesPerDateHistogram)
 	prometheus.MustRegister(BytesPerDateHistogram)
@@ -102,6 +103,7 @@ var (
 	)
 
 	// StateTimeSummary measures the time spent in different task states.
+	// DEPRECATED - remove soon.
 	// Provides metrics:
 	//    gardener_state_time_summary
 	// Example usage:
@@ -113,6 +115,23 @@ var (
 	}, []string{"state"},
 	)
 
+	// StateTimeHistogram tracks the time spent in each state.  Not necessary to label data type, as
+	// we currently have separate gardener deployments for each type.
+	// Usage example:
+	//   metrics.StateTimeHistogram.WithLabelValues(
+	//           StateName[state]).Observe(time.Since(start).Seconds())
+	StateTimeHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "gardener_state_time_histogram",
+			Help: "time-in-state distributions.",
+			// These values range from seconds to hours.
+			Buckets: []float64{
+				0.1, 0.3, 1, 3, 10, 30,
+				100, 300, 1000, 1800, 3600, 2 * 3600, 4 * 3600, 8 * 3600, 12 * 3600,
+			},
+		},
+		[]string{"state"})
+
 	// FilesPerDateHistogram provides a histogram of files per date submitted to pipeline.
 	//
 	// Provides metrics:

state/state.go

@@ -219,7 +219,9 @@ func (t *Task) Save(ctx context.Context) error {
 
 // Update updates the task state, and saves to the "saver".
 func (t *Task) Update(ctx context.Context, st State) error {
-	duration := t.UpdateTime.Sub(time.Now())
+	duration := time.Since(t.UpdateTime)
+	metrics.StateTimeHistogram.WithLabelValues(StateNames[t.State]).Observe(duration.Seconds())
+	// TODO - remove this once we have some histogram history.
 	metrics.StateTimeSummary.WithLabelValues(StateNames[t.State]).Observe(duration.Seconds())
 	t.State = st
 	t.UpdateTime = time.Now()