Merge pull request #128 from m-lab/preserve-errors

Preserve error states

reproc/dispatch.go

@@ -90,6 +90,7 @@ func NewTaskHandler(exec state.Executor, queues []string, saver state.Saver) *Ta
 var ErrTerminating = errors.New("TaskHandler is terminating")
 
 // StartTask starts a single task.  It should be properly initialized except for saver.
+// If the task had previously errored, this should clear the error from datastore.
 func (th *TaskHandler) StartTask(ctx context.Context, t state.Task) {
 	t.SetSaver(th.saver)
 

rex/rex_test.go

@@ -98,9 +98,10 @@ func TestWithTaskQueue(t *testing.T) {
 	if counter.Count() != 9 {
 		t.Error("Wrong number of client calls", counter.Count())
 	}
-	if len(saver.GetDeletes()) != 3 {
-		t.Error("Wrong number of task deletes", len(saver.GetDeletes()))
-	}
+
+	// TODO: The tasks currently end in error, so there are no task
+	// deletes.  If we change this behavior, we might want to check the
+	// value of saver.Deletes() here.
 
 	tasks := saver.GetTasks()
 	for _, task := range tasks {

state/state.go

@@ -288,8 +288,13 @@ loop:
 			}
 		}
 	}
-	metrics.CompletedCount.WithLabelValues("todo - add exp type").Inc()
-	t.Delete(ctx)
+	if t.ErrMsg == "" {
+		// Only delete the state entry if it completed without error.
+		t.Delete(ctx)
+		metrics.CompletedCount.WithLabelValues("todo - add exp type").Inc()
+	} else {
+		metrics.CompletedCount.WithLabelValues(StateNames[t.State] + " Error").Inc()
+	}
 	term.Done()
 }