Preserve error states

m-lab · Feb 5, 2019 · 3d42bbb · 3d42bbb
1 parent ee0ef0a
commit 3d42bbb
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/reproc/dispatch.go b/reproc/dispatch.go
@@ -90,6 +90,7 @@ func NewTaskHandler(exec state.Executor, queues []string, saver state.Saver) *Ta
 var ErrTerminating = errors.New("TaskHandler is terminating")
 
 // StartTask starts a single task.  It should be properly initialized except for saver.
+// If the task had previously errored, this should clear the error from datastore.
 func (th *TaskHandler) StartTask(ctx context.Context, t state.Task) {
 	t.SetSaver(th.saver)
 

diff --git a/state/state.go b/state/state.go
@@ -288,8 +288,13 @@ loop:
 			}
 		}
 	}
-	metrics.CompletedCount.WithLabelValues("todo - add exp type").Inc()
-	t.Delete(ctx)
+	if t.ErrMsg == "" {
+		// Only delete the state entry if it completed without error.
+		t.Delete(ctx)
+		metrics.CompletedCount.WithLabelValues("todo - add exp type").Inc()
+	} else {
+		metrics.CompletedCount.WithLabelValues(StateNames[t.State] + " Error").Inc()
+	}
 	term.Done()
 }