RHINENG-9452: refactor error handling

base/errors.go

@@ -1,11 +1,36 @@
 package base
 
 import (
-	"errors"
+	stdErrors "errors"
+
+	"github.com/pkg/errors"
 )
 
 var (
 	ErrDatabase   = errors.New("database error")
+	ErrKafka      = errors.New("kafka error")
 	ErrBadRequest = errors.New("bad request")
 	ErrNotFound   = errors.New("not found")
+	ErrFatal      = errors.New("fatal error restarting pod")
 )
+
+func WrapFatalError(err error, message string) error {
+	return wrapErrors(err, message)
+}
+
+func WrapFatalDBError(err error, message string) error {
+	return wrapErrors(err, message, ErrFatal, ErrDatabase)
+}
+
+func WrapFatalKafkaError(err error, message string) error {
+	return wrapErrors(err, message, ErrFatal, ErrKafka)
+}
+
+func wrapErrors(err error, message string, errs ...error) error {
+	if err == nil {
+		return nil
+	}
+	errsJoined := stdErrors.Join(errs...)
+	err = stdErrors.Join(errsJoined, err)
+	return errors.Wrap(err, message)
+}

base/mqueue/mqueue.go

@@ -5,6 +5,7 @@ import (
 	"app/base"
 	"app/base/utils"
 	"context"
+	"errors"
 	format "fmt"
 	"io"
 	"strings"
@@ -55,13 +56,16 @@ func MakeRetryingHandler(handler MessageHandler) MessageHandler {
 		backoffState, cancel := policy.Start(base.Context)
 		defer cancel()
 		for backoff.Continue(backoffState) {
-			if err = handler(message); err == nil {
+			if err = handler(message); err == nil || !errors.Is(err, base.ErrFatal) {
 				return nil
 			}
 			utils.LogError("err", err.Error(), "attempt", attempt, "Try failed")
 			attempt++
 		}
-		return err
+		if err != nil && errors.Is(err, base.ErrFatal) {
+			return err
+		}
+		return nil
 	}
 }
 

evaluator/evaluate.go

@@ -214,7 +214,7 @@ func evaluateInDatabase(ctx context.Context, event *mqueue.PlatformEvent, invent
 
 	thirdParty, err := analyzeRepos(system)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "Repo analysis failed")
+		return nil, nil, errors.Wrap(err, "repo analysis failed")
 	}
 	system.ThirdParty = thirdParty // to set "system_platform.third_party" column
 
@@ -414,7 +414,7 @@ func tryGetSystem(accountID int, inventoryID string,
 	system, err := loadSystemData(accountID, inventoryID)
 	if err != nil {
 		evaluationCnt.WithLabelValues("error-db-read-inventory-data").Inc()
-		return nil, errors.Wrap(err, "error loading system from DB")
+		return nil, base.WrapFatalDBError(err, "error loading system from DB")
 	}
 	if system.ID == 0 {
 		evaluationCnt.WithLabelValues("error-db-read-inventory-data").Inc()
@@ -496,7 +496,7 @@ func evaluateAndStore(system *models.SystemPlatform,
 	err = commitWithObserve(tx)
 	if err != nil {
 		evaluationCnt.WithLabelValues("error-database-commit").Inc()
-		return errors.New("database commit failed")
+		return base.WrapFatalDBError(err, "database commit failed")
 	}
 
 	return nil

evaluator/evaluate_baseline.go

@@ -1,6 +1,7 @@
 package evaluator
 
 import (
+	"app/base"
 	"app/base/database"
 	"app/base/models"
 	"app/base/vmaas"
@@ -24,7 +25,7 @@ func limitVmaasToBaseline(system *models.SystemPlatform, vmaasData *vmaas.Update
 		Where("public_date >= ?", baselineConfig.ToTime.Truncate(24*time.Hour)).
 		Pluck("name", &filterOutNames).Error
 	if err != nil {
-		return err
+		return base.WrapFatalDBError(err, "load reported advisories")
 	}
 
 	// create map of advisories we need to filter out

evaluator/remediations.go

@@ -82,7 +82,8 @@ func publishRemediationsState(system *models.SystemPlatform, response *vmaas.Upd
 	state := createRemediationsStateMsg(system.InventoryID, response)
 	msg, err := mqueue.MessageFromJSON(system.InventoryID, state)
 	if err != nil {
-		return errors.Wrap(err, "Formatting message")
+		return errors.Wrap(err, "formatting message")
 	}
-	return remediationsPublisher.WriteMessages(base.Context, msg)
+	err = remediationsPublisher.WriteMessages(base.Context, msg)
+	return base.WrapFatalKafkaError(err, "write message")
 }