Start inBackupHealthReport during migrations (#13120)

* Start `inBackupHealthReport` during migrations

* Update to StartUpHealthReport

.changeset/soft-maps-ring.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#changed Rename the `InBackupHealthReport` to `StartUpHealthReport` and enable it for DB migrations as well. This will enable health report to be available during long start-up tasks (db backups and migrations).

core/cmd/shell.go

@@ -248,6 +248,9 @@ func handleNodeVersioning(ctx context.Context, db *sqlx.DB, appLggr logger.Logge
 	var err error
 	// Set up the versioning Configs
 	verORM := versioning.NewORM(db, appLggr)
+	ibhr := services.NewStartUpHealthReport(healthReportPort, appLggr)
+	ibhr.Start()
+	defer ibhr.Stop()
 
 	if static.Version != static.Unset {
 		var appv, dbv *semver.Version
@@ -312,9 +315,6 @@ func takeBackupIfVersionUpgrade(dbUrl url.URL, rootDir string, cfg periodicbacku
 
 	//Because backups can take a long time we must start a "fake" health report to prevent
 	//node shutdown because of healthcheck fail/timeout
-	ibhr := services.NewInBackupHealthReport(healthReportPort, lggr)
-	ibhr.Start()
-	defer ibhr.Stop()
 	err = databaseBackup.RunBackup(appv.String())
 	return err
 }

core/cmd/shell_local.go

@@ -768,7 +768,7 @@ func (s *Shell) ResetDatabase(c *cli.Context) error {
 		return s.errorOut(err)
 	}
 	lggr.Debugf("Migrating database: %#v", parsed.String())
-	if err := migrateDB(ctx, cfg, lggr); err != nil {
+	if err := migrateDB(ctx, cfg); err != nil {
 		return s.errorOut(err)
 	}
 	schema, err := dumpSchema(parsed)
@@ -777,7 +777,7 @@ func (s *Shell) ResetDatabase(c *cli.Context) error {
 	}
 	lggr.Debugf("Testing rollback and re-migrate for database: %#v", parsed.String())
 	var baseVersionID int64 = 54
-	if err := downAndUpDB(ctx, cfg, lggr, baseVersionID); err != nil {
+	if err := downAndUpDB(ctx, cfg, baseVersionID); err != nil {
 		return s.errorOut(err)
 	}
 	if err := checkSchema(parsed, schema); err != nil {
@@ -931,7 +931,7 @@ func (s *Shell) MigrateDatabase(_ *cli.Context) error {
 	}
 
 	s.Logger.Infof("Migrating database: %#v", parsed.String())
-	if err := migrateDB(ctx, cfg, s.Logger); err != nil {
+	if err := migrateDB(ctx, cfg); err != nil {
 		return s.errorOut(err)
 	}
 	return nil
@@ -1131,7 +1131,7 @@ func dropAndCreatePristineDB(db *sqlx.DB, template string) (err error) {
 	return nil
 }
 
-func migrateDB(ctx context.Context, config dbConfig, lggr logger.Logger) error {
+func migrateDB(ctx context.Context, config dbConfig) error {
 	db, err := newConnection(config)
 	if err != nil {
 		return fmt.Errorf("failed to initialize orm: %v", err)
@@ -1143,7 +1143,7 @@ func migrateDB(ctx context.Context, config dbConfig, lggr logger.Logger) error {
 	return db.Close()
 }
 
-func downAndUpDB(ctx context.Context, cfg dbConfig, lggr logger.Logger, baseVersionID int64) error {
+func downAndUpDB(ctx context.Context, cfg dbConfig, baseVersionID int64) error {
 	db, err := newConnection(cfg)
 	if err != nil {
 		return fmt.Errorf("failed to initialize orm: %v", err)

core/services/health.go

@@ -32,37 +32,40 @@ type Checker interface {
 	Close() error
 }
 
-type InBackupHealthReport struct {
+type StartUpHealthReport struct {
 	server http.Server
 	lggr   logger.Logger
+	mux    *http.ServeMux
 }
 
-func (i *InBackupHealthReport) Stop() {
+func (i *StartUpHealthReport) Stop() {
 	shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), time.Second)
 	defer shutdownRelease()
 	if err := i.server.Shutdown(shutdownCtx); err != nil {
-		i.lggr.Errorf("InBackupHealthReport shutdown error: %v", err)
+		i.lggr.Errorf("StartUpHealthReport shutdown error: %v", err)
 	}
-	i.lggr.Info("InBackupHealthReport shutdown complete")
+	i.lggr.Info("StartUpHealthReport shutdown complete")
 }
 
-func (i *InBackupHealthReport) Start() {
+func (i *StartUpHealthReport) Start() {
 	go func() {
-		http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		i.mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusNoContent)
 		})
-		i.lggr.Info("Starting InBackupHealthReport")
+		i.lggr.Info("Starting StartUpHealthReport")
 		if err := i.server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
-			i.lggr.Errorf("InBackupHealthReport server error: %v", err)
+			i.lggr.Errorf("StartUpHealthReport server error: %v", err)
 		}
 	}()
 }
 
-// NewInBackupHealthReport creates a new InBackupHealthReport that will serve the /health endpoint, useful for
-// preventing shutdowns due to health-checks when running long backup tasks
-func NewInBackupHealthReport(port uint16, lggr logger.Logger) *InBackupHealthReport {
-	return &InBackupHealthReport{
-		server: http.Server{Addr: fmt.Sprintf(":%d", port), ReadHeaderTimeout: time.Second * 5},
+// NewStartUpHealthReport creates a new StartUpHealthReport that will serve the /health endpoint, useful for
+// preventing shutdowns due to health-checks when running long backup tasks or migrations
+func NewStartUpHealthReport(port uint16, lggr logger.Logger) *StartUpHealthReport {
+	mux := http.NewServeMux()
+	return &StartUpHealthReport{
 		lggr:   lggr,
+		mux:    mux,
+		server: http.Server{Addr: fmt.Sprintf(":%d", port), ReadHeaderTimeout: time.Second * 5, Handler: mux},
 	}
 }

core/services/health_test.go

@@ -13,13 +13,13 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services"
 )
 
-func TestNewInBackupHealthReport(t *testing.T) {
+func TestNewStartUpHealthReport(t *testing.T) {
 	lggr, observed := logger.TestLoggerObserved(t, zapcore.InfoLevel)
-	ibhr := services.NewInBackupHealthReport(1234, lggr)
+	ibhr := services.NewStartUpHealthReport(1234, lggr)
 
 	ibhr.Start()
 	require.Eventually(t, func() bool { return observed.Len() >= 1 }, time.Second*5, time.Millisecond*100)
-	require.Equal(t, "Starting InBackupHealthReport", observed.TakeAll()[0].Message)
+	require.Equal(t, "Starting StartUpHealthReport", observed.TakeAll()[0].Message)
 
 	req, err := http.NewRequestWithContext(tests.Context(t), "GET", "http://localhost:1234/health", nil)
 	require.NoError(t, err)
@@ -29,5 +29,5 @@ func TestNewInBackupHealthReport(t *testing.T) {
 
 	ibhr.Stop()
 	require.Eventually(t, func() bool { return observed.Len() >= 1 }, time.Second*5, time.Millisecond*100)
-	require.Equal(t, "InBackupHealthReport shutdown complete", observed.TakeAll()[0].Message)
+	require.Equal(t, "StartUpHealthReport shutdown complete", observed.TakeAll()[0].Message)
 }