fix(HMS-4864): flush cloudwatch logs (PR #410 redux)

cmd/db-tool/main.go

@@ -12,5 +12,7 @@ func main() {
 	logger.LogBuildInfo(component)
 	cfg := config.Get()
 	logger.InitLogger(cfg, component)
+	defer logger.DoneLogger()
+
 	cmd.Execute()
 }

cmd/mock-rbac/main.go

@@ -34,6 +34,8 @@ func main() {
 
 	cfg := config.Get()
 	logger.InitLogger(cfg, component)
+	defer logger.DoneLogger()
+
 	if cfg.Clients.RbacBaseURL == "" {
 		panic("'RbacBaseURL' is empty")
 	}

cmd/pendo-client/main.go

@@ -93,6 +93,8 @@ func initCLI() *cobra.Command {
 
 func main() {
 	rootCmd := initCLI()
+	defer logger.DoneLogger()
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
 	}

cmd/service/main.go

@@ -49,6 +49,8 @@ func main() {
 	logger.LogBuildInfo(component)
 	cfg := config.Get()
 	logger.InitLogger(cfg, component)
+	defer logger.DoneLogger()
+
 	db := datastore.NewDB(cfg)
 	defer datastore.Close(db)
 

internal/infrastructure/logger/init.go

@@ -3,6 +3,7 @@ package logger
 import (
 	"context"
 	"fmt"
+	"io"
 	"log/slog"
 	"os"
 	"runtime/debug"
@@ -36,6 +37,11 @@ type Clonable interface {
 	Clone() interface{}
 }
 
+var (
+	cloudwatchCloser io.Closer    = nil
+	oldSlog          *slog.Logger = nil
+)
+
 // Early logging setup so we can use slog, this will just log to stderr.
 // This will change once the configuration has been parsed and we setup the
 // logger accordingly.
@@ -45,6 +51,8 @@ func init() {
 }
 
 func InitLogger(cfg *config.Config, componentName string) {
+	var ok bool
+
 	if cfg == nil {
 		panic("'cfg' cannot be nil")
 	}
@@ -122,6 +130,9 @@ func InitLogger(cfg *config.Config, componentName string) {
 			w,
 			&opts,
 		)
+		if cloudwatchCloser, ok = w.(io.Closer); !ok {
+			slog.Warn("cloudwatchCloser is nil; log output could stay un-flushed and be lost")
+		}
 		metaHandler.Add(cloudwatch_handler)
 	} else if cfg.Logging.Console {
 		h := slog.NewTextHandler(
@@ -136,6 +147,11 @@ func InitLogger(cfg *config.Config, componentName string) {
 		)
 		metaHandler.Add(h)
 	}
+	if oldSlog == nil {
+		oldSlog = slog.Default()
+	} else {
+		slog.Warn("InitLogger called twice")
+	}
 	slog.SetDefault(slog.New(metaHandler))
 
 	// set global log level
@@ -204,3 +220,14 @@ func LogBuildInfo(msg string) {
 		slog.Bool("Dirty", dirty),
 	)
 }
+
+// DoneLogger should be called before exiting any process that
+// invokes InitLogger.  e.g. via `defer logger.DoneLogger()`.
+func DoneLogger() {
+	slog.SetDefault(oldSlog)
+	if cloudwatchCloser != nil {
+		if err := cloudwatchCloser.Close(); err != nil {
+			slog.Error(err.Error())
+		}
+	}
+}

internal/test/smoke/suite_base_test.go

@@ -169,6 +169,7 @@ func (s *SuiteBase) TearDownTest() {
 	s.svcRbac.Stop()
 	s.svc.Stop()
 	s.wg.Wait()
+	logger.DoneLogger()
 }
 
 // As is a helper function that makes it easy to select