Merge branch 'useGoBackupLib' into APPS-1237-backup-service-secret-agent

# Conflicts:
#	pkg/shared/backup_go.go
#	pkg/shared/restore_go.go

.golangci.yml

@@ -9,6 +9,10 @@ run:
 linters:
   disable-all: true
   enable:
+    - gci
+    - bodyclose
+    - unused
+    - dogsled
     - dupl
     - errcheck
     - exportloopref
@@ -19,21 +23,30 @@ linters:
     - gocyclo
     - gofmt
     - goimports
+    - gocyclo
+    - gosec
     - gosimple
     - govet
     - ineffassign
-    - lll
     - misspell
     - prealloc
     - promlinter
+    - nolintlint
+    - nakedret
+    - prealloc # pre-allocate slices with define size if the slice size is known in advance
+    - predeclared
     - revive
     - sloglint
     - staticcheck
     - stylecheck
+    - thelper
+    - tparallel
     - typecheck
     - unconvert
     - unparam
     - unused
+    - whitespace
+    - lll
 
 issues:
   exclude-rules:

cmd/backup/main.go

@@ -27,7 +27,7 @@ var (
 
 // run parses the CLI parameters and executes backup.
 //
-//nolint:funlen
+
 func run() int {
 	var (
 		configFile string

go.mod

@@ -5,7 +5,7 @@ go 1.22
 require (
 	github.com/aerospike/aerospike-client-go/v7 v7.6.0
 	github.com/aerospike/aerospike-management-lib v1.4.0
-	github.com/aerospike/backup-go v0.0.0-20240728070104-6c6b315f52d8
+	github.com/aerospike/backup-go v0.0.0-20240730103930-5ef82c03f86f
 	github.com/aws/aws-sdk-go-v2 v1.30.3
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2

go.sum

@@ -218,6 +218,10 @@ github.com/aerospike/backup-go v0.0.0-20240728070104-6c6b315f52d8 h1:novGz78Gq6E
 github.com/aerospike/backup-go v0.0.0-20240728070104-6c6b315f52d8/go.mod h1:WfFo03sY4w5qu3IALjqNQaPZVe+0p7GqYPG+qifkm+8=
 github.com/aerospike/backup-go v0.0.0-20240729112203-dee1dc63b0c9 h1:fSXsqkD5XWCdDn8NfKnQVevYHTMgxtQ8c4twYAo+2Go=
 github.com/aerospike/backup-go v0.0.0-20240729112203-dee1dc63b0c9/go.mod h1:WfFo03sY4w5qu3IALjqNQaPZVe+0p7GqYPG+qifkm+8=
+github.com/aerospike/backup-go v0.0.0-20240730084743-64d81fa5d7a4 h1:YvY6Gyedc3h6qX4tKaFaERqK0CjoHTMwgTvnffWig14=
+github.com/aerospike/backup-go v0.0.0-20240730084743-64d81fa5d7a4/go.mod h1:WfFo03sY4w5qu3IALjqNQaPZVe+0p7GqYPG+qifkm+8=
+github.com/aerospike/backup-go v0.0.0-20240730103930-5ef82c03f86f h1:7gVhto19JTcDKBG2cMnEP3I+ktr+05ny32NjtaLvBH4=
+github.com/aerospike/backup-go v0.0.0-20240730103930-5ef82c03f86f/go.mod h1:WfFo03sY4w5qu3IALjqNQaPZVe+0p7GqYPG+qifkm+8=
 github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
 github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
 github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo=

internal/server/backup_handlers.go

@@ -133,7 +133,6 @@ func readBackupsLogic(routines map[string]*model.BackupRoutine,
 	backends service.BackendsHolder,
 	timeBounds *model.TimeBounds,
 	isFullBackup bool) (map[string][]model.BackupDetails, error) {
-
 	result := make(map[string][]model.BackupDetails)
 	for routine := range routines {
 		reader, _ := backends.GetReader(routine)
@@ -149,7 +148,6 @@ func readBackupsLogic(routines map[string]*model.BackupRoutine,
 
 func backupsReadFunction(
 	backend service.BackupListReader, fullBackup bool) func(*model.TimeBounds) ([]model.BackupDetails, error) {
-
 	if fullBackup {
 		return backend.FullBackupList
 	}

internal/server/config_cluster_handlers.go

@@ -118,7 +118,7 @@ func (ws *HTTPServer) readAerospikeCluster(w http.ResponseWriter, r *http.Reques
 // @Success     200
 // @Failure     400 {string} string
 //
-//nolint:dupl
+
 func (ws *HTTPServer) updateAerospikeCluster(w http.ResponseWriter, r *http.Request) {
 	var updatedCluster model.AerospikeCluster
 	err := json.NewDecoder(r.Body).Decode(&updatedCluster)

internal/server/config_handlers.go

@@ -1,4 +1,3 @@
-//nolint:dupl
 package server
 
 import (
@@ -69,10 +68,11 @@ func (ws *HTTPServer) updateConfig(w http.ResponseWriter, r *http.Request) {
 // @Success     200
 // @Failure     400 {string} string
 func (ws *HTTPServer) applyConfig(w http.ResponseWriter, _ *http.Request) {
-	err := service.ApplyNewConfig(ws.scheduler, ws.config, ws.backupBackends)
+	handlers, err := service.ApplyNewConfig(ws.scheduler, ws.config, ws.backupBackends)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+	ws.handlerHolder = handlers
 	w.WriteHeader(http.StatusOK)
 }

internal/server/server.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"net/netip"
 	"strings"
+	"time"
 
 	"github.com/aerospike/backup/pkg/model"
 	"github.com/aerospike/backup/pkg/service"
@@ -104,7 +105,8 @@ func NewHTTPServer(
 	return &HTTPServer{
 		config: config,
 		server: &http.Server{
-			Addr: addr,
+			Addr:              addr,
+			ReadHeaderTimeout: 5 * time.Second,
 		},
 		rateLimiter:    rateLimiter,
 		whiteList:      newIPWhiteList(serverConfig.GetRateOrDefault().GetWhiteListOrDefault()),

pkg/model/aerospike_cluster.go

@@ -1,4 +1,4 @@
-// nolint:lll
+//nolint:lll
 package model
 
 import (
@@ -174,7 +174,6 @@ func initTLS(t *TLS, clusterLabel *string) *tls.Config {
 	var clientPool []tls.Certificate
 	if (t.Certfile != nil && len(*t.Certfile) > 0) ||
 		t.Keyfile != nil && len(*t.Keyfile) > 0 {
-
 		// Read cert file
 		certFileBytes, err := readFromFile(*t.Certfile)
 		if err != nil {
@@ -231,6 +230,7 @@ func initTLS(t *TLS, clusterLabel *string) *tls.Config {
 		RootCAs:                  serverPool,
 		InsecureSkipVerify:       false,
 		PreferServerCipherSuites: true,
+		MinVersion:               tls.VersionTLS12,
 	}
 
 	return tlsConfig

pkg/model/config.go

@@ -80,7 +80,7 @@ func (c *Config) Validate() error {
 		return err
 	}
 
-	if err := c.ServiceConfig.Logger.Validate(); err != nil { //nolint:revive
+	if err := c.ServiceConfig.Logger.Validate(); err != nil {
 		return err
 	}
 

pkg/model/restore_request.go

@@ -79,7 +79,7 @@ func (r *RestoreRequest) Validate() error {
 	if err := r.SourceStorage.Validate(); err != nil {
 		return err
 	}
-	if err := r.Policy.Validate(); err != nil { //nolint:revive
+	if err := r.Policy.Validate(); err != nil {
 		return err
 	}
 	return nil

pkg/service/backup_handler.go

@@ -139,7 +139,7 @@ func (h *BackupHandler) startFullBackupForAllNamespaces(
 		backupFolder := getFullPath(h.backend.fullBackupsPath, h.backupFullPolicy, namespace, upperBound)
 		backupPath := h.backend.wrapWithPrefix(backupFolder)
 		handler, err := backupService.BackupRun(ctx, h.backupRoutine, h.backupFullPolicy, client,
-			h.storage, h.secretAgent, timebounds, &namespace, backupPath)
+			h.storage, h.secretAgent, timebounds, namespace, backupPath)
 		if err != nil {
 			backupFailureCounter.Inc()
 			return fmt.Errorf("could not start backup of namespace %s, routine %s: %w", namespace, h.routineName, err)
@@ -259,7 +259,6 @@ func (h *BackupHandler) runIncrementalBackup(ctx context.Context, now time.Time)
 
 func (h *BackupHandler) startIncrementalBackupForAllNamespaces(
 	ctx context.Context, client *aerospike.Client, upperBound time.Time) {
-
 	timebounds := model.NewTimeBoundsFrom(h.state.LastRun())
 	if h.backupFullPolicy.IsSealed() {
 		timebounds.ToTime = &upperBound
@@ -270,7 +269,7 @@ func (h *BackupHandler) startIncrementalBackupForAllNamespaces(
 		backupFolder := getIncrementalPath(h.backend.incrementalBackupsPath, namespace, upperBound)
 		backupPath := h.backend.wrapWithPrefix(backupFolder)
 		handler, err := backupService.BackupRun(ctx,
-			h.backupRoutine, h.backupIncrPolicy, client, h.storage, h.secretAgent, *timebounds, &namespace, backupPath)
+			h.backupRoutine, h.backupIncrPolicy, client, h.storage, h.secretAgent, *timebounds, namespace, backupPath)
 		if err != nil {
 			incrBackupFailureCounter.Inc()
 			slog.Warn("could not start backup", "namespace", namespace, "routine", h.routineName, "err", err)

pkg/service/backup_scheduler.go

@@ -43,15 +43,22 @@ func NewAdHocFullBackupJobForRoutine(name string) *quartz.JobDetail {
 	return quartz.NewJobDetail(job.Job(), jobKey)
 }
 
-func ApplyNewConfig(scheduler quartz.Scheduler, config *model.Config, backends BackendsHolder) error {
+func ApplyNewConfig(scheduler quartz.Scheduler, config *model.Config, backends BackendsHolder,
+) (BackupHandlerHolder, error) {
 	err := scheduler.Clear()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	backends.SetData(BuildBackupBackends(config))
 
-	return scheduleRoutines(scheduler, config, MakeHandlers(config, backends))
+	handlers := MakeHandlers(config, backends)
+	err = scheduleRoutines(scheduler, config, handlers)
+	if err != nil {
+		return nil, err
+	}
+
+	return handlers, nil
 }
 
 // ScheduleBackup creates a new quartz.Scheduler, schedules all the configured backup jobs,

pkg/service/configuration_manager.go

@@ -23,6 +23,7 @@ type Reader interface {
 type HTTPReader struct{}
 
 func (h HTTPReader) read(url string) ([]byte, error) {
+	// #nosec G107
 	resp, err := http.Get(url)
 	if err != nil {
 		return nil, err

pkg/service/configuration_manager_local.go

@@ -63,7 +63,7 @@ func (cm *FileConfigurationManager) WriteConfiguration(config *model.Config) err
 		return fmt.Errorf("failed to marshal configuration data: %w", err)
 	}
 
-	err = os.WriteFile(filePath, data, 0644)
+	err = os.WriteFile(filePath, data, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to write configuration to file %q: %w", filePath, err)
 	}

pkg/service/configuration_service_policy_test.go

@@ -39,10 +39,10 @@ func TestPolicy_AddErrors(t *testing.T) {
 		Storage:           map[string]*model.Storage{storage: {}},
 	}
 
-	for _, testPolicy := range fails {
-		err := AddPolicy(config, policy, &testPolicy.policy)
+	for i := range fails {
+		err := AddPolicy(config, policy, &fails[i].policy)
 		if err == nil {
-			t.Errorf("Expected an error on %s", testPolicy.name)
+			t.Errorf("Expected an error on %s", fails[i].name)
 		}
 	}
 }

pkg/service/configuration_service_routine_test.go

@@ -56,10 +56,10 @@ func TestRoutine_AddErrors(t *testing.T) {
 		Storage:           map[string]*model.Storage{storage: {}},
 	}
 
-	for _, testRoutine := range fails {
-		err := AddRoutine(config, routine, &testRoutine.routine)
+	for i := range fails {
+		err := AddRoutine(config, routine, &fails[i].routine)
 		if err == nil {
-			t.Errorf("Expected an error on %s", testRoutine.name)
+			t.Errorf("Expected an error on %s", fails[i].name)
 		}
 	}
 }

pkg/service/jobs_holder.go

@@ -20,6 +20,7 @@ func NewJobsHolder() *JobsHolder {
 }
 
 func (h *JobsHolder) newJob() int {
+	// #nosec G404
 	jobID := rand.Int()
 	h.Lock()
 	defer h.Unlock()
@@ -38,21 +39,21 @@ func (h *JobsHolder) getStatus(jobID int) (*model.RestoreJobStatus, error) {
 	return &copyJob, nil
 }
 
-func (h *JobsHolder) increaseStats(jobID int, new *model.RestoreResult) {
+func (h *JobsHolder) increaseStats(jobID int, newStats *model.RestoreResult) {
 	h.Lock()
 	defer h.Unlock()
 	current, found := h.restoreJobs[jobID]
 	if found {
-		current.TotalBytes += new.TotalBytes
-		current.TotalRecords += new.TotalRecords
-		current.ExpiredRecords += new.ExpiredRecords
-		current.SkippedRecords += new.SkippedRecords
-		current.IgnoredRecords += new.IgnoredRecords
-		current.InsertedRecords += new.InsertedRecords
-		current.ExistedRecords += new.ExistedRecords
-		current.FresherRecords += new.FresherRecords
-		current.IndexCount += new.IndexCount
-		current.UDFCount += new.UDFCount
+		current.TotalBytes += newStats.TotalBytes
+		current.TotalRecords += newStats.TotalRecords
+		current.ExpiredRecords += newStats.ExpiredRecords
+		current.SkippedRecords += newStats.SkippedRecords
+		current.IgnoredRecords += newStats.IgnoredRecords
+		current.InsertedRecords += newStats.InsertedRecords
+		current.ExistedRecords += newStats.ExistedRecords
+		current.FresherRecords += newStats.FresherRecords
+		current.IndexCount += newStats.IndexCount
+		current.UDFCount += newStats.UDFCount
 	}
 }
 

pkg/service/os_accessor.go

@@ -75,7 +75,7 @@ func (o *OSDiskAccessor) write(filePath string, data []byte) error {
 		return err
 	}
 
-	return os.WriteFile(filePath, data, 0644)
+	return os.WriteFile(filePath, data, 0600)
 }
 
 func (o *OSDiskAccessor) lsDir(path string) ([]string, error) {

pkg/service/os_accessor_test.go

@@ -19,7 +19,7 @@ func TestDeleteFolder(t *testing.T) {
 	parentFolder := tempFolder + "/parent"
 	folderToDelete := parentFolder + "/nested"
 	_ = os.MkdirAll(folderToDelete, 0744)
-	_ = os.WriteFile(folderToDelete+"/file.txt", []byte("hello world"), 0666)
+	_ = os.WriteFile(folderToDelete+"/file.txt", []byte("hello world"), 0600)
 
 	err := NewOSDiskAccessor().DeleteFolder(folderToDelete)
 
@@ -70,7 +70,7 @@ func TestLsDir(t *testing.T) {
 			setup: func() string {
 				dir := t.TempDir()
 				file := filepath.Join(dir, "file")
-				_ = os.WriteFile(file, []byte("test content"), 0644)
+				_ = os.WriteFile(file, []byte("test content"), 0600)
 				return dir
 			},
 			expected: nil,
@@ -124,7 +124,7 @@ func TestLsFiles(t *testing.T) {
 			setup: func() string {
 				dir := t.TempDir()
 				file := filepath.Join(dir, "file")
-				_ = os.WriteFile(file, []byte("test content"), 0644)
+				_ = os.WriteFile(file, []byte("test content"), 0600)
 				return dir
 			},
 			expected: []string{"file"},
@@ -220,7 +220,6 @@ func TestReadBackupDetails(t *testing.T) {
 }
 
 func TestReadBackupDetailsNegative(t *testing.T) {
-
 	accessor := &OSDiskAccessor{}
 	tests := []struct {
 		name  string