APPS-1349 Review and update default configuration values (#269)

* set default parallel

* updagte tests

* tps and bandwitdh

* timeout

* timeouts

* bandwidth

* use util.ValueOrZero

* set file limit default to 0

* set total timeout

* max retries

* update doc

* clean test

* reformat

* set from empty for full backups

docs/docs.go

@@ -1565,7 +1565,7 @@ const docTemplate = `{
                     "example": 1
                 },
                 "from": {
-                    "description": "The lower time bound of backup entities in the ISO 8601 format (for incremental backups).",
+                    "description": "The lower time bound of backup entities in the ISO 8601 format (for incremental backups only).",
                     "type": "string",
                     "example": "2023-03-19T14:50:00Z"
                 },
@@ -1680,7 +1680,7 @@ const docTemplate = `{
                     "type": "boolean"
                 },
                 "socket-timeout": {
-                    "description": "Socket timeout in milliseconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.",
+                    "description": "Socket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.",
                     "type": "integer",
                     "example": 1000
                 },
@@ -2366,11 +2366,16 @@ const docTemplate = `{
                         "set2"
                     ]
                 },
-                "timeout": {
-                    "description": "Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.",
+                "socket-timeout": {
+                    "description": "Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.\nSocket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.",
                     "type": "integer",
                     "example": 1000
                 },
+                "total-timeout": {
+                    "description": "Total socket timeout in milliseconds. Default is 0, that is, no timeout.",
+                    "type": "integer",
+                    "example": 2000
+                },
                 "tps": {
                     "description": "Throttles read operations from the backup file(s) to not exceed the given number of transactions\nper second.",
                     "type": "integer",

docs/openapi.json

@@ -1748,7 +1748,7 @@
             "type" : "integer"
           },
           "from" : {
-            "description" : "The lower time bound of backup entities in the ISO 8601 format (for incremental backups).",
+            "description" : "The lower time bound of backup entities in the ISO 8601 format (for incremental backups only).",
             "example" : "2023-03-19T14:50:00Z",
             "type" : "string"
           },
@@ -1854,7 +1854,7 @@
             "type" : "boolean"
           },
           "socket-timeout" : {
-            "description" : "Socket timeout in milliseconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.",
+            "description" : "Socket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.",
             "example" : 1000,
             "type" : "integer"
           },
@@ -2448,11 +2448,16 @@
             },
             "type" : "array"
           },
-          "timeout" : {
-            "description" : "Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.",
+          "socket-timeout" : {
+            "description" : "Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.\nSocket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.",
             "example" : 1000,
             "type" : "integer"
           },
+          "total-timeout" : {
+            "description" : "Total socket timeout in milliseconds. Default is 0, that is, no timeout.",
+            "example" : 2000,
+            "type" : "integer"
+          },
           "tps" : {
             "description" : "Throttles read operations from the backup file(s) to not exceed the given number of transactions\nper second.",
             "example" : 4000,

docs/openapi.yaml

@@ -1249,7 +1249,7 @@ components:
           type: integer
         from:
           description: The lower time bound of backup entities in the ISO 8601 format
-            (for incremental backups).
+            (for incremental backups only).
           example: 2023-03-19T14:50:00Z
           type: string
         key:
@@ -1357,7 +1357,7 @@ components:
           type: boolean
         socket-timeout:
           description: |-
-            Socket timeout in milliseconds. If this value is 0, it is set to total-timeout.
+            Socket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.
             If both are 0, there is no socket idle time limit.
           example: 1000
           type: integer
@@ -2025,11 +2025,18 @@ components:
           items:
             type: string
           type: array
-        timeout:
-          description: "Timeout (ms) for Aerospike commands to write records, create\
-            \ indexes and create UDFs."
+        socket-timeout:
+          description: |-
+            Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.
+            Socket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.
+            If both are 0, there is no socket idle time limit.
           example: 1000
           type: integer
+        total-timeout:
+          description: "Total socket timeout in milliseconds. Default is 0, that is,\
+            \ no timeout."
+          example: 2000
+          type: integer
         tps:
           description: |-
             Throttles read operations from the backup file(s) to not exceed the given number of transactions

internal/server/handlers/restore_test.go

@@ -8,16 +8,16 @@ import (
 	"time"
 
 	"github.com/aerospike/aerospike-backup-service/v2/pkg/dto"
+	"github.com/aerospike/aerospike-backup-service/v2/pkg/util"
 	"github.com/aws/smithy-go/ptr"
 	"github.com/gorilla/mux"
 	"github.com/steinfletcher/apitest"
 	"github.com/stretchr/testify/require"
 )
 
 func testConfigRestorePolicy() *dto.RestorePolicy {
-	testIn32 := int32(10)
 	return &dto.RestorePolicy{
-		Parallel: &testIn32,
+		Parallel: util.Ptr(10),
 	}
 }
 

pkg/dto/backup_details.go

@@ -19,8 +19,8 @@ type BackupDetails struct {
 type BackupMetadata struct {
 	// The backup time in the ISO 8601 format.
 	Created time.Time `yaml:"created" json:"created" example:"2023-03-20T14:50:00Z"`
-	// The lower time bound of backup entities in the ISO 8601 format (for incremental backups).
-	From time.Time `yaml:"from" json:"from" example:"2023-03-19T14:50:00Z"`
+	// The lower time bound of backup entities in the ISO 8601 format (for incremental backups only).
+	From time.Time `yaml:"from,omitempty" json:"from,omitempty" example:"2023-03-19T14:50:00Z"`
 	// The namespace of a backup.
 	Namespace string `yaml:"namespace" json:"namespace" example:"testNamespace"`
 	// The total number of records backed up.

pkg/dto/backup_policy.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/aerospike/aerospike-backup-service/v2/pkg/model"
 )
@@ -25,7 +26,7 @@ type RemoveFilesType string
 type BackupPolicy struct {
 	// Maximum number of scan calls to run in parallel.
 	Parallel *int `yaml:"parallel,omitempty" json:"parallel,omitempty" example:"1"`
-	// Socket timeout in milliseconds. If this value is 0, it is set to total-timeout.
+	// Socket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.
 	// If both are 0, there is no socket idle time limit.
 	SocketTimeout *int32 `yaml:"socket-timeout,omitempty" json:"socket-timeout,omitempty" example:"1000"`
 	// Total socket timeout in milliseconds. Default is 0, that is, no timeout.
@@ -43,13 +44,13 @@ type BackupPolicy struct {
 	NoUdfs *bool `yaml:"no-udfs,omitempty" json:"no-udfs,omitempty"`
 	// Throttles backup write operations to the backup file(s) to not exceed the given
 	// bandwidth in MiB/s.
-	Bandwidth *int64 `yaml:"bandwidth,omitempty" json:"bandwidth,omitempty" example:"10000"`
+	Bandwidth *int `yaml:"bandwidth,omitempty" json:"bandwidth,omitempty" example:"10000"`
 	// Limit total returned records per second (RPS). If RPS is zero (the default),
 	// the records-per-second limit is not applied.
-	RecordsPerSecond *int32 `yaml:"records-per-second,omitempty" json:"records-per-second,omitempty" example:"1000"`
+	RecordsPerSecond *int `yaml:"records-per-second,omitempty" json:"records-per-second,omitempty" example:"1000"`
 	// File size limit (in MB) for the backup directory. If an .asb backup file crosses this size threshold,
 	// a new backup file will be created.
-	FileLimit *int64 `yaml:"file-limit,omitempty" json:"file-limit,omitempty" example:"1024"`
+	FileLimit *int `yaml:"file-limit,omitempty" json:"file-limit,omitempty" example:"1024"`
 	// Encryption details.
 	EncryptionPolicy *EncryptionPolicy `yaml:"encryption,omitempty" json:"encryption,omitempty"`
 	// Compression details.
@@ -116,8 +117,8 @@ func (p *BackupPolicy) Validate() error {
 func (p *BackupPolicy) ToModel() *model.BackupPolicy {
 	return &model.BackupPolicy{
 		Parallel:          p.Parallel,
-		SocketTimeout:     p.SocketTimeout,
-		TotalTimeout:      p.TotalTimeout,
+		SocketTimeout:     millisToDuration(p.SocketTimeout),
+		TotalTimeout:      millisToDuration(p.TotalTimeout),
 		RetryPolicy:       p.RetryPolicy.ToModel(),
 		RemoveFiles:       (*model.RemoveFilesType)(p.RemoveFiles),
 		NoRecords:         p.NoRecords,
@@ -132,6 +133,22 @@ func (p *BackupPolicy) ToModel() *model.BackupPolicy {
 	}
 }
 
+func millisToDuration(ms *int32) *time.Duration {
+	if ms == nil {
+		return nil
+	}
+	duration := time.Duration(*ms) * time.Millisecond
+	return &duration
+}
+
+func durationToMillis(d *time.Duration) *int32 {
+	if d == nil {
+		return nil
+	}
+	ms := int32((*d) / time.Millisecond)
+	return &ms
+}
+
 func NewBackupPolicyFromModel(m *model.BackupPolicy) *BackupPolicy {
 	if m == nil {
 		return nil
@@ -144,8 +161,8 @@ func NewBackupPolicyFromModel(m *model.BackupPolicy) *BackupPolicy {
 
 func (p *BackupPolicy) fromModel(m *model.BackupPolicy) {
 	p.Parallel = m.Parallel
-	p.SocketTimeout = m.SocketTimeout
-	p.TotalTimeout = m.TotalTimeout
+	p.SocketTimeout = durationToMillis(m.SocketTimeout)
+	p.TotalTimeout = durationToMillis(m.TotalTimeout)
 	p.RetryPolicy = newRetryPolicyFromModel(m.RetryPolicy)
 	p.RemoveFiles = (*RemoveFilesType)(m.RemoveFiles)
 	p.NoRecords = m.NoRecords

pkg/dto/restore_policy.go

@@ -11,7 +11,7 @@ import (
 // @Description RestorePolicy represents a policy for the restore operation.
 type RestorePolicy struct {
 	// The number of concurrent record readers from backup files.
-	Parallel *int32 `json:"parallel,omitempty" example:"8"`
+	Parallel *int `json:"parallel,omitempty" example:"8"`
 	// Do not restore any record data (metadata or bin data).
 	// By default, record data, secondary index definitions, and UDF modules
 	// will be restored.
@@ -21,15 +21,19 @@ type RestorePolicy struct {
 	// Do not restore any UDF modules.
 	NoUdfs *bool `json:"no-udfs,omitempty"`
 	// Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.
-	Timeout *int32 `json:"timeout,omitempty" example:"1000"`
+	// Socket timeout in milliseconds. Default is 10 seconds. If this value is 0, it is set to total-timeout.
+	// If both are 0, there is no socket idle time limit.
+	SocketTimeout *int32 `yaml:"socket-timeout,omitempty" json:"socket-timeout,omitempty" example:"1000"`
+	// Total socket timeout in milliseconds. Default is 0, that is, no timeout.
+	TotalTimeout *int32 `yaml:"total-timeout,omitempty" json:"total-timeout,omitempty" example:"2000"`
 	// Disables the use of batch writes when restoring records to the Aerospike cluster.
 	// By default, the cluster is checked for batch write support.
 	DisableBatchWrites *bool `json:"disable-batch-writes,omitempty"`
 	// The max number of outstanding async record batch write calls at a time.
-	MaxAsyncBatches *int32 `json:"max-async-batches,omitempty" example:"32"`
+	MaxAsyncBatches *int `json:"max-async-batches,omitempty" example:"32"`
 	// The max allowed number of records per an async batch write call.
 	// Default is 128 with batch writes enabled, or 16 without batch writes.
-	BatchSize *int32 `json:"batch-size,omitempty" example:"128"`
+	BatchSize *int `json:"batch-size,omitempty" example:"128"`
 	// Namespace details for the restore operation.
 	// By default, the data is restored to the namespace from which it was taken.
 	Namespace *RestoreNamespace `json:"namespace,omitempty"`
@@ -50,10 +54,10 @@ type RestorePolicy struct {
 	// the namespace, regardless of generation numbers.
 	NoGeneration *bool `json:"no-generation,omitempty"`
 	// Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth in bytes/sec.
-	Bandwidth *int64 `json:"bandwidth,omitempty" example:"50000"`
+	Bandwidth *int `json:"bandwidth,omitempty" example:"50000"`
 	// Throttles read operations from the backup file(s) to not exceed the given number of transactions
 	// per second.
-	Tps *int32 `json:"tps,omitempty" example:"4000"`
+	Tps *int `json:"tps,omitempty" example:"4000"`
 	// Encryption details.
 	EncryptionPolicy *EncryptionPolicy `yaml:"encryption,omitempty" json:"encryption,omitempty"`
 	// Compression details.
@@ -74,8 +78,11 @@ func (p *RestorePolicy) Validate() error {
 	if p.Parallel != nil && *p.Parallel <= 0 {
 		return fmt.Errorf("parallel %d invalid, should be positive number", *p.Parallel)
 	}
-	if p.Timeout != nil && *p.Timeout <= 0 {
-		return fmt.Errorf("timeout %d invalid, should be positive number", *p.Timeout)
+	if p.TotalTimeout != nil && *p.TotalTimeout <= 0 {
+		return fmt.Errorf("total timeout %d invalid, should be positive number", *p.TotalTimeout)
+	}
+	if p.SocketTimeout != nil && *p.SocketTimeout <= 0 {
+		return fmt.Errorf("socket timeout %d invalid, should be positive number", *p.SocketTimeout)
 	}
 	if p.MaxAsyncBatches != nil && *p.MaxAsyncBatches <= 0 {
 		return fmt.Errorf("maxAsyncBatches %d invalid, should be positive number", *p.MaxAsyncBatches)
@@ -123,7 +130,8 @@ func (p *RestorePolicy) ToModel() *model.RestorePolicy {
 		NoRecords:          p.NoRecords,
 		NoIndexes:          p.NoIndexes,
 		NoUdfs:             p.NoUdfs,
-		Timeout:            p.Timeout,
+		TotalTimeout:       millisToDuration(p.TotalTimeout),
+		SocketTimeout:      millisToDuration(p.SocketTimeout),
 		DisableBatchWrites: p.DisableBatchWrites,
 		MaxAsyncBatches:    p.MaxAsyncBatches,
 		BatchSize:          p.BatchSize,

pkg/model/backup_policy.go

@@ -1,6 +1,8 @@
 package model
 
 import (
+	"time"
+
 	"github.com/aerospike/aerospike-backup-service/v2/pkg/util"
 	"github.com/aerospike/backup-go/models"
 )
@@ -21,9 +23,9 @@ type BackupPolicy struct {
 	Parallel *int
 	// Socket timeout in milliseconds. If this value is 0, it is set to total-timeout.
 	// If both are 0, there is no socket idle time limit.
-	SocketTimeout *int32
+	SocketTimeout *time.Duration
 	// Total socket timeout in milliseconds. Default is 0, that is, no timeout.
-	TotalTimeout *int32
+	TotalTimeout *time.Duration
 	// RetryPolicy defines the configuration for retry attempts in case of failures.
 	RetryPolicy *models.RetryPolicy
 	// Whether to clear the output directory (default: KeepAll).
@@ -36,13 +38,13 @@ type BackupPolicy struct {
 	NoUdfs *bool
 	// Throttles backup write operations to the backup file(s) to not exceed the given
 	// bandwidth in MiB/s.
-	Bandwidth *int64
+	Bandwidth *int
 	// Limit total returned records per second (RPS). If RPS is zero (the default),
 	// the records-per-second limit is not applied.
-	RecordsPerSecond *int32
-	// File size limit (in MB) for the backup directory. If an .asb backup file crosses this size threshold,
+	RecordsPerSecond *int
+	// File size limit (in MiB) for the backup directory. If an .asb backup file crosses this size threshold,
 	// a new backup file will be created.
-	FileLimit *int64
+	FileLimit *int
 	// Encryption details.
 	EncryptionPolicy *EncryptionPolicy
 	// Compression details.
@@ -53,13 +55,13 @@ type BackupPolicy struct {
 	Sealed *bool
 }
 
-// IsSealed returns the value of the Sealed property.
+// IsSealedOrDefault returns the value of the Sealed property.
 // If the property is not set, it returns the default value.
-func (p *BackupPolicy) IsSealed() bool {
+func (p *BackupPolicy) IsSealedOrDefault() bool {
 	if p.Sealed != nil {
 		return *p.Sealed
 	}
-	return defaultConfig.backupPolicy.sealed
+	return defaultConfig.backupPolicy.Sealed != nil && *defaultConfig.backupPolicy.Sealed
 }
 
 // CopySMDDisabled creates a new instance of the BackupPolicy struct with identical field values.
@@ -82,11 +84,48 @@ func (p *BackupPolicy) CopySMDDisabled() *BackupPolicy {
 }
 
 func (p *BackupPolicy) GetRetryPolicyOrDefault() models.RetryPolicy {
-	if p.RetryPolicy == nil {
-		return defaultConfig.backupPolicy.retryPolicy
+	if p.RetryPolicy != nil {
+		return *p.RetryPolicy
+	}
+
+	return *defaultConfig.backupPolicy.RetryPolicy
+}
+
+func (p *BackupPolicy) GetParallelOrDefault() int {
+	if p.Parallel != nil {
+		return *p.Parallel
+	}
+
+	return *defaultConfig.backupPolicy.Parallel
+}
+
+func (p *BackupPolicy) GetFileLimitOrDefault() int {
+	if p.FileLimit != nil {
+		return *p.FileLimit
+	}
+
+	return *defaultConfig.backupPolicy.FileLimit
+}
+
+func (p *BackupPolicy) GetTotalTimeoutOrDefault() time.Duration {
+	if p.TotalTimeout != nil {
+		return *p.TotalTimeout
+	}
+
+	return *defaultConfig.backupPolicy.TotalTimeout
+}
+
+func (p *BackupPolicy) GetSocketTimeoutOrDefault() time.Duration {
+	if p.SocketTimeout == nil {
+		return *defaultConfig.backupPolicy.SocketTimeout
+	}
+
+	// If this value is 0, its set to total-timeout.
+	if *p.SocketTimeout == 0 {
+		return p.GetTotalTimeoutOrDefault()
 	}
 
-	return *p.RetryPolicy
+	return *p.SocketTimeout
 }
 
 func (r *RemoveFilesType) RemoveFullBackup() bool {